Merge branch 'master' into realtime-ui

changedetectionio/__init__.py

@@ -4,21 +4,17 @@
 
 __version__ = '0.49.17'
 
-# Set environment variables before importing other modules
-import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
-# Import eventlet for WSGI server - no monkey patching to avoid conflicts
-import eventlet
-
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
+import os
+os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+import eventlet
 import eventlet.wsgi
 import getopt
 import platform
 import signal
-import socket  # Make sure socket is imported at the module level
+import socket
 import sys
-from flask import request
 
 from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
@@ -145,21 +141,7 @@ def main():
         logger.critical(str(e))
         return
 
-    # Get the Flask app 
     app = changedetection_app(app_config, datastore)
-    
-    # Initialize Socket.IO integrated with the main Flask app
-    try:
-        from changedetectionio.realtime.socket_server import init_socketio
-
-        # Initialize Socket.IO with the main Flask app
-        # This will be used later when we run the app with socketio.run()
-        socketio = init_socketio(app, datastore)
-        app.config['SOCKETIO'] = socketio
-
-        logger.info("Socket.IO server initialized successfully (integrated with main app)")
-    except Exception as e:
-        logger.warning(f"Failed to initialize Socket.IO server: {str(e)}")
 
     signal.signal(signal.SIGTERM, sigshutdown_handler)
     signal.signal(signal.SIGINT, sigshutdown_handler)
@@ -186,9 +168,6 @@ def main():
 
     @app.context_processor
     def inject_version():
-        # Socket.IO is now integrated with the main app
-        # The client will automatically connect to the Socket.IO endpoint on the same host/port
-
         return dict(right_sticky="v{}".format(datastore.data['version_tag']),
                     new_version_available=app.config['NEW_VERSION_AVAILABLE'],
                     has_password=datastore.data['settings']['application']['password'] != False
@@ -225,20 +204,5 @@ def main():
                                                server_side=True), app)
 
     else:
-        # Run the app with Socket.IO's integrated server
-        if 'SOCKETIO' in app.config:
-            # When using eventlet or threading, we need to make sure the host is valid
-            # Use '0.0.0.0' for all interfaces if host is empty or invalid
-            try:
-                socket_host = host if host else '0.0.0.0'
-                logger.info(f"Starting integrated Socket.IO server on http://{socket_host}:{port}")
-                app.config['SOCKETIO'].run(app, host=socket_host, port=int(port), debug=False, use_reloader=False, allow_unsafe_werkzeug=True)
-            except socket.gaierror:
-                # If the hostname is invalid, fall back to '0.0.0.0'
-                logger.warning(f"Invalid hostname '{host}', falling back to '0.0.0.0'")
-                app.config['SOCKETIO'].run(app, host='0.0.0.0', port=int(port), debug=False, use_reloader=False, allow_unsafe_werkzeug=True)
-        else:
-            # Fallback to eventlet if Socket.IO initialization failed
-            logger.info(f"Starting standard Flask server on http://{host}:{port}")
-            eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
 

changedetectionio/html_tools.py

@@ -309,10 +309,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         soup = BeautifulSoup(content, 'html.parser')
 
         if ensure_is_ldjson_info_type:
-            bs_result = soup.findAll('script', {"type": "application/ld+json"})
+            bs_result = soup.find_all('script', {"type": "application/ld+json"})
         else:
-            bs_result = soup.findAll('script')
-        bs_result += soup.findAll('body')
+            bs_result = soup.find_all('script')
+        bs_result += soup.find_all('body')
 
         bs_jsons = []
         for result in bs_result:

changedetectionio/tests/test_history_consistency.py

@@ -35,7 +35,8 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     )
     assert b"Settings updated." in res.data
 
-    wait_for_all_checks(client)
+
+    time.sleep(2)
 
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
 
@@ -47,14 +48,10 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"
     i=0
     # each one should have a history.txt containing just one line
-    i=0
     for w in json_obj['watching'].keys():
-        i+1
-#        res = client.get(url_for("watchlist.index"))
-#        with open('/tmp/debug.html', 'wb') as f:
-#            f.write(res.data)
+        i+=1
         history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
-        assert os.path.isfile(history_txt_index_file), f"history.txt for i: {i} should exist where I expect it at {history_txt_index_file}"
+        assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
 
         # Same like in model.Watch
         with open(history_txt_index_file, "r") as f:

changedetectionio/tests/util.py

@@ -126,6 +126,7 @@ def extract_UUID_from_client(client):
     uuid = m.group(1)
     return uuid.strip()
 
+
 def wait_for_all_checks(client=None):
     """
     Waits until the queue is empty and remains empty for at least `required_empty_duration` seconds,
@@ -136,15 +137,13 @@ def wait_for_all_checks(client=None):
 
     # Configuration
     attempt = 0
-
     i=0
-
     max_attempts = 60
-    wait_between_attempts = 1
-    required_empty_duration = 0.6
+    wait_between_attempts = 2
+    required_empty_duration = 2
 
     logger = logging.getLogger()
-    time.sleep(0.5)
+    time.sleep(1.2)
 
     empty_since = None
 
@@ -152,7 +151,7 @@ def wait_for_all_checks(client=None):
         q_length = global_update_q.qsize()
 
         # Check if any threads are still processing
-        time.sleep(wait_between_attempts)
+        time.sleep(1.2)
         any_threads_busy = any(t.current_uuid for t in running_update_threads)
 
 
@@ -172,10 +171,9 @@ def wait_for_all_checks(client=None):
                 busy_threads = [t.name for t in running_update_threads if t.current_uuid]
                 logger.info(f"Threads still busy: {busy_threads}, resetting timer.")
             empty_since = None
-
         attempt += 1
 
-    time.sleep(wait_between_attempts)
+    time.sleep(1)
 
 def live_server_setup(live_server):
 

requirements.txt

@@ -45,7 +45,7 @@ paho-mqtt!=2.0.*
 cryptography~=42.0.8
 
 # Used for CSS filtering
-beautifulsoup4
+beautifulsoup4>=4.0.0
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware