WIP

changedetectionio/__init__.py

@@ -4,11 +4,14 @@
 
 __version__ = '0.49.15'
 
-from changedetectionio.strtobool import strtobool
+# Set environment variables before importing other modules
-from json.decoder import JSONDecodeError
 import os
 os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+# Import eventlet for WSGI server - no monkey patching to avoid conflicts
 import eventlet
+
+from changedetectionio.strtobool import strtobool
+from json.decoder import JSONDecodeError
 import eventlet.wsgi
 import getopt
 import platform
@@ -141,7 +144,27 @@ def main():
         logger.critical(str(e))
         return
 
+    # Get the Flask app 
     app = changedetection_app(app_config, datastore)
+    
+    # Now initialize Socket.IO after the app is fully set up
+    try:
+        from changedetectionio.realtime.socket_server import ChangeDetectionSocketIO
+        from changedetectionio.flask_app import socketio_server
+        import threading
+
+        # Create the Socket.IO server
+        socketio_server = ChangeDetectionSocketIO(app, datastore)
+        
+        # Run the Socket.IO server in a separate thread on port 5005
+        socket_thread = threading.Thread(target=socketio_server.run, 
+                                         kwargs={'host': host, 'port': 5005})
+        socket_thread.daemon = True
+        socket_thread.start()
+        
+        logger.info("Socket.IO server initialized successfully on port 5005")
+    except Exception as e:
+        logger.warning(f"Failed to initialize Socket.IO server: {str(e)}")
 
     signal.signal(signal.SIGTERM, sigshutdown_handler)
     signal.signal(signal.SIGINT, sigshutdown_handler)
@@ -204,5 +227,7 @@ def main():
                                                server_side=True), app)
 
     else:
+        # We'll integrate the Socket.IO server with the WSGI server
+        # The Socket.IO server is already attached to the Flask app
         eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
 

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -102,7 +102,7 @@
 
                 {% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
                 {% set checking_now = is_checking_now(watch) %}
-            <tr id="{{ watch.uuid }}"
+            <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}

changedetectionio/flask_app.py

@@ -30,6 +30,7 @@ from flask_restful import abort, Api
 from flask_cors import CORS
 from flask_wtf import CSRFProtect
 from loguru import logger
+import eventlet
 
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
@@ -54,6 +55,9 @@ app = Flask(__name__,
             static_folder="static",
             template_folder="templates")
 
+# Will be initialized in changedetection_app
+socketio_server = None
+
 # Enable CORS, especially useful for the Chrome extension to operate from anywhere
 CORS(app)
 
@@ -215,7 +219,7 @@ class User(flask_login.UserMixin):
 def changedetection_app(config=None, datastore_o=None):
     logger.trace("TRACE log is enabled")
 
-    global datastore
+    global datastore, socketio_server
     datastore = datastore_o
 
     # so far just for read-only via tests, but this will be moved eventually to be the main source
@@ -467,6 +471,8 @@ def changedetection_app(config=None, datastore_o=None):
     if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')):
         threading.Thread(target=check_for_new_version).start()
 
+    # Return the Flask app - the Socket.IO will be attached to it but initialized separately
+    # This avoids circular dependencies
     return app
 
 

changedetectionio/realtime/__init__.py

@@ -0,0 +1,3 @@
+"""
+Socket.IO realtime updates module for changedetection.io
+"""

changedetectionio/realtime/socket_server.py

@@ -0,0 +1,126 @@
+from flask import Flask
+from flask_socketio import SocketIO
+import threading
+import json
+import time
+from loguru import logger
+
+class ChangeDetectionSocketIO:
+    def __init__(self, app, datastore):
+        self.main_app = app
+        self.datastore = datastore
+        
+        # Create a separate app for Socket.IO
+        self.app = Flask(__name__)
+        
+        # Use threading mode instead of eventlet
+        self.socketio = SocketIO(self.app, 
+                                async_mode='threading', 
+                                cors_allowed_origins="*",
+                                logger=False,
+                                engineio_logger=False)
+        
+        # Set up event handlers
+        self.socketio.on_event('connect', self.handle_connect)
+        self.socketio.on_event('disconnect', self.handle_disconnect)
+        
+        # Don't patch the update_watch method - this was causing issues
+        # Just start a background thread to periodically emit watch status
+        self.thread = None
+        self.thread_lock = threading.Lock()
+        
+        # Set up a simple index route for the Socket.IO app
+        @self.app.route('/')
+        def index():
+            return """
+            <html>
+                <head>
+                    <title>ChangeDetection.io Socket.IO Server</title>
+                </head>
+                <body>
+                    <h1>ChangeDetection.io Socket.IO Server</h1>
+                    <p>This is the Socket.IO server for ChangeDetection.io real-time updates.</p>
+                    <p>Socket.IO endpoint is available at: <code>/socket.io/</code></p>
+                </body>
+            </html>
+            """
+        
+    def start_background_task(self):
+        """Start the background task if it's not already running"""
+        with self.thread_lock:
+            if self.thread is None:
+                self.thread = threading.Thread(target=self.background_task)
+                self.thread.daemon = True
+                self.thread.start()
+                logger.info("Socket.IO: Started background task thread")
+    
+    def handle_connect(self):
+        """Handle client connection"""
+        logger.info("Socket.IO: Client connected")
+        
+        # Start the background task when the first client connects
+        self.start_background_task()
+    
+    def handle_disconnect(self):
+        """Handle client disconnection"""
+        logger.info("Socket.IO: Client disconnected")
+    
+    def background_task(self):
+        """Background task that emits watch status periodically"""
+        check_interval = 3  # seconds between updates
+        
+        try:
+            with self.main_app.app_context():
+                while True:
+                    try:
+                        # Collect all watch data
+                        watches_data = []
+                        
+                        # Get list of watches that are currently running
+                        from changedetectionio.flask_app import running_update_threads
+                        currently_checking = []
+                        
+                        # Make a copy to avoid issues if the list changes
+                        threads_snapshot = list(running_update_threads)
+                        for thread in threads_snapshot:
+                            if hasattr(thread, 'current_uuid') and thread.current_uuid:
+                                currently_checking.append(thread.current_uuid)
+                        
+                        # Send all watch data periodically
+                        for uuid, watch in self.datastore.data['watching'].items():
+                            # Simplified watch data to avoid sending everything
+                            simplified_data = {
+                                'uuid': uuid,
+                                'url': watch.get('url', ''),
+                                'title': watch.get('title', ''),
+                                'last_checked': watch.get('last_checked', 0),
+                                'last_changed': watch.get('newest_history_key', 0),
+                                'history_n': watch.history_n if hasattr(watch, 'history_n') else 0,
+                                'viewed': watch.get('viewed', True),
+                                'paused': watch.get('paused', False),
+                                'checking': uuid in currently_checking
+                            }
+                            watches_data.append(simplified_data)
+                        
+                        # Emit all watch data periodically
+                        self.socketio.emit('watch_data', watches_data)
+                        logger.debug(f"Socket.IO: Emitted watch data for {len(watches_data)} watches")
+                        
+                    except Exception as e:
+                        logger.error(f"Socket.IO error in background task: {str(e)}")
+                    
+                    # Wait before next update
+                    time.sleep(check_interval)
+            
+        except Exception as e:
+            logger.error(f"Socket.IO background task failed: {str(e)}")
+            
+    def run(self, host='0.0.0.0', port=5005):
+        """Run the Socket.IO server on a separate port"""
+        # Start the background task when the server starts
+        self.start_background_task()
+        
+        # Run the Socket.IO server
+        # Use 0.0.0.0 to listen on all interfaces
+        logger.info(f"Starting Socket.IO server on http://{host}:{port}")
+        self.socketio.run(self.app, host=host, port=port, debug=False, use_reloader=False, allow_unsafe_werkzeug=True)

changedetectionio/static/js/edit-columns-resizer.js

@@ -0,0 +1,108 @@
+$(document).ready(function() {
+    // Global variables for resize functionality
+    let isResizing = false;
+    let initialX, initialLeftWidth;
+    
+    // Setup document-wide mouse move and mouse up handlers
+    $(document).on('mousemove', handleMouseMove);
+    $(document).on('mouseup', function() {
+        isResizing = false;
+    });
+    
+    // Handle mouse move for resizing
+    function handleMouseMove(e) {
+        if (!isResizing) return;
+        
+        const $container = $('#filters-and-triggers > div');
+        const containerWidth = $container.width();
+        const diffX = e.clientX - initialX;
+        const newLeftWidth = ((initialLeftWidth + diffX) / containerWidth) * 100;
+        
+        // Limit the minimum width percentage
+        if (newLeftWidth > 20 && newLeftWidth < 80) {
+            $('#edit-text-filter').css('flex', `0 0 ${newLeftWidth}%`);
+            $('#text-preview').css('flex', `0 0 ${100 - newLeftWidth}%`);
+        }
+    }
+    
+    // Function to create and setup the resizer
+    function setupResizer() {
+        // Only proceed if text-preview is visible
+        if (!$('#text-preview').is(':visible')) return;
+        
+        // Don't add another resizer if one already exists
+        if ($('#column-resizer').length > 0) return;
+        
+        // Create resizer element
+        const $resizer = $('<div>', {
+            class: 'column-resizer',
+            id: 'column-resizer'
+        });
+        
+        // Insert before the text preview div
+        const $container = $('#filters-and-triggers > div');
+        if ($container.length) {
+            $resizer.insertBefore('#text-preview');
+            
+            // Setup mousedown handler for the resizer
+            $resizer.on('mousedown', function(e) {
+                isResizing = true;
+                initialX = e.clientX;
+                initialLeftWidth = $('#edit-text-filter').width();
+                
+                // Prevent text selection during resize
+                e.preventDefault();
+            });
+        }
+    }
+    
+    // Setup resizer when preview is activated
+    $('#activate-text-preview').on('click', function() {
+        // Give it a small delay to ensure the preview is visible
+        setTimeout(setupResizer, 100);
+    });
+    
+    // Also setup resizer when the filters-and-triggers tab is clicked
+    $('#filters-and-triggers-tab a').on('click', function() {
+        // Give it a small delay to ensure everything is loaded
+        setTimeout(setupResizer, 100);
+    });
+    
+    // Run the setupResizer function when the page is fully loaded
+    // to ensure it's added if the text-preview is already visible
+    setTimeout(setupResizer, 500);
+    
+    // Handle window resize events
+    $(window).on('resize', function() {
+        // Make sure the resizer is added if text-preview is visible
+        if ($('#text-preview').is(':visible')) {
+            setupResizer();
+        }
+    });
+    
+    // Keep checking if the resizer needs to be added
+    // This ensures it's restored if something removes it
+    setInterval(function() {
+        if ($('#text-preview').is(':visible')) {
+            setupResizer();
+        }
+    }, 500);
+    
+    // Add a MutationObserver to watch for DOM changes
+    // This will help restore the resizer if it gets removed
+    const observer = new MutationObserver(function(mutations) {
+        mutations.forEach(function(mutation) {
+            if (mutation.type === 'childList' && 
+                $('#text-preview').is(':visible') && 
+                $('#column-resizer').length === 0) {
+                setupResizer();
+            }
+        });
+    });
+    
+    // Start observing the container for DOM changes
+    observer.observe(document.getElementById('filters-and-triggers'), {
+        childList: true,
+        subtree: true
+    });
+});

changedetectionio/static/js/socket.js

@@ -0,0 +1,78 @@
+// Socket.IO client-side integration for changedetection.io
+
+$(document).ready(function() {
+  // Try to create the socket connection to port 5005 - if it fails, the site will still work normally
+  try {
+    // Connect to the dedicated Socket.IO server on port 5005
+    const socket = io('http://127.0.0.1:5005');
+    
+    // Connection status logging
+    socket.on('connect', function() {
+      console.log('Socket.IO connected');
+    });
+    
+    socket.on('disconnect', function() {
+      console.log('Socket.IO disconnected');
+    });
+    
+    // Listen for periodically emitted watch data
+    socket.on('watch_data', function(watches) {
+      console.log('Received watch data updates');
+      
+      // First, remove checking-now class from all rows
+      $('.checking-now').removeClass('checking-now');
+      
+      // Update all watches with their current data
+      watches.forEach(function(watch) {
+        const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
+        if ($watchRow.length) {
+          updateWatchRow($watchRow, watch);
+        }
+      });
+    });
+    
+    // Function to update a watch row with new data
+    function updateWatchRow($row, data) {
+      // Update the last-checked time
+      const $lastChecked = $row.find('.last-checked');
+      if ($lastChecked.length && data.last_checked) {
+        // Format as timeago if we have the timeago library available
+        if (typeof timeago !== 'undefined') {
+          $lastChecked.text(timeago.format(data.last_checked, Date.now()/1000));
+        } else {
+          // Simple fallback if timeago isn't available
+          const date = new Date(data.last_checked * 1000);
+          $lastChecked.text(date.toLocaleString());
+        }
+      }
+      
+      // Toggle the unviewed class based on viewed status
+      $row.toggleClass('unviewed', data.viewed === false);
+      
+      // If the watch is currently being checked
+      $row.toggleClass('checking-now', data.checking === true);
+      
+      // If a change was detected and not viewed, add highlight effect
+      if (data.history_n > 0 && data.viewed === false) {
+        // Don't add the highlight effect too often
+        if (!$row.hasClass('socket-highlight')) {
+          $row.addClass('socket-highlight');
+          setTimeout(function() {
+            $row.removeClass('socket-highlight');
+          }, 2000);
+          
+          console.log('New change detected for:', data.title || data.url);
+        }
+        
+        // Update any change count indicators if present
+        const $changeCount = $row.find('.change-count');
+        if ($changeCount.length) {
+          $changeCount.text(data.history_n);
+        }
+      }
+    }
+  } catch (e) {
+    // If Socket.IO fails to initialize, just log it and continue
+    console.log('Socket.IO initialization error:', e);
+  }
+});

changedetectionio/static/styles/scss/parts/_socket.scss

@@ -0,0 +1,30 @@
+// Styles for Socket.IO real-time updates
+
+@keyframes socket-highlight-flash {
+  0% {
+    background-color: rgba(var(--color-change-highlight-rgb), 0);
+  }
+  50% {
+    background-color: rgba(var(--color-change-highlight-rgb), 0.4);
+  }
+  100% {
+    background-color: rgba(var(--color-change-highlight-rgb), 0);
+  }
+}
+
+.socket-highlight {
+  animation: socket-highlight-flash 2s ease-in-out;
+}
+
+// Animation for the checking-now state
+@keyframes checking-progress {
+  0% { background-size: 0% 100%; }
+  100% { background-size: 100% 100%; }
+}
+
+tr.checking-now .last-checked {
+  background-image: linear-gradient(to right, rgba(0, 120, 255, 0.2) 0%, rgba(0, 120, 255, 0.1) 100%);
+  background-size: 100% 100%;
+  background-repeat: no-repeat;
+  animation: checking-progress 10s linear;
+}

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -101,6 +101,7 @@
 
   --color-watch-table-error: var(--color-dark-red);
   --color-watch-table-row-text: var(--color-grey-100);
+  --color-change-highlight-rgb: 255, 215, 0; /* Gold color for socket.io highlight */
 }
 
 html[data-darkmode="true"] {

changedetectionio/static/styles/scss/styles.scss

@@ -1070,6 +1070,7 @@ ul {
 }
 
 @import "parts/_visualselector";
+@import "parts/_socket";
 
 #webdriver_delay {
     width: 5em;

changedetectionio/templates/base.html

@@ -31,6 +31,8 @@
     </script>
     <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
+    <script src="https://cdn.socket.io/4.6.0/socket.io.min.js" integrity="sha384-c79GN5VsunZvi+Q/WObgk2in0CbZsHnjEqvFxC5DxHn9lTfNce2WW6h2pH6u/kF+" crossorigin="anonymous"></script>
+    <script src="{{url_for('static_content', group='js', filename='socket.js')}}" defer></script>
   </head>
 
   <body class="">

requirements.txt

@@ -9,6 +9,9 @@ flask_restful
 flask_cors # For the Chrome extension to operate
 flask_wtf~=1.2
 flask~=2.3
+flask-socketio>=5.5.1
+python-socketio>=5.13.0
+python-engineio>=4.12.0
 inscriptis~=2.2
 pytz
 timeago~=1.0