tweaks

* improving unique key fix

* also bump timestamp along 1 sec

changedetectionio/flask_app.py

@@ -45,7 +45,6 @@ running_update_threads = []
 ticker_thread = None
 
 extra_stylesheets = []
-
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
 
@@ -124,10 +123,10 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
 
 @app.template_filter('format_timestamp_timeago')
 def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    if timestamp == False:
+    if not timestamp:
         return 'Not yet'
 
-    return timeago.format(timestamp, time.time())
+    return timeago.format(int(timestamp), time.time())
 
 
 @app.template_filter('pagination_slice')
@@ -1538,6 +1537,11 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+    @app.route("/queue_size", methods=['GET'])
+    @login_optionally_required
+    def get_queue_size():
+        return update_q.qsize()
+
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
     @login_optionally_required
     def highlight_submit_ignore_url():

changedetectionio/model/Watch.py

@@ -217,6 +217,8 @@ class model(dict):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
+
+
             with open(fname, "r") as f:
                 for i in f.readlines():
                     if ',' in i:
@@ -333,7 +335,9 @@ class model(dict):
         # Small hack so that we sleep just enough to allow 1 second  between history snapshots
         # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            time.sleep(timestamp - self.__newest_history_key)
+            logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+            timestamp = str(int(timestamp) + 1)
+            time.sleep(1)
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))

changedetectionio/static/js/preview.js

@@ -40,9 +40,13 @@ if (selectElement) {
     if (selectedOption) {
         if (selectedOption.previousElementSibling) {
             document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value;
+        } else {
+            document.getElementById('btn-previous').remove()
         }
         if (selectedOption.nextElementSibling) {
             document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value;
+        } else {
+            document.getElementById('btn-next').remove()
         }
 
     }

changedetectionio/store.py

@@ -178,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.viewed == False:
+            if watch.history_n >= 2 and watch.viewed == False:
                 return True
         return False
 

changedetectionio/templates/preview.html

@@ -66,7 +66,7 @@
         </div>
 
         <div class="tab-pane-inner" id="text">
-            <div class="snapshot-age">{{ watch.snapshot_text_ctime|format_timestamp_timeago }}</div>
+            <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div>
             <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
             <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
 

changedetectionio/templates/watch-overview.html

@@ -78,7 +78,7 @@
             <tbody>
             {% if not watches|length %}
             <tr>
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+                <td colspan="6" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}

changedetectionio/tests/restock/test_restock.py

@@ -53,7 +53,6 @@ def test_restock_detection(client, live_server):
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
-    time.sleep(1)
     live_server_setup(live_server)
     #####################
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -71,6 +71,8 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
+
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -135,6 +137,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -40,8 +40,6 @@ def test_setup(client, live_server):
 
 def test_check_removed_line_contains_trigger(client, live_server):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_auth.py

@@ -2,13 +2,12 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
+
 
 def test_basic_auth(client, live_server):
 
     live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -19,7 +18,7 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Check form validation
     res = client.post(
@@ -29,7 +28,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -81,7 +81,7 @@ def test_setup(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_ldjson_price_autodetect(client, live_server):
-
+    #live_server_setup(live_server)
     set_response_with_ldjson()
 
     # Add our URL to the import page
@@ -104,7 +104,6 @@ def test_check_ldjson_price_autodetect(client, live_server):
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
-    # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
@@ -114,6 +113,9 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # and last snapshop (via API) should be just the price
     api_key = extract_api_key_from_UI(client)
+    # Time for writes to happen to history text
+    time.sleep(0.5)
+
     res = client.get(
         url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
         headers={'x-api-key': api_key},

changedetectionio/tests/test_backend.py

@@ -135,6 +135,9 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
+    # Be sure the last_viewed is going to be greater than the last snapshot
+    time.sleep(1)
+
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 

changedetectionio/tests/test_backup.py

@@ -13,9 +13,6 @@ def test_backup(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_block_while_text_present.py

@@ -68,8 +68,6 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
 
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_clone.py

@@ -10,9 +10,6 @@ def test_trigger_functionality(client, live_server):
 
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_css_selector.py

@@ -77,9 +77,6 @@ def test_check_markup_include_filters_restriction(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -89,8 +86,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -100,22 +96,22 @@ def test_check_markup_include_filters_restriction(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
     assert bytes(include_filters.encode('utf-8')) in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
@@ -138,8 +134,6 @@ def test_check_multiple_filters(client, live_server):
      </html>
     """)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -149,7 +143,7 @@ def test_check_multiple_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -164,9 +158,7 @@ def test_check_multiple_filters(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -5,7 +5,7 @@ import time
 from flask import url_for
 
 from ..html_tools import *
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -111,16 +111,13 @@ def test_element_removal_full(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
     res = client.post(
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -27,9 +27,6 @@ def set_html_response():
 def test_check_encoding_detection(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -56,9 +53,6 @@ def test_check_encoding_detection(client, live_server):
 def test_check_encoding_detection_missing_content_type_header(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(

changedetectionio/tests/test_errorhandling.py

@@ -64,8 +64,6 @@ def test_http_error_handler(client, live_server):
 
 # Just to be sure error text is properly handled
 def test_DNS_errors(client, live_server):
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -89,8 +87,6 @@ def test_DNS_errors(client, live_server):
 # Re 1513
 def test_low_level_errors_clear_correctly(client, live_server):
     #live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")

changedetectionio/tests/test_filter_exist_changes.py

@@ -48,8 +48,6 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     set_response_without_filter()
 
     # Add our URL to the import page

changedetectionio/tests/test_filter_failure_notification.py

@@ -23,8 +23,6 @@ def set_response_with_filter():
 
 def run_filter_test(client, content_filter):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -105,6 +103,10 @@ def run_filter_test(client, content_filter):
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
+
+    # Give apprise time to fire
+    time.sleep(3)
+
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -124,6 +126,9 @@ def run_filter_test(client, content_filter):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
+    # Give apprise time to fire
+    time.sleep(3)
+
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
     # but it should not contain the info about a failed filter (because there was none in this case)
@@ -149,11 +154,13 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
+    #live_server_setup(live_server)
     set_original_response()
     wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
+    # live_server_setup(live_server)
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '//*[@id="nope-doesnt-exist"]')

changedetectionio/tests/test_history_consistency.py

@@ -11,8 +11,6 @@ from urllib.parse import urlparse, parse_qs
 def test_consistent_history(client, live_server):
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     r = range(1, 50)
 
     for one in r:

changedetectionio/tests/test_ignorehyperlinks.py

@@ -45,9 +45,6 @@ def test_render_anchor_tag_content_true(client, live_server):
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # set original html text
     set_original_ignore_response()
 

changedetectionio/tests/test_notification_errors.py

@@ -9,9 +9,6 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/util.py

@@ -120,18 +120,18 @@ def extract_UUID_from_client(client):
     uuid = m.group(1)
     return uuid.strip()
 
-def wait_for_all_checks(client):
-    # Loop waiting until done..
-    attempt=0
-    time.sleep(0.1)
-    while attempt < 60:
-        time.sleep(1)
-        res = client.get(url_for("index"))
-        if not b'Checking now' in res.data:
-            break
-        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
 
-        attempt += 1
+def wait_for_all_checks(client):
+    now = time.time()
+    while time.time() - now <= 30:
+        time.sleep(0.1)
+        p = client.application.view_functions['get_queue_size']()
+        if not p:
+            break
+        logging.getLogger().info(f"Waiting for queue to be empty, queue size {p}  - {time.time() - now}")
+
+    # Empty queue still means that one could be processing, give time for the processing to complete
+    time.sleep(0.2)
 
 def live_server_setup(live_server):
 

changedetectionio/update_worker.py

@@ -160,8 +160,8 @@ class update_worker(threading.Thread):
 
 
     def send_filter_failure_notification(self, watch_uuid):
-
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        logger.trace(f"Watch UUID: {watch_uuid} - Sending filter failure notification - threshold attempts {threshold}")
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -335,6 +335,7 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
+                        logger.debug(f"Watch UUID: {uuid} - Got FilterNotFoundInResponse exception, Consecutive failures - {self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)} - handling..")
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
@@ -543,4 +544,4 @@ class update_worker(threading.Thread):
                 # Give the CPU time to interrupt
                 time.sleep(0.1)
 
-            self.app.config.exit.wait(1)
+            self.app.config.exit.wait(0.5)

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet==0.33.3 # related to dnspython fixes
+eventlet==0.35.2 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -73,8 +73,6 @@ openpyxl
 jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
 jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
-# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
-pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
 pyppeteer-ng==2.0.0rc5