0.40.3

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask import (
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.2'
+__version__ = '0.40.3'
 
 datastore = None
 

changedetectionio/api/api_v1.py

@@ -33,7 +33,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Single watch information
+        @api {get} /api/v1/watch/:uuid Get a single watch data
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -70,13 +70,16 @@ class Watch(Resource):
             return "OK", 200
 
         # Return without history, get that via another API call
+        # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
+        watch['last_changed'] = watch.last_changed
+
         return watch
 
     @auth.check_token
     def delete(self, uuid):
         """
-        @api {delete} /api/v1/watch/:uuid Delete watch information
+        @api {delete} /api/v1/watch/:uuid Delete a watch and related history
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiParam {uuid} uuid Watch unique ID.
@@ -90,21 +93,18 @@ class Watch(Resource):
         self.datastore.delete(uuid)
         return 'OK', 204
 
-    # Update an existing
     @auth.check_token
     @expects_json(schema_update_watch)
     def put(self, uuid):
         """
         @api {put} /api/v1/watch/:uuid Update watch information
         @apiExample {curl} Example usage:
-            Create a watch (POST)
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
             Update (PUT)
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
 
-        @apiDescription Updates an existing watch using JSON, accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
         @apiParam {uuid} uuid Watch unique ID.
-        @apiName Update
+        @apiName Update a watch
         @apiGroup Watch
         @apiSuccess (200) {String} OK Was updated
         @apiSuccess (500) {String} ERR Some other error
@@ -131,6 +131,21 @@ class WatchHistory(Resource):
     # Get a list of available history for a watch by UUID
     # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
     def get(self, uuid):
+        """
+        @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
+        @apiDescription Requires `uuid`, returns list
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            {
+                "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
+                "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
+                "1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
+            }
+        @apiName Get list of available stored snapshots for watch
+        @apiGroup Watch History
+        @apiSuccess (200) {String} OK
+        @apiSuccess (404) {String} ERR Not found
+        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -142,11 +157,18 @@ class WatchSingleHistory(Resource):
         # datastore is a black box dependency
         self.datastore = kwargs['datastore']
 
-    # Read a given history snapshot and return its content
-    # <string:timestamp> or "latest"
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
     @auth.check_token
     def get(self, uuid, timestamp):
+        """
+        @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
+        @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+        @apiName Get single snapshot content
+        @apiGroup Watch History
+        @apiSuccess (200) {String} OK
+        @apiSuccess (404) {String} ERR Not found
+        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -157,6 +179,7 @@ class WatchSingleHistory(Resource):
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
+        # @todo - Check for UTF-8 compatability
         with open(watch.history[timestamp], 'r') as f:
             content = f.read()
 
@@ -175,21 +198,19 @@ class CreateWatch(Resource):
     @expects_json(schema_create_watch)
     def post(self):
         """
-        @api {post} /api/v1/watch Create a watch
-        @apiDescription requires `url`, Creates a watch, also accepts accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @api {post} /api/v1/watch Create a single watch
+        @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
         @apiName Create
-        @apiGroup CreateWatch
+        @apiGroup Watch
         @apiSuccess (200) {String} OK Was created
         @apiSuccess (500) {String} ERR Some other error
         """
 
-        #
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-
         if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
@@ -211,17 +232,32 @@ class CreateWatch(Resource):
     @auth.check_token
     def get(self):
         """
-        @api {get} /api/v1/watch
+        @api {get} /api/v1/watch List watches
         @apiDescription Return concise list of available watches and some very basic info
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            recheck_all=1 to recheck all
+            {
+                "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
+                    "last_changed": 1677103794,
+                    "last_checked": 1677103794,
+                    "last_error": false,
+                    "title": "",
+                    "url": "http://www.quotationspage.com/random.php"
+                },
+                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
+                    "last_changed": 0,
+                    "last_checked": 1676662819,
+                    "last_error": false,
+                    "title": "QuickLook",
+                    "url": "https://github.com/QL-Win/QuickLook/tags"
+                }
+            }
+
         @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
         @apiParam {String} [tag]               Optional name of tag to limit results
         @apiName ListWatches
-        @apiGroup CreateWatch
-
-        :return:
+        @apiGroup Watch Management
+        @apiSuccess (200) {String} OK JSON dict
         """
         list = {}
 
@@ -252,6 +288,22 @@ class SystemInfo(Resource):
 
     @auth.check_token
     def get(self):
+        """
+        @api {get} /api/v1/systeminfo Return system info
+        @apiDescription Return some info about the current system state
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            HTTP/1.0 200
+            {
+                'queue_size': 10 ,
+                'overdue_watches': ["watch-uuid-list"],
+                'uptime': 38344.55,
+                'watch_count': 800,
+                'version': "0.40.1"
+            }
+        @apiName Get Info
+        @apiGroup System Information
+        """
         import time
         overdue_watches = []
 
@@ -270,10 +322,11 @@ class SystemInfo(Resource):
             # Allow 5 minutes of grace time before we decide it's overdue
             if time_since_check - (5 * 60) > t:
                 overdue_watches.append(uuid)
-
+        from changedetectionio import __version__ as main_version
         return {
                    'queue_size': self.update_q.qsize(),
                    'overdue_watches': overdue_watches,
                    'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {}))
+                   'watch_count': len(self.datastore.data.get('watching', {})),
+                   'version': main_version
                }, 200

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -31,6 +31,7 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Uncheck checkbox': '1 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
+                          'Wait for text in element': '1 1',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -132,6 +133,17 @@ class steppable_browser_interface():
     def action_wait_for_seconds(self, selector, value):
         self.page.wait_for_timeout(int(value) * 1000)
 
+    def action_wait_for_text(self, selector, value):
+        import json
+        v = json.dumps(value)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+
+    def action_wait_for_text_in_element(self, selector, value):
+        import json
+        s = json.dumps(selector)
+        v = json.dumps(value)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard
     def action_press_enter(self, selector, value):

changedetectionio/model/Watch.py

@@ -241,7 +241,7 @@ class model(dict):
 
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
-    def save_history_text(self, contents, timestamp):
+    def save_history_text(self, contents, timestamp, snapshot_id):
 
         self.ensure_data_dir_exists()
 
@@ -250,13 +250,16 @@ class model(dict):
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
             time.sleep(timestamp - self.__newest_history_key)
 
-        snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
+        snapshot_fname = f"{snapshot_id}.txt"
 
-        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
-        # most sites are utf-8 and some are even broken utf-8
-        with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
-            f.write(contents)
-            f.close()
+        # Only write if it does not exist, this is so that we dont bother re-saving the same data by checksum under different filenames.
+        dest = os.path.join(self.watch_data_dir, snapshot_fname)
+        if not os.path.exists(dest):
+            # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
+            # most sites are utf-8 and some are even broken utf-8
+            with open(dest, 'wb') as f:
+                f.write(contents)
+                f.close()
 
         # Append to index
         # @todo check last char was \n

changedetectionio/templates/watch-overview.html

@@ -57,9 +57,9 @@
                 <th></th>
                 {% set link_order = "desc" if sort_order else "asc" %}
                 {% set arrow_span = "" %}
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order)}}">Website <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th></th>
             </tr>
             </thead>

changedetectionio/tests/test_filter_exist_changes.py

@@ -117,18 +117,3 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     assert 'Ticket now on sale' in notification
     os.unlink("test-datastore/notification.txt")
-
-
-    # Test that if it gets removed, then re-added, we get a notification
-    # Remove the target and re-add it, we should get a new notification
-    set_response_without_filter()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-    assert not os.path.isfile("test-datastore/notification.txt")
-
-    set_response_with_filter()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt")
-
-# Also test that the filter was updated after the first one was requested

changedetectionio/update_worker.py

@@ -169,10 +169,8 @@ class update_worker(threading.Thread):
                 if uuid in list(self.datastore.data['watching'].keys()):
                     changed_detected = False
                     contents = b''
-                    screenshot = False
-                    update_obj= {}
-                    xpath_data = False
                     process_changedetection_results = True
+                    update_obj= {}
                     print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority, self.datastore.data['watching'][uuid]['url']))
                     now = time.time()
 
@@ -212,9 +210,7 @@ class update_worker(threading.Thread):
                         if e.page_text:
                             self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
@@ -222,9 +218,7 @@ class update_worker(threading.Thread):
                             continue
 
                         err_text = "Warning, no filters were found, no change detection ran."
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Only when enabled, send the notification
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -241,11 +235,12 @@ class update_worker(threading.Thread):
 
                             self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
 
-                        process_changedetection_results = True
+                        process_changedetection_results = False
 
                     except content_fetcher.checksumFromPreviousCheckWasTheSame as e:
-                        # Yes fine, so nothing todo
-                        pass
+                        # Yes fine, so nothing todo, don't continue to process.
+                        process_changedetection_results = False
+                        changed_detected = False
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -253,9 +248,7 @@ class update_worker(threading.Thread):
                             continue
 
                         err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
 
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -271,6 +264,7 @@ class update_worker(threading.Thread):
                                 c = 0
 
                             self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+
                         process_changedetection_results = False
 
                     except content_fetcher.EmptyReply as e:
@@ -278,6 +272,7 @@ class update_worker(threading.Thread):
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except content_fetcher.ScreenshotUnavailable as e:
                         err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
@@ -289,6 +284,7 @@ class update_worker(threading.Thread):
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except content_fetcher.PageUnloadable as e:
                         err_text = "Page request from server didnt respond correctly"
                         if e.message:
@@ -299,6 +295,7 @@ class update_worker(threading.Thread):
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
@@ -318,16 +315,15 @@ class update_worker(threading.Thread):
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
                         try:
-                            watch = self.datastore.data['watching'][uuid]
-                            fname = "" # Saved history text filename
-
-                            # For the FIRST time we check a site, or a change detected, save the snapshot.
-                            if changed_detected or not watch['last_checked']:
-                                # A change was detected
-                                watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
-
+                            watch = self.datastore.data['watching'].get(uuid)
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
+                            # Also save the snapshot on the first time checked
+                            if changed_detected or not watch['last_checked']:
+                                watch.save_history_text(contents=contents,
+                                                        timestamp=str(round(time.time())),
+                                                        snapshot_id=update_obj.get('previous_md5', 'none'))
+
                             # A change was detected
                             if changed_detected:
                                 print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))

docs/api_v1/assets/main.css

@@ -49,6 +49,7 @@ input[type="date"] {
   src: url('./glyphicons-halflings-regular.eot');
   src: url('./glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),
     url('./glyphicons-halflings-regular.woff') format('woff'),
+    url('./glyphicons-halflings-regular.woff2') format('woff2'),
     url('./glyphicons-halflings-regular.ttf') format('truetype'),
     url('./glyphicons-halflings-regular.svg#glyphicons-halflingsregular') format('svg');
 }

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css" rel="stylesheet" media="screen">
-  <link href="assets/prism.css" rel="stylesheet" />
-  <link href="assets/main.css" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png"rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
+  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js"></script>
+<script src="assets/main.bundle.js?v=1677105736053"></script>
 </body>
 </html>

docs/apidoc.json

@@ -3,5 +3,6 @@
   "version": "0.1.0",
   "description": "Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.",
   "title": "changedetection.io API",
-  "url" : "https://changedetection.io/docs/api_v1/index.html"
+  "url" : "",
+  "sampleUrl" : false
 }

docs/package.json

@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "apidoc": "^0.53.1"
+    "apidoc": "^0.54.0"
   }
 }

requirements.txt

@@ -31,7 +31,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.2.1
+apprise~=1.3.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -42,7 +42,7 @@ paho-mqtt
 cryptography~=3.4
 
 # Used for CSS filtering
-bs4
+beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml