Closes #1450

README.md

@@ -49,6 +49,7 @@ Requires Playwright to be enabled.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
+- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -63,6 +64,7 @@ Requires Playwright to be enabled.
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
 - Get notified when certain keywords appear in Twitter search results
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
+- Get alerts when new job positions are open on Bamboo HR and other job platforms
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask import (
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.2'
+__version__ = '0.40.3'
 
 datastore = None
 
@@ -505,41 +505,6 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("clear_all_history.html")
         return output
 
-
-    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
-    # the excluded text.
-    def get_current_checksum_include_ignore_text(uuid):
-
-        import hashlib
-
-        from changedetectionio import fetch_site_status
-
-        # Get the most recent one
-        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
-
-        # 0 means that theres only one, so that there should be no 'unviewed' history available
-        if newest_history_key == 0:
-            newest_history_key = list(datastore.data['watching'][uuid].history.keys())[0]
-
-        if newest_history_key:
-            with open(datastore.data['watching'][uuid].history[newest_history_key],
-                      encoding='utf-8') as file:
-                raw_content = file.read()
-
-                handler = fetch_site_status.perform_site_check(datastore=datastore)
-                stripped_content = html_tools.strip_ignore_text(raw_content,
-                                                             datastore.data['watching'][uuid]['ignore_text'])
-
-                if datastore.data['settings']['application'].get('ignore_whitespace', False):
-                    checksum = hashlib.md5(stripped_content.translate(None, b'\r\n\t ')).hexdigest()
-                else:
-                    checksum = hashlib.md5(stripped_content).hexdigest()
-
-                return checksum
-
-        return datastore.data['watching'][uuid]['previous_md5']
-
-
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@@ -946,7 +911,7 @@ def changedetection_app(config=None, datastore_o=None):
         is_html_webdriver = False
         if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
             is_html_webdriver = True
-            
+
         # Never requested successfully, but we detected a fetch error
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
@@ -1036,7 +1001,8 @@ def changedetection_app(config=None, datastore_o=None):
             os.unlink(previous_backup_filename)
 
         # create a ZipFile object
-        backupname = "changedetection-backup-{}.zip".format(int(time.time()))
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        backupname = "changedetection-backup-{}.zip".format(timestamp)
         backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
 
         with zipfile.ZipFile(backup_filepath, "w",

changedetectionio/api/api_v1.py

@@ -33,7 +33,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Single watch information
+        @api {get} /api/v1/watch/:uuid Get a single watch data
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -70,13 +70,16 @@ class Watch(Resource):
             return "OK", 200
 
         # Return without history, get that via another API call
+        # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
+        watch['last_changed'] = watch.last_changed
+
         return watch
 
     @auth.check_token
     def delete(self, uuid):
         """
-        @api {delete} /api/v1/watch/:uuid Delete watch information
+        @api {delete} /api/v1/watch/:uuid Delete a watch and related history
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiParam {uuid} uuid Watch unique ID.
@@ -90,21 +93,18 @@ class Watch(Resource):
         self.datastore.delete(uuid)
         return 'OK', 204
 
-    # Update an existing
     @auth.check_token
     @expects_json(schema_update_watch)
     def put(self, uuid):
         """
         @api {put} /api/v1/watch/:uuid Update watch information
         @apiExample {curl} Example usage:
-            Create a watch (POST)
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
             Update (PUT)
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
 
-        @apiDescription Updates an existing watch using JSON, accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
         @apiParam {uuid} uuid Watch unique ID.
-        @apiName Update
+        @apiName Update a watch
         @apiGroup Watch
         @apiSuccess (200) {String} OK Was updated
         @apiSuccess (500) {String} ERR Some other error
@@ -131,6 +131,21 @@ class WatchHistory(Resource):
     # Get a list of available history for a watch by UUID
     # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
     def get(self, uuid):
+        """
+        @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
+        @apiDescription Requires `uuid`, returns list
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            {
+                "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
+                "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
+                "1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
+            }
+        @apiName Get list of available stored snapshots for watch
+        @apiGroup Watch History
+        @apiSuccess (200) {String} OK
+        @apiSuccess (404) {String} ERR Not found
+        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -142,11 +157,18 @@ class WatchSingleHistory(Resource):
         # datastore is a black box dependency
         self.datastore = kwargs['datastore']
 
-    # Read a given history snapshot and return its content
-    # <string:timestamp> or "latest"
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
     @auth.check_token
     def get(self, uuid, timestamp):
+        """
+        @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
+        @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+        @apiName Get single snapshot content
+        @apiGroup Watch History
+        @apiSuccess (200) {String} OK
+        @apiSuccess (404) {String} ERR Not found
+        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -157,6 +179,7 @@ class WatchSingleHistory(Resource):
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
+        # @todo - Check for UTF-8 compatability
         with open(watch.history[timestamp], 'r') as f:
             content = f.read()
 
@@ -175,21 +198,19 @@ class CreateWatch(Resource):
     @expects_json(schema_create_watch)
     def post(self):
         """
-        @api {post} /api/v1/watch Create a watch
-        @apiDescription requires `url`, Creates a watch, also accepts accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @api {post} /api/v1/watch Create a single watch
+        @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
         @apiName Create
-        @apiGroup CreateWatch
+        @apiGroup Watch
         @apiSuccess (200) {String} OK Was created
         @apiSuccess (500) {String} ERR Some other error
         """
 
-        #
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-
         if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
@@ -211,17 +232,32 @@ class CreateWatch(Resource):
     @auth.check_token
     def get(self):
         """
-        @api {get} /api/v1/watch
+        @api {get} /api/v1/watch List watches
         @apiDescription Return concise list of available watches and some very basic info
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            recheck_all=1 to recheck all
+            {
+                "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
+                    "last_changed": 1677103794,
+                    "last_checked": 1677103794,
+                    "last_error": false,
+                    "title": "",
+                    "url": "http://www.quotationspage.com/random.php"
+                },
+                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
+                    "last_changed": 0,
+                    "last_checked": 1676662819,
+                    "last_error": false,
+                    "title": "QuickLook",
+                    "url": "https://github.com/QL-Win/QuickLook/tags"
+                }
+            }
+
         @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
         @apiParam {String} [tag]               Optional name of tag to limit results
         @apiName ListWatches
-        @apiGroup CreateWatch
-
-        :return:
+        @apiGroup Watch Management
+        @apiSuccess (200) {String} OK JSON dict
         """
         list = {}
 
@@ -252,6 +288,22 @@ class SystemInfo(Resource):
 
     @auth.check_token
     def get(self):
+        """
+        @api {get} /api/v1/systeminfo Return system info
+        @apiDescription Return some info about the current system state
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            HTTP/1.0 200
+            {
+                'queue_size': 10 ,
+                'overdue_watches': ["watch-uuid-list"],
+                'uptime': 38344.55,
+                'watch_count': 800,
+                'version': "0.40.1"
+            }
+        @apiName Get Info
+        @apiGroup System Information
+        """
         import time
         overdue_watches = []
 
@@ -270,10 +322,11 @@ class SystemInfo(Resource):
             # Allow 5 minutes of grace time before we decide it's overdue
             if time_since_check - (5 * 60) > t:
                 overdue_watches.append(uuid)
-
+        from changedetectionio import __version__ as main_version
         return {
                    'queue_size': self.update_q.qsize(),
                    'overdue_watches': overdue_watches,
                    'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {}))
+                   'watch_count': len(self.datastore.data.get('watching', {})),
+                   'version': main_version
                }, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -106,8 +106,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             if step_operation == 'Goto site':
                 step_operation = 'goto_url'
-                step_optional_value = None
-                step_selector = datastore.data['watching'][uuid].get('url')
+                step_optional_value = datastore.data['watching'][uuid].get('url')
+                step_selector = None
 
             # @todo try.. accept.. nice errors not popups..
             try:

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -25,12 +25,14 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Execute JS': '0 1',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
+                          'Goto URL': '0 1',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
                           'Uncheck checkbox': '1 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
+                          'Wait for text in element': '1 1',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -53,7 +55,7 @@ class steppable_browser_interface():
 
         print("> action calling", call_action_name)
         # https://playwright.dev/python/docs/selectors#xpath-selectors
-        if selector.startswith('/') and not selector.startswith('//'):
+        if selector and selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
 
         action_handler = getattr(self, "action_" + call_action_name)
@@ -72,10 +74,10 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(3 * 1000)
         print("Call action done in", time.time() - now)
 
-    def action_goto_url(self, url, optional_value):
+    def action_goto_url(self, selector, value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(url, timeout=0, wait_until='commit')
+        response = self.page.goto(value, timeout=0, wait_until='commit')
 
         # Wait_until = commit
         # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
@@ -132,6 +134,17 @@ class steppable_browser_interface():
     def action_wait_for_seconds(self, selector, value):
         self.page.wait_for_timeout(int(value) * 1000)
 
+    def action_wait_for_text(self, selector, value):
+        import json
+        v = json.dumps(value)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+
+    def action_wait_for_text_in_element(self, selector, value):
+        import json
+        s = json.dumps(selector)
+        v = json.dumps(value)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard
     def action_press_enter(self, selector, value):

changedetectionio/diff.py

@@ -10,7 +10,7 @@ def same_slicer(l, a, b):
         return l[a:b]
 
 # like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False):
+def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True):
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
 
     # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
@@ -18,20 +18,20 @@ def customSequenceMatcher(before, after, include_equal=False):
         if include_equal and tag == 'equal':
             g = before[alo:ahi]
             yield g
-        elif tag == 'delete':
+        elif include_removed and tag == 'delete':
             g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
             yield g
         elif tag == 'replace':
             g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
-            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
+            g += ["(into) " + i for i in same_slicer(after, blo, bhi)]
             yield g
-        elif tag == 'insert':
-            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
+        elif include_added and tag == 'insert':
+            g = ["(added) " + i for i in same_slicer(after, blo, bhi)]
             yield g
 
 # only_differences - only return info about the differences, no context
 # line_feed_sep could be "<br/>" or "<li>" or "\n" etc
-def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+def render_diff(previous_file, newest_file, include_equal=False, include_removed=True, include_added=True, line_feed_sep="\n"):
     with open(newest_file, 'r') as f:
         newest_version_file_contents = f.read()
         newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
@@ -45,7 +45,7 @@ def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="
 
     rendered_diff = customSequenceMatcher(previous_version_file_contents,
                                           newest_version_file_contents,
-                                          include_equal)
+                                          include_equal, include_removed, include_added)
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])

changedetectionio/model/Watch.py

@@ -153,7 +153,9 @@ class model(dict):
     @property
     def is_pdf(self):
         # content_type field is set in the future
-        return '.pdf' in self.get('url', '').lower() or 'pdf' in self.get('content_type', '').lower()
+        # https://github.com/dgtlmoon/changedetection.io/issues/1392
+        # Not sure the best logic here
+        return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content_type', '').lower()
 
     @property
     def label(self):
@@ -239,7 +241,7 @@ class model(dict):
 
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
-    def save_history_text(self, contents, timestamp):
+    def save_history_text(self, contents, timestamp, snapshot_id):
 
         self.ensure_data_dir_exists()
 
@@ -248,13 +250,16 @@ class model(dict):
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
             time.sleep(timestamp - self.__newest_history_key)
 
-        snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
+        snapshot_fname = f"{snapshot_id}.txt"
 
-        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
-        # most sites are utf-8 and some are even broken utf-8
-        with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
-            f.write(contents)
-            f.close()
+        # Only write if it does not exist, this is so that we dont bother re-saving the same data by checksum under different filenames.
+        dest = os.path.join(self.watch_data_dir, snapshot_fname)
+        if not os.path.exists(dest):
+            # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
+            # most sites are utf-8 and some are even broken utf-8
+            with open(dest, 'wb') as f:
+                f.write(contents)
+                f.close()
 
         # Append to index
         # @todo check last char was \n

changedetectionio/notification.py

@@ -10,6 +10,8 @@ valid_tokens = {
     'watch_title': '',
     'watch_tag': '',
     'diff': '',
+    'diff_added': '',
+    'diff_removed': '',
     'diff_full': '',
     'diff_url': '',
     'preview_url': '',
@@ -215,6 +217,8 @@ def create_notification_parameters(n_object, datastore):
             'watch_tag': watch_tag if watch_tag is not None else '',
             'diff_url': diff_url,
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
+            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
             'preview_url': preview_url,
             'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''

changedetectionio/store.py

@@ -192,27 +192,24 @@ class ChangeDetectionStore:
         tags.sort()
         return tags
 
-    def unlink_history_file(self, path):
-        try:
-            unlink(path)
-        except (FileNotFoundError, IOError):
-            pass
-
     # Delete a single watch by UUID
     def delete(self, uuid):
+        import pathlib
+        import shutil
+
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
 
                 # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
-                    for path in self.data['watching'][uuid].history.values():
-                        self.unlink_history_file(path)
+                    path = pathlib.Path(os.path.join(self.datastore_path, uuid))
+                    shutil.rmtree(path)
+                    self.needs_write_urgent = True
 
             else:
-                for path in self.data['watching'][uuid].history.values():
-                    self.unlink_history_file(path)
-
+                path = pathlib.Path(os.path.join(self.datastore_path, uuid))
+                shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
             self.needs_write_urgent = True

changedetectionio/templates/_common_fields.jinja

@@ -55,39 +55,51 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{{ '{{ base_url }}' }}</code></td>
+                                        <td><code>{{ '{{base_url}}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
+                                        <td><code>{{ '{{watch_url}}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
+                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
+                                        <td><code>{{ '{{watch_title}}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
+                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
                                         <td>The watch label / tag</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
+                                        <td><code>{{ '{{preview_url}}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
-                                        <td>The diff output - differences only</td>
+                                        <td><code>{{ '{{diff_url}}' }}</code></td>
+                                        <td>The URL of the diff output for the watch.</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff}}' }}</code></td>
+                                        <td>The diff output - only changes, additions, and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_added}}' }}</code></td>
+                                        <td>The diff output - only changes and additions</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
+                                        <td>The diff output - only changes and removals</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
+                                        <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
+                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
@@ -95,8 +107,10 @@
                                 </table>
                                 <div class="pure-form-message-inline">
                                     <br>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
+									<br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> </br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/watch-overview.html

@@ -57,9 +57,9 @@
                 <th></th>
                 {% set link_order = "desc" if sort_order else "asc" %}
                 {% set arrow_span = "" %}
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order)}}">Website <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th></th>
             </tr>
             </thead>

changedetectionio/tests/test_backend.py

@@ -82,7 +82,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'<rss' in res.data
 
     # re #16 should have the diff in here too
-    assert b'(into   ) which has this one new line' in res.data
+    assert b'(into) which has this one new line' in res.data
     assert b'CDATA' in res.data
 
     assert expected_url.encode('utf-8') in res.data

changedetectionio/tests/test_filter_exist_changes.py

@@ -117,18 +117,3 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     assert 'Ticket now on sale' in notification
     os.unlink("test-datastore/notification.txt")
-
-
-    # Test that if it gets removed, then re-added, we get a notification
-    # Remove the target and re-add it, we should get a new notification
-    set_response_without_filter()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-    assert not os.path.isfile("test-datastore/notification.txt")
-
-    set_response_with_filter()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt")
-
-# Also test that the filter was updated after the first one was requested

changedetectionio/tests/test_notification.py

@@ -100,6 +100,8 @@ def test_check_notification(client, live_server):
                                                    "Diff URL: {{diff_url}}\n"
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
+                                                   "Diff Added: {{diff_added}}\n"
+                                                   "Diff Removed: {{diff_removed}}\n"
                                                    "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
@@ -147,7 +149,7 @@ def test_check_notification(client, live_server):
     assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
-    assert "(into   ) which has this one new line" in notification_submission
+    assert "(into) which has this one new line" in notification_submission
     # Re #342 - check for accidental python byte encoding of non-utf8/string
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)

changedetectionio/tests/unit/test_notification_diff.py

@@ -16,15 +16,30 @@ class TestDiffBuilder(unittest.TestCase):
         output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt")
         output = output.split("\n")
         self.assertIn('(changed) ok', output)
-        self.assertIn('(into   ) xok', output)
-        self.assertIn('(into   ) next-x-ok', output)
-        self.assertIn('(added  ) and something new', output)
+        self.assertIn('(into) xok', output)
+        self.assertIn('(into) next-x-ok', output)
+        self.assertIn('(added) and something new', output)
 
 
         output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt")
         output = output.split("\n")
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
+        
+        #diff_removed
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt", include_equal=False, include_removed=True, include_added=False)
+        output = output.split("\n")
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into) xok', output)
+        self.assertIn('(into) next-x-ok', output)
+        self.assertNotIn('(added) and something new', output)
+        
+        #diff_removed
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt", include_equal=False, include_removed=True, include_added=False)
+        output = output.split("\n")
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
+        
 
         # @todo test blocks of changed, blocks of added, blocks of removed
 

changedetectionio/update_worker.py

@@ -78,7 +78,9 @@ class update_worker(threading.Thread):
                 'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None,
                 'current_snapshot': snapshot_contents.decode('utf-8'),
                 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep),
-                'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep)
+                'diff_added': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_removed=False, line_feed_sep=line_feed_sep),
+                'diff_removed': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_added=False, line_feed_sep=line_feed_sep),
+                'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_equal=True, line_feed_sep=line_feed_sep)
             })
             logging.info (">> SENDING NOTIFICATION")
             self.notification_q.put(n_object)
@@ -169,10 +171,8 @@ class update_worker(threading.Thread):
                 if uuid in list(self.datastore.data['watching'].keys()):
                     changed_detected = False
                     contents = b''
-                    screenshot = False
-                    update_obj= {}
-                    xpath_data = False
                     process_changedetection_results = True
+                    update_obj= {}
                     print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority, self.datastore.data['watching'][uuid]['url']))
                     now = time.time()
 
@@ -212,9 +212,7 @@ class update_worker(threading.Thread):
                         if e.page_text:
                             self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
@@ -222,9 +220,7 @@ class update_worker(threading.Thread):
                             continue
 
                         err_text = "Warning, no filters were found, no change detection ran."
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Only when enabled, send the notification
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -241,11 +237,12 @@ class update_worker(threading.Thread):
 
                             self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
 
-                        process_changedetection_results = True
+                        process_changedetection_results = False
 
                     except content_fetcher.checksumFromPreviousCheckWasTheSame as e:
-                        # Yes fine, so nothing todo
-                        pass
+                        # Yes fine, so nothing todo, don't continue to process.
+                        process_changedetection_results = False
+                        changed_detected = False
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -253,9 +250,7 @@ class update_worker(threading.Thread):
                             continue
 
                         err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           # So that we get a trigger when the content is added again
-                                                                           'previous_md5': ''})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
 
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -271,6 +266,7 @@ class update_worker(threading.Thread):
                                 c = 0
 
                             self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+
                         process_changedetection_results = False
 
                     except content_fetcher.EmptyReply as e:
@@ -278,6 +274,7 @@ class update_worker(threading.Thread):
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except content_fetcher.ScreenshotUnavailable as e:
                         err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
@@ -289,6 +286,7 @@ class update_worker(threading.Thread):
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except content_fetcher.PageUnloadable as e:
                         err_text = "Page request from server didnt respond correctly"
                         if e.message:
@@ -299,6 +297,7 @@ class update_worker(threading.Thread):
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
+                        process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
@@ -318,16 +317,15 @@ class update_worker(threading.Thread):
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
                         try:
-                            watch = self.datastore.data['watching'][uuid]
-                            fname = "" # Saved history text filename
-
-                            # For the FIRST time we check a site, or a change detected, save the snapshot.
-                            if changed_detected or not watch['last_checked']:
-                                # A change was detected
-                                watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
-
+                            watch = self.datastore.data['watching'].get(uuid)
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
+                            # Also save the snapshot on the first time checked
+                            if changed_detected or not watch['last_checked']:
+                                watch.save_history_text(contents=contents,
+                                                        timestamp=str(round(time.time())),
+                                                        snapshot_id=update_obj.get('previous_md5', 'none'))
+
                             # A change was detected
                             if changed_detected:
                                 print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))

docs/api_v1/assets/main.css

@@ -49,6 +49,7 @@ input[type="date"] {
   src: url('./glyphicons-halflings-regular.eot');
   src: url('./glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),
     url('./glyphicons-halflings-regular.woff') format('woff'),
+    url('./glyphicons-halflings-regular.woff2') format('woff2'),
     url('./glyphicons-halflings-regular.ttf') format('truetype'),
     url('./glyphicons-halflings-regular.svg#glyphicons-halflingsregular') format('svg');
 }

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css" rel="stylesheet" media="screen">
-  <link href="assets/prism.css" rel="stylesheet" />
-  <link href="assets/main.css" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png"rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
+  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js"></script>
+<script src="assets/main.bundle.js?v=1677105736053"></script>
 </body>
 </html>

docs/apidoc.json

@@ -3,5 +3,6 @@
   "version": "0.1.0",
   "description": "Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.",
   "title": "changedetection.io API",
-  "url" : "https://changedetection.io/docs/api_v1/index.html"
+  "url" : "",
+  "sampleUrl" : false
 }

docs/package.json

@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "apidoc": "^0.53.1"
+    "apidoc": "^0.54.0"
   }
 }

requirements.txt

@@ -31,7 +31,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.2.1
+apprise~=1.3.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -42,7 +42,7 @@ paho-mqtt
 cryptography~=3.4
 
 # Used for CSS filtering
-bs4
+beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml