Cleaner history suffix handling

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,20 @@ assignees: 'dgtlmoon'
 
 ---
 
+**DO NOT USE THIS FORM TO REPORT THAT A PARTICULAR WEBSITE IS NOT SCRAPING/WATCHING AS EXPECTED**
+
+This form is only for direct bugs and feature requests todo directly with the software.
+
+Please report watched websites (full URL and _any_ settings) that do not work with changedetection.io as expected [**IN THE DISCUSSION FORUMS**](https://github.com/dgtlmoon/changedetection.io/discussions) or your report will be deleted
+
+CONSIDER TAKING OUT A SUBSCRIPTION FOR A SMALL PRICE PER MONTH, YOU GET THE BENEFIT OF USING OUR PAID PROXIES AND FURTHERING THE DEVELOPMENT OF CHANGEDETECTION.IO
+
+THANK YOU
+
+
+
+
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 

README.md

@@ -1,7 +1,6 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-[**Try our $6.99/month subscription - Unlimited checks and watches!**](https://lemonade.changedetection.io/start)
- 
+Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
 
@@ -11,7 +10,7 @@
 
 Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
 
-[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 
 
@@ -40,7 +39,18 @@ Know when important content changes, we support notifications via Discord, Teleg
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
 
-_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
+
+#### Key Features
+
+- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JsonPath rules
+- Switch between fast non-JS and Chrome JS based "fetchers"
+- Easily specify how often a site should be checked
+- Execute JS before extracting text (Good for logging in, see examples in the UI!)
+- Override Request Headers, Specify `POST` or `GET` and other methods
+- Use the "Visual Selector" to help target specific elements
+
 
 ## Screenshots
 

changedetectionio/__init__.py

@@ -500,10 +500,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         import hashlib
 
-        from changedetectionio import fetch_site_status
+        from .fetch_processor import json_html_plaintext
 
         # Get the most recent one
-        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
 
         # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:
@@ -514,7 +514,7 @@ def changedetection_app(config=None, datastore_o=None):
                       encoding='utf-8') as file:
                 raw_content = file.read()
 
-                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                handler = json_html_plaintext.perform_site_check(datastore=datastore)
                 stripped_content = html_tools.strip_ignore_text(raw_content,
                                                              datastore.data['watching'][uuid]['ignore_text'])
 

changedetectionio/fetch_processor/__init__.py

@@ -0,0 +1,41 @@
+class fetch_processor():
+    contents = b''
+    screenshot = None
+    history_artifact_suffix = 'txt'
+
+    """
+    base class for all fetch processors
+    - json_html_plaintext
+    - image (future)
+    """
+    def __init__(self, *args, datastore, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.datastore = datastore
+
+    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
+    # if watch.proxy use that
+    # fetcher.proxy_override = watch.proxy or main config proxy
+    # Allows override the proxy on a per-request basis
+    # ALWAYS use the first one is nothing selected
+
+    def set_proxy_from_list(self, watch):
+        proxy_args = None
+        if self.datastore.proxy_list is None:
+            return None
+
+        # If its a valid one
+        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
+            proxy_args = watch['proxy']
+
+        # not valid (including None), try the system one
+        else:
+            system_proxy = self.datastore.data['settings']['requests']['proxy']
+            # Is not None and exists
+            if any([system_proxy in p for p in self.datastore.proxy_list]):
+                proxy_args = system_proxy
+
+        # Fallback - Did not resolve anything, use the first available
+        if proxy_args is None:
+            proxy_args = self.datastore.proxy_list[0][0]
+
+        return proxy_args

changedetectionio/fetch_processor/json_html_plaintext.py

@@ -9,45 +9,14 @@ from changedetectionio import content_fetcher, html_tools
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+from . import fetch_processor
 
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
-class perform_site_check():
-    screenshot = None
+class perform_site_check(fetch_processor):
+
     xpath_data = None
 
-    def __init__(self, *args, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
-    # if watch.proxy use that
-    # fetcher.proxy_override = watch.proxy or main config proxy
-    # Allows override the proxy on a per-request basis
-    # ALWAYS use the first one is nothing selected
-
-    def set_proxy_from_list(self, watch):
-        proxy_args = None
-        if self.datastore.proxy_list is None:
-            return None
-
-        # If its a valid one
-        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
-            proxy_args = watch['proxy']
-
-        # not valid (including None), try the system one
-        else:
-            system_proxy = self.datastore.data['settings']['requests']['proxy']
-            # Is not None and exists
-            if any([system_proxy in p for p in self.datastore.proxy_list]):
-                proxy_args = system_proxy
-
-        # Fallback - Did not resolve anything, use the first available
-        if proxy_args is None:
-            proxy_args = self.datastore.proxy_list[0][0]
-
-        return proxy_args
-
     # Doesn't look like python supports forward slash auto enclosure in re.findall
     # So convert it to inline flag "foobar(?i)" type configuration
     def forward_slash_enclosed_regex_to_options(self, regex):
@@ -63,13 +32,11 @@ class perform_site_check():
 
 
     def run(self, uuid):
-        timestamp = int(time.time())  # used for storage etc too
-
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        watch = self.datastore.data['watching'][uuid]
+        watch = self.datastore.data['watching'].get(uuid)
 
         # Protect against file:// access
         if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -80,7 +47,7 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers = self.datastore.get_val(uuid, 'headers')
+        extra_headers =self.datastore.data['watching'][uuid].get('headers')
 
         # Tweak the base config with the per-watch ones
         request_headers = self.datastore.data['settings']['headers'].copy()
@@ -93,9 +60,9 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests']['timeout']
-        url = self.datastore.get_val(uuid, 'url')
-        request_body = self.datastore.get_val(uuid, 'body')
-        request_method = self.datastore.get_val(uuid, 'method')
+        url = watch.get('url')
+        request_body = self.datastore.data['watching'][uuid].get('body')
+        request_method = self.datastore.data['watching'][uuid].get('method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
 
         # source: support
@@ -317,4 +284,6 @@ class perform_site_check():
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        return changed_detected, update_obj, text_content_before_ignored_filter
+        self.contents = text_content_before_ignored_filter
+
+        return changed_detected, update_obj

changedetectionio/forms.py

@@ -355,6 +355,8 @@ class watchForm(commonSettingsForm):
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
 
+    notification_use_default = BooleanField('Use default/system notification settings', default=True)
+
     def validate(self, **kwargs):
         if not super().validate():
             return False

changedetectionio/model/Watch.py

@@ -35,6 +35,7 @@ class model(dict):
             'notification_title': default_notification_title,
             'notification_body': default_notification_body,
             'notification_format': default_notification_format,
+            'notification_use_default': True, # Use default for new
             'notification_muted': False,
             'css_filter': '',
             'last_error': False,
@@ -147,9 +148,7 @@ class model(dict):
         bump = self.history
         return self.__newest_history_key
 
-    # Save some text file to the appropriate path and bump the history
-    # result_obj from fetch_site_status.run()
-    def save_history_text(self, contents, timestamp):
+    def save_history_artifact(self, contents: bytes, timestamp, suffix='txt'):
         import uuid
         import logging
 
@@ -157,8 +156,8 @@ class model(dict):
 
         self.ensure_data_dir_exists()
 
-        snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
-        logging.debug("Saving history text {}".format(snapshot_fname))
+        snapshot_fname = "{}/{}.{}".format(output_path, uuid.uuid4(), suffix)
+        logging.debug("Saving history artifact {}".format(snapshot_fname))
 
         with open(snapshot_fname, 'wb') as f:
             f.write(contents)

changedetectionio/static/js/watch-settings.js

@@ -1,7 +1,7 @@
-$(document).ready(function() {
-    function toggle() {
+$(document).ready(function () {
+    function toggle_fetch_backend() {
         if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if(playwright_enabled) {
+            if (playwright_enabled) {
                 // playwright supports headers, so hide everything else
                 // See #664
                 $('#requests-override-options #request-method').hide();
@@ -13,12 +13,8 @@ $(document).ready(function() {
                 // selenium/webdriver doesnt support anything afaik, hide it all
                 $('#requests-override-options').hide();
             }
-
-
             $('#webdriver-override-options').show();
-
         } else {
-
             $('#requests-override-options').show();
             $('#requests-override-options *:hidden').show();
             $('#webdriver-override-options').hide();
@@ -26,8 +22,27 @@ $(document).ready(function() {
     }
 
     $('input[name="fetch_backend"]').click(function (e) {
-        toggle();
+        toggle_fetch_backend();
     });
-    toggle();
+    toggle_fetch_backend();
 
+    function toggle_default_notifications() {
+        var n=$('#notification_urls, #notification_title, #notification_body, #notification_format');
+        if ($('#notification_use_default').is(':checked')) {
+            $('#notification-field-group').fadeOut();
+            $(n).each(function (e) {
+                $(this).attr('readonly', true);
+            });
+        } else {
+            $('#notification-field-group').show();
+            $(n).each(function (e) {
+                $(this).attr('readonly', false);
+            });
+        }
+    }
+
+    $('#notification_use_default').click(function (e) {
+        toggle_default_notifications();
+    });
+    toggle_default_notifications();
 });

changedetectionio/store.py

@@ -244,10 +244,6 @@ class ChangeDetectionStore:
 
         return False
 
-    def get_val(self, uuid, val):
-        # Probably their should be dict...
-        return self.data['watching'][uuid].get(val)
-
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         import pathlib
@@ -539,4 +535,28 @@ class ChangeDetectionStore:
                 del(watch['last_changed'])
             except:
                 continue
+        return
+
+
+    def update_5(self):
+
+        from changedetectionio.notification import (
+            default_notification_body,
+            default_notification_format,
+            default_notification_title,
+        )
+
+        for uuid, watch in self.data['watching'].items():
+            try:
+                # If it's all the same to the system settings, then prefer system notification settings
+                # include \r\n -> \n incase they already hit submit and the browser put \r in
+                if watch.get('notification_body').replace('\r\n', '\n') == default_notification_body.replace('\r\n', '\n') and \
+                        watch.get('notification_format') == default_notification_format and \
+                        watch.get('notification_title').replace('\r\n', '\n') == default_notification_title.replace('\r\n', '\n') and \
+                        watch.get('notification_urls') == self.__data['settings']['application']['notification_urls']:
+                        watch['notification_use_default'] = True
+                else:
+                    watch['notification_use_default'] = False
+            except:
+                continue
         return

changedetectionio/templates/edit.html

@@ -135,9 +135,11 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="notifications">
-                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
-                    <div class="field-group">
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_use_default) }}
+                    </div>
+                    <div class="field-group" id="notification-field-group">
                         {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>

changedetectionio/templates/preview.html

@@ -57,6 +57,7 @@
          </br>
          {% if is_html_webdriver %}
            {% if screenshot %}
+             <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
              <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
            {% else %}
               No screenshot available just yet! Try rechecking the page.

changedetectionio/tests/test_css_selector.py

@@ -47,7 +47,6 @@ def set_modified_response():
 
 # Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
 def test_css_filter_output():
-    from changedetectionio import fetch_site_status
     from inscriptis import get_text
 
     # Check text with sub-parts renders correctly

changedetectionio/tests/test_element_removal.py

@@ -71,7 +71,6 @@ def set_modified_response():
 
 
 def test_element_removal_output():
-    from changedetectionio import fetch_site_status
     from inscriptis import get_text
 
     # Check text with sub-parts renders correctly

changedetectionio/tests/test_ignore_regex_text.py

@@ -1,7 +1,5 @@
 #!/usr/bin/python3
 
-import time
-from flask import url_for
 from . util import live_server_setup
 from changedetectionio import html_tools
 
@@ -11,7 +9,7 @@ def test_setup(live_server):
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_regex_text_func():
-    from changedetectionio import fetch_site_status
+    from ..fetch_processor import json_html_plaintext
 
     test_content = """
     but sometimes we want to remove the lines.
@@ -23,7 +21,7 @@ def test_strip_regex_text_func():
 
     ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
 
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    fetcher = json_html_plaintext.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"but 1 lines" in stripped_content

changedetectionio/tests/test_ignore_text.py

@@ -11,7 +11,7 @@ def test_setup(live_server):
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_text_func():
-    from changedetectionio import fetch_site_status
+    from ..fetch_processor import json_html_plaintext
 
     test_content = """
     Some content
@@ -23,7 +23,9 @@ def test_strip_text_func():
 
     ignore_lines = ["sometimes"]
 
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    from ..fetch_processor import json_html_plaintext
+
+    fetcher = json_html_plaintext.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"sometimes" not in stripped_content

changedetectionio/tests/test_notification.py

@@ -71,6 +71,7 @@ def test_check_notification(client, live_server):
         "url": test_url,
         "tag": "my tag",
         "title": "my title",
+        # No 'notification_use_default' here, so it's effectively False/off
         "headers": "",
         "fetch_backend": "html_requests"})
 
@@ -215,3 +216,82 @@ def test_notification_validation(client, live_server):
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
+
+# Check that the default VS watch specific notification is hit
+def test_check_notification_use_default(client, live_server):
+    set_original_response()
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    test_url = url_for('test_endpoint', _external=True)
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tag": ''},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+
+    ## Setup the local one and enable it
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "watch-notification",
+              "notification_body": "watch-body",
+              'notification_use_default': "True",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_title": "global-notifications-title",
+              "application-notification_body": "global-notifications-body\n",
+              "application-notification_format": "Text",
+              "application-notification_urls": notification_url,
+              "requests-time_between_check-minutes": 180,
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    # A change should by default trigger a notification of the global-notifications
+    time.sleep(1)
+    set_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    with open("test-datastore/notification.txt", "r") as f:
+        assert 'global-notifications-title' in f.read()
+
+    ## Setup the local one and enable it
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "watch-notification",
+              "notification_body": "watch-body",
+              # No 'notification_use_default' here, so it's effectively False/off = "dont use default, use this one"
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+    set_original_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    assert os.path.isfile("test-datastore/notification.txt")
+    with open("test-datastore/notification.txt", "r") as f:
+        assert 'watch-notification' in f.read()
+
+
+    # cleanup for the next
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -7,6 +7,7 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
+    import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
     live_server_setup(live_server)
@@ -33,3 +34,7 @@ def test_visual_selector_content_ready(client, live_server):
     uuid = extract_UUID_from_client(client)
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
+
+    # Open it and see if it roughly looks correct
+    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
+        json.load(f)

changedetectionio/update_worker.py

@@ -41,7 +41,7 @@ class update_worker(threading.Thread):
             )
 
         # Did it have any notification alerts to hit?
-        if len(watch['notification_urls']):
+        if not watch.get('notification_use_default') and len(watch['notification_urls']):
             print(">>> Notifications queued for UUID from watch {}".format(watch_uuid))
             n_object['notification_urls'] = watch['notification_urls']
             n_object['notification_title'] = watch['notification_title']
@@ -49,7 +49,7 @@ class update_worker(threading.Thread):
             n_object['notification_format'] = watch['notification_format']
 
         # No? maybe theres a global setting, queue them all
-        elif len(self.datastore.data['settings']['application']['notification_urls']):
+        elif watch.get('notification_use_default') and len(self.datastore.data['settings']['application']['notification_urls']):
             print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(watch_uuid))
             n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
             n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
@@ -117,9 +117,9 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
-        from changedetectionio import fetch_site_status
+        from .fetch_processor import json_html_plaintext
+
 
-        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
 
         while not self.app.config.exit.is_set():
 
@@ -132,21 +132,20 @@ class update_worker(threading.Thread):
                 self.current_uuid = uuid
 
                 if uuid in list(self.datastore.data['watching'].keys()):
+                    update_handler = None # Interface object
                     changed_detected = False
-                    contents = b''
-                    screenshot = False
                     update_obj= {}
-                    xpath_data = False
                     process_changedetection_results = True
                     print("> Processing UUID {} Priority {} URL {}".format(uuid, priority, self.datastore.data['watching'][uuid]['url']))
                     now = time.time()
 
                     try:
-                        changed_detected, update_obj, contents = update_handler.run(uuid)
+                        update_handler = json_html_plaintext.perform_site_check(datastore=self.datastore)
+                        changed_detected, update_obj = update_handler.run(uuid)
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
-                        if not isinstance(contents, (bytes, bytearray)):
+                        if not isinstance(update_handler.contents, (bytes, bytearray)):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
@@ -251,12 +250,11 @@ class update_worker(threading.Thread):
                     if process_changedetection_results:
                         try:
                             watch = self.datastore.data['watching'][uuid]
-                            fname = "" # Saved history text filename
 
                             # For the FIRST time we check a site, or a change detected, save the snapshot.
                             if changed_detected or not watch['last_checked']:
                                 # A change was detected
-                                watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
+                                watch.save_history_artifact(contents=update_handler.contents, timestamp=str(round(time.time())), suffix=update_handler.history_artifact_suffix)
 
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)