Refactor form handling (#548)

changedetectionio/__init__.py

@@ -94,16 +94,6 @@ def init_app_secret(datastore_path):
 
     return secret
 
-# Remember python is by reference
-# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
-def populate_form_from_watch(form, watch):
-    for i in form.__dict__.keys():
-        if i[0] != '_':
-            p = getattr(form, i)
-            if hasattr(p, 'data') and i in watch:
-                setattr(p, "data", watch[i])
-
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -320,6 +310,7 @@ def changedetection_app(config=None, datastore_o=None):
                 guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
                 fe = fg.add_entry()
 
+
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
                 base_url = datastore.data['settings']['application']['base_url']
@@ -520,49 +511,46 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_required
+    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
+    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
+
     def edit_page(uuid):
         from changedetectionio import forms
-        form = forms.watchForm(request.form)
+
 
         # More for testing, possible to return the first/only
+        if not datastore.data['watching'].keys():
+            flash("No watches to edit", "error")
+            return redirect(url_for('index'))
+
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
+        if not uuid in datastore.data['watching']:
+            flash("No watch with the UUID %s found." % (uuid), "error")
+            return redirect(url_for('index'))
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=datastore.data['watching'][uuid]
+                                        )
 
         if request.method == 'GET':
-            if not uuid in datastore.data['watching']:
+            # Set some defaults that refer to the main config when None, we do the same in POST,
-                flash("No watch with the UUID %s found." % (uuid), "error")
+            # probably there should be a nice little handler for this.
-                return redirect(url_for('index'))
-
-            populate_form_from_watch(form, datastore.data['watching'][uuid])
-
             if datastore.data['watching'][uuid]['fetch_backend'] is None:
                 form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+            if datastore.data['watching'][uuid]['minutes_between_check'] is None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
 
         if request.method == 'POST' and form.validate():
 
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
             if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
                 form.minutes_between_check.data = None
-
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
                 form.fetch_backend.data = None
 
-            update_obj = {'url': form.url.data.strip(),
+            extra_update_obj = {}
-                          'minutes_between_check': form.minutes_between_check.data,
-                          'tag': form.tag.data.strip(),
-                          'title': form.title.data.strip(),
-                          'headers': form.headers.data,
-                          'body': form.body.data,
-                          'method': form.method.data,
-                          'ignore_status_codes': form.ignore_status_codes.data,
-                          'fetch_backend': form.fetch_backend.data,
-                          'trigger_text': form.trigger_text.data,
-                          'notification_title': form.notification_title.data,
-                          'notification_body': form.notification_body.data,
-                          'notification_format': form.notification_format.data,
-                          'extract_title_as_title': form.extract_title_as_title.data,
-                          }
 
             # Notification URLs
             datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
@@ -574,18 +562,15 @@ def changedetection_app(config=None, datastore_o=None):
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form_ignore_text:
                 if len(datastore.data['watching'][uuid]['history']):
-                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
-
-            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
-            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                 if len(datastore.data['watching'][uuid]['history']):
-                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-            datastore.data['watching'][uuid].update(update_obj)
+            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(extra_update_obj)
 
             flash("Updated watch.")
 
@@ -610,17 +595,12 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            # Re #110 offer the default minutes
+            has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime
-            using_default_minutes = False
-            if form.minutes_between_check.data == None:
-                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
-                using_default_minutes = True
-
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
-                                     using_default_minutes=using_default_minutes,
+                                     has_empty_checktime=has_empty_checktime,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -630,61 +610,39 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/settings", methods=['GET', "POST"])
     @login_required
     def settings_page():
-
         from changedetectionio import content_fetcher, forms
 
-        form = forms.globalSettingsForm(request.form)
+        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=datastore.data['settings']
+                                        )
 
-        if request.method == 'GET':
+        if request.method == 'POST':
-            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
-            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
-            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
-            form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
-            form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
-            form.render_anchor_tag_content.data = datastore.data['settings']['application']['render_anchor_tag_content']
-            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
-            form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
-            form.notification_title.data = datastore.data['settings']['application']['notification_title']
-            form.notification_body.data = datastore.data['settings']['application']['notification_body']
-            form.notification_format.data = datastore.data['settings']['application']['notification_format']
-            form.base_url.data = datastore.data['settings']['application']['base_url']
-            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
-
-        if request.method == 'POST' and form.data.get('removepassword_button') == True:
             # Password unset is a GET, but we can lock the session to a salted env password to always need the password
-            if not os.getenv("SALTED_PASS", False):
+            if form.application.form.data.get('removepassword_button', False):
-                datastore.data['settings']['application']['password'] = False
+                # SALTED_PASS means the password is "locked" to what we set in the Env var
-                flash("Password protection removed.", 'notice')
+                if not os.getenv("SALTED_PASS", False):
-                flask_login.logout_user()
+                    datastore.remove_password()
-                return redirect(url_for('settings_page'))
+                    flash("Password protection removed.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('settings_page'))
 
-        if request.method == 'POST' and form.validate():
+            if form.validate():
-            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+                datastore.data['settings']['application'].update(form.data['application'])
-            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
+                datastore.data['settings']['requests'].update(form.data['requests'])
-            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+                datastore.needs_write = True
-            datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
-            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
-            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
-            datastore.data['settings']['application']['notification_format'] = form.notification_format.data
-            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-            datastore.data['settings']['application']['base_url'] = form.base_url.data
-            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
-            datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
-            datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
-            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
-            datastore.data['settings']['application']['render_anchor_tag_content'] = form.render_anchor_tag_content.data
 
-            if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password:
+                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
-                datastore.data['settings']['application']['password'] = form.password.encrypted_password
+                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
-                flash("Password protection enabled.", 'notice')
+                    datastore.needs_write = True
-                flask_login.logout_user()
+                    flash("Password protection enabled.", 'notice')
-                return redirect(url_for('index'))
+                    flask_login.logout_user()
+                    return redirect(url_for('index'))
 
-            datastore.needs_write = True
+                flash("Settings updated.")
-            flash("Settings updated.")
 
-        if request.method == 'POST' and not form.validate():
+            else:
-            flash("An error occurred, please see below.", "error")
+                flash("An error occurred, please see below.", "error")
 
         output = render_template("settings.html",
                                  form=form,
@@ -1172,8 +1130,6 @@ def notification_runner():
                 notification_debug_log = notification_debug_log[-100:]
 
 
-
-
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
     from changedetectionio import update_worker
@@ -1210,7 +1166,9 @@ def ticker_thread_check_time_launch_checks():
 
         # Check for watches outside of the time threshold to put in the thread queue.
         now = time.time()
-        max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
+
+        recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
+        recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
 
         for uuid, watch in copied_datastore.data['watching'].items():
 
@@ -1219,18 +1177,14 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            watch_minutes_between_check = watch.get('minutes_between_check', None)
+            threshold = now
-            if watch_minutes_between_check is not None:
+            if watch.threshold_seconds:
-                # Cast to int just incase
+                threshold -= watch.threshold_seconds
-                max_time = int(watch_minutes_between_check) * 60
             else:
-                # Default system wide.
+                threshold -= recheck_time_system_seconds
-                max_time = max_system_wide
-
-            threshold = now - max_time
 
             # Yeah, put it in the queue, it's more than time
-            if watch['last_checked'] <= threshold:
+            if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds):
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
@@ -1238,4 +1192,4 @@ def ticker_thread_check_time_launch_checks():
         time.sleep(3)
 
         # Should be low so we can break this out in testing
-        app.config.exit.wait(1)
+        app.config.exit.wait(1)

changedetectionio/forms.py

@@ -25,6 +25,8 @@ from changedetectionio.notification import (
     valid_notification_formats,
 )
 
+from wtforms.fields import FormField
+
 valid_method = {
     'GET',
     'POST',
@@ -121,7 +123,6 @@ class ValidateContentFetcherIsReady(object):
 
     def __call__(self, form, field):
         import urllib3.exceptions
-
         from changedetectionio import content_fetcher
 
         # Better would be a radiohandler that keeps a reference to each class
@@ -297,6 +298,7 @@ class quickWatchForm(Form):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
 
+# Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
     notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
@@ -342,19 +344,31 @@ class watchForm(commonSettingsForm):
 
         return result
 
-class globalSettingsForm(commonSettingsForm):
+
-    password = SaltyPasswordField()
+# datastore.data['settings']['requests']..
+class globalSettingsRequestForm(Form):
     minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
                                                [validators.NumberRange(min=1)])
-    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+# datastore.data['settings']['application']..
+class globalSettingsApplicationForm(commonSettingsForm):
+
     base_url = StringField('Base URL', validators=[validators.Optional()])
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     ignore_whitespace = BooleanField('Ignore whitespace')
-
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
-    render_anchor_tag_content = BooleanField('Render anchor tag content',
-                                             default=False)
-
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome')
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    password = SaltyPasswordField()
+
+
+class globalSettingsForm(Form):
+    # Define these as FormFields/"sub forms", this way it matches the JSON storage
+    # datastore.data['settings']['application']..
+    # datastore.data['settings']['requests']..
+
+    requests = FormField(globalSettingsRequestForm)
+    application = FormField(globalSettingsApplicationForm)
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+

changedetectionio/model/App.py

@@ -0,0 +1,49 @@
+import collections
+import os
+
+import uuid as uuid_builder
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+class model(dict):
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update({
+            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
+            'watching': {},
+            'settings': {
+                'headers': {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
+                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
+                },
+                'requests': {
+                    'timeout': 15,  # Default 15 seconds
+                    # Default 3 hours
+                    'minutes_between_check': 3 * 60,  # Default 3 hours
+                    'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'base_url' : None,
+                    'extract_title_as_title': False,
+                    'fetch_backend': 'html_requests',
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
+                    'ignore_whitespace': False,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
+                    'schema_version' : 0
+                }
+            }
+        })

changedetectionio/model/Watch.py

@@ -0,0 +1,60 @@
+import os
+
+import uuid as uuid_builder
+
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5))
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+
+class model(dict):
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update({
+            'url': None,
+            'tag': None,
+            'last_checked': 0,
+            'last_changed': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'newest_history_key': "",
+            'title': None,
+            'previous_md5': "",
+            'uuid': str(uuid_builder.uuid4()),
+            'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'css_filter': "",
+            'subtractive_selectors': [],
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
+            'extract_title_as_title': False,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            # Should be all None by default, so we use the system default in this case.
+            'minutes_between_check': None
+        })
+
+    @property
+    def has_empty_checktime(self):
+        if self.get('minutes_between_check', None):
+            return False
+        return True
+
+    @property
+    def threshold_seconds(self):
+        sec = self.get('minutes_between_check', None)
+        if sec:
+            sec = sec * 60
+        return sec

changedetectionio/store.py

@@ -8,11 +8,7 @@ from copy import deepcopy
 from os import mkdir, path, unlink
 from threading import Lock
 
-from changedetectionio.notification import (
+from changedetectionio.model import Watch, App
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-)
 
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -29,71 +25,10 @@ class ChangeDetectionStore:
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
 
-        self.__data = {
+        self.__data = App.model()
-            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
-            'watching': {},
-            'settings': {
-                'headers': {
-                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
-                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
-                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
-                },
-                'requests': {
-                    'timeout': 15,  # Default 15 seconds
-                    'minutes_between_check': 3 * 60,  # Default 3 hours
-                    'workers': 10  # Number of threads, lower is better for slow connections
-                },
-                'application': {
-                    'password': False,
-                    'base_url' : None,
-                    'extract_title_as_title': False,
-                    'fetch_backend': 'html_requests',
-                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
-                    'global_subtractive_selectors': [],
-                    'ignore_whitespace': False,
-                    'render_anchor_tag_content': False,
-                    'notification_urls': [], # Apprise URL list
-                    # Custom notification content
-                    'notification_title': default_notification_title,
-                    'notification_body': default_notification_body,
-                    'notification_format': default_notification_format,
-                    'real_browser_save_screenshot': True,
-                }
-            }
-        }
 
         # Base definition for all watchers
-        self.generic_definition = {
+        self.generic_definition = Watch.model()
-            'url': None,
-            'tag': None,
-            'last_checked': 0,
-            'last_changed': 0,
-            'paused': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'newest_history_key': "",
-            'title': None,
-            # Re #110, so then if this is set to None, we know to use the default value instead
-            # Requires setting to None on submit if it's the same as the default
-            'minutes_between_check': None,
-            'previous_md5': "",
-            'uuid': str(uuid_builder.uuid4()),
-            'headers': {},  # Extra headers to send
-            'body': None,
-            'method': 'GET',
-            'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
-            # Custom notification content
-            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': default_notification_title,
-            'notification_body': default_notification_body,
-            'notification_format': default_notification_format,
-            'css_filter': "",
-            'subtractive_selectors': [],
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'fetch_backend': None,
-            'extract_title_as_title': False
-        }
 
         if path.isfile('changedetectionio/source.txt'):
             with open('changedetectionio/source.txt') as f:
@@ -190,6 +125,10 @@ class ChangeDetectionStore:
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
+    def remove_password(self):
+        self.__data['settings']['application']['password'] = False
+        self.needs_write = True
+
     def update_watch(self, uuid, update_obj):
 
         with self.lock:

changedetectionio/templates/edit.html

@@ -42,7 +42,7 @@
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.minutes_between_check) }}
-                        {% if using_default_minutes %}
+                        {% if has_empty_checktime %}
                         <span class="pure-form-message-inline">Currently using the <a
                                 href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
                         {% else %}

changedetectionio/templates/settings.html

@@ -28,15 +28,15 @@
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
-                        {{ render_field(form.minutes_between_check) }}
+                        {{ render_field(form.requests.form.minutes_between_check) }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}
-                                {{ render_button(form.removepassword_button) }}
+                                {{ render_button(form.application.form.removepassword_button) }}
                             {% else %}
-                            {{ render_field(form.password) }}
+                            {{ render_field(form.application.form.password) }}
                             <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                             {% endif %}
                         {% else %}
@@ -44,7 +44,7 @@
                         {% endif %}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
+                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
                             Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
@@ -53,12 +53,12 @@
                     </div>
 
                     <div class="pure-control-group">
-                        {{ render_checkbox_field(form.extract_title_as_title) }}
+                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                     </div>
 
                     <div class="pure-control-group">
-                        {{ render_checkbox_field(form.real_browser_save_screenshot) }}
+                        {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
 
@@ -68,14 +68,14 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
+                        {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="fetching">
                 <div class="pure-control-group">
-                    {{ render_field(form.fetch_backend) }}
+                    {{ render_field(form.application.form.fetch_backend) }}
                     <span class="pure-form-message-inline">
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
@@ -87,20 +87,20 @@
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.ignore_whitespace) }}
+                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
                     <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
                     <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                 <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.render_anchor_tag_content) }}
+                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
                     <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
                         <br/>
                     <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                     <fieldset class="pure-group">
-                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
+                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
@@ -112,7 +112,7 @@ nav
                       </span>
                     </fieldset>
                     <fieldset class="pure-group">
-                    {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                     ") }}
                     <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>

changedetectionio/tests/conftest.py

@@ -16,6 +16,7 @@ def cleanup(datastore_path):
     # Unlink test output files
     files = ['output.txt',
              'url-watches.json',
+             'secret.txt',
              'notification.txt',
              'count.txt',
              'endpoint-content.txt'

changedetectionio/tests/test_access_control.py

@@ -1,5 +1,5 @@
 from flask import url_for
-
+from . util import live_server_setup
 
 def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
@@ -12,9 +12,9 @@ def test_check_access_control(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "foobar",
+            data={"application-password": "foobar",
-                  "minutes_between_check": 180,
+                  "requests-minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
+                  'application-fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
@@ -49,74 +49,31 @@ def test_check_access_control(app, client):
         assert b"minutes_between_check" in res.data
         assert b"fetch_backend" in res.data
 
+        ##################################################
+        # Remove password button, and check that it worked
+        ##################################################
         res = c.post(
             url_for("settings_page"),
             data={
-                "minutes_between_check": 180,
+                "requests-minutes_between_check": 180,
-                "tag": "",
+                "application-fetch_backend": "html_webdriver",
-                "headers": "",
+                "application-removepassword_button": "Remove password"
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
             },
             follow_redirects=True,
         )
+        assert b"Password protection removed." in res.data
+        assert b"LOG OUT" not in res.data
 
-# There was a bug where saving the settings form would submit a blank password
+        ############################################################
-def test_check_access_control_no_blank_password(app, client):
+        # Be sure a blank password doesnt setup password protection
-    # Still doesnt work, but this is closer.
+        ############################################################
-
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
-        assert b"Remove password" not in res.data
-
-        # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "",
+            data={"application-password": "",
-                  "minutes_between_check": 180,
+                  "requests-minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
+                  'application-fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
-        assert b"Password protection enabled." not in res.data
+        assert b"Password protection enabled" not in res.data
-        assert b"Login" not in res.data
 
-
-# There was a bug where saving the settings form would submit a blank password
-def test_check_access_no_remote_access_to_remove_password(app, client):
-    # Still doesnt work, but this is closer.
-
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
-        assert b"Remove password" not in res.data
-
-        # Enable password check.
-        res = c.post(
-            url_for("settings_page"),
-            data={"password": "password",
-                  "minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
-            follow_redirects=True
-        )
-
-        assert b"Password protection enabled." in res.data
-        assert b"Login" in res.data
-
-        res = c.post(
-            url_for("settings_page"),
-            data={
-                "minutes_between_check": 180,
-                "tag": "",
-                "headers": "",
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
-            },
-            follow_redirects=True,
-        )
-        assert b"Password protection removed." not in res.data
-
-        res = c.get(url_for("index"),
-              follow_redirects=True)
-        assert b"watch-table-wrapper" not in res.data

changedetectionio/tests/test_backend.py

@@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Enable auto pickup of <title> in settings
     res = client.post(
         url_for("settings_page"),
-        data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
+        data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_ignore_text.py

@@ -196,9 +196,9 @@ def test_check_global_ignore_text_functionality(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "global_ignore_text": ignore_text,
+            "application-global_ignore_text": ignore_text,
-            'fetch_backend': "html_requests"
+            'application-fetch_backend': "html_requests"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_ignorehyperlinks.py

@@ -55,9 +55,9 @@ def test_render_anchor_tag_content_true(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "render_anchor_tag_content": "true",
+            "application-render_anchor_tag_content": "true",
-            "fetch_backend": "html_requests",
+            "application-fetch_backend": "html_requests",
         },
         follow_redirects=True,
     )
@@ -116,9 +116,9 @@ def test_render_anchor_tag_content_false(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "render_anchor_tag_content": "false",
+            "application-render_anchor_tag_content": "false",
-            "fetch_backend": "html_requests",
+            "application-fetch_backend": "html_requests",
         },
         follow_redirects=True,
     )
@@ -175,8 +175,8 @@ def test_render_anchor_tag_content_default(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "fetch_backend": "html_requests",
+            "application-fetch_backend": "html_requests",
         },
         follow_redirects=True,
     )

changedetectionio/tests/test_ignorestatuscode.py

@@ -51,9 +51,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "ignore_status_codes": "y",
+            "application-ignore_status_codes": "y",
-            'fetch_backend': "html_requests"
+            'application-fetch_backend': "html_requests"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_ignorewhitespace.py

@@ -61,9 +61,9 @@ def test_check_ignore_whitespace(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "minutes_between_check": 180,
+            "requests-minutes_between_check": 180,
-            "ignore_whitespace": "y",
+            "application-ignore_whitespace": "y",
-            'fetch_backend': "html_requests"
+            "application-fetch_backend": "html_requests"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_notification.py

@@ -195,11 +195,11 @@ def test_notification_validation(client, live_server):
     # Now adding a wrong token should give us an error
     res = client.post(
         url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+        data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_body": "Rubbish: {rubbish}\n",
+              "application-notification_body": "Rubbish: {rubbish}\n",
-              "notification_format": "Text",
+              "application-notification_format": "Text",
-              "notification_urls": "json://localhost/foobar",
+              "application-notification_urls": "json://localhost/foobar",
-              "time_between_check": {'seconds': 180},
+              "requests-minutes_between_check": 180,
               "fetch_backend": "html_requests"
               },
         follow_redirects=True

changedetectionio/tests/test_request.py

@@ -84,9 +84,25 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    body_value = 'Test Body Value'
+    time.sleep(3)
 
-    # Add a properly formatted body with a proper method
+    # add the first 'version'
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "POST",
+              "fetch_backend": "html_requests",
+              "body": "something something"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    time.sleep(3)
+
+    # Now the change which should trigger a change
+    body_value = 'Test Body Value'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={

changedetectionio/tests/test_watch_fields_storage.py

@@ -56,8 +56,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "minutes_between_check": 1566,
+               "requests-minutes_between_check": 1566,
-               'fetch_backend': "html_requests"
+               'application-fetch_backend': "html_requests"
                },
         follow_redirects=True
     )
@@ -88,8 +88,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "minutes_between_check": 222,
+               "requests-minutes_between_check": 222,
-                'fetch_backend': "html_requests"
+                'application-fetch_backend': "html_requests"
                },
         follow_redirects=True
     )
@@ -124,8 +124,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "minutes_between_check": 666,
+               "requests-minutes_between_check": 666,
-                'fetch_backend': "html_requests"
+                'application-fetch_backend': "html_requests"
                },
         follow_redirects=True
     )

changedetectionio/tests/util.py

@@ -85,6 +85,7 @@ def live_server_setup(live_server):
     # Just return the body in the request
     @live_server.app.route('/test-body', methods=['POST', 'GET'])
     def test_body():
+        print ("TEST-BODY GOT", request.data, "returning")
         return request.data
 
     # Just return the verb in the request