Refactor form handling (#548)

2022-04-19 21:43:07 +02:00
parent 380c512cc2
commit 8f062bfec9
19 changed files with 283 additions and 292 deletions
--- a/changedetectionio/init.py
+++ b/changedetectionio/init.py
@@ -94,16 +94,6 @@ def init_app_secret(datastore_path):

    return secret

-# Remember python is by reference
-# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
-def populate_form_from_watch(form, watch):
-    for i in form.__dict__.keys():
-        if i[0] != '_':
-            p = getattr(form, i)
-            if hasattr(p, 'data') and i in watch:
-                setattr(p, "data", watch[i])
-
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
@app.template_filter('format_last_checked_time')
@@ -320,6 +310,7 @@ def changedetection_app(config=None, datastore_o=None):
                guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
                fe = fg.add_entry()

+
                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                # Description is the page you watch, link takes you to the diff JS UI page
                base_url = datastore.data['settings']['application']['base_url']
@@ -520,49 +511,46 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
    @login_required
+    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
+    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
+
    def edit_page(uuid):
        from changedetectionio import forms
-        form = forms.watchForm(request.form)
+

        # More for testing, possible to return the first/only
+        if not datastore.data['watching'].keys():
+            flash("No watches to edit", "error")
+            return redirect(url_for('index'))
+
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()

+        if not uuid in datastore.data['watching']:
+            flash("No watch with the UUID %s found." % (uuid), "error")
+            return redirect(url_for('index'))
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=datastore.data['watching'][uuid]
+                                        )

        if request.method == 'GET':
-            if not uuid in datastore.data['watching']:
-                flash("No watch with the UUID %s found." % (uuid), "error")
-                return redirect(url_for('index'))
-
-            populate_form_from_watch(form, datastore.data['watching'][uuid])
-
+            # Set some defaults that refer to the main config when None, we do the same in POST,
+            # probably there should be a nice little handler for this.
            if datastore.data['watching'][uuid]['fetch_backend'] is None:
                form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+            if datastore.data['watching'][uuid]['minutes_between_check'] is None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']

        if request.method == 'POST' and form.validate():

            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
                form.minutes_between_check.data = None
-
            if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
                form.fetch_backend.data = None

-            update_obj = {'url': form.url.data.strip(),
-                          'minutes_between_check': form.minutes_between_check.data,
-                          'tag': form.tag.data.strip(),
-                          'title': form.title.data.strip(),
-                          'headers': form.headers.data,
-                          'body': form.body.data,
-                          'method': form.method.data,
-                          'ignore_status_codes': form.ignore_status_codes.data,
-                          'fetch_backend': form.fetch_backend.data,
-                          'trigger_text': form.trigger_text.data,
-                          'notification_title': form.notification_title.data,
-                          'notification_body': form.notification_body.data,
-                          'notification_format': form.notification_format.data,
-                          'extract_title_as_title': form.extract_title_as_title.data,
-                          }
+            extra_update_obj = {}

            # Notification URLs
            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
@@ -574,18 +562,15 @@ def changedetection_app(config=None, datastore_o=None):
            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
            if form_ignore_text:
                if len(datastore.data['watching'][uuid]['history']):
-                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
-
-            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
-            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
+                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)

            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                if len(datastore.data['watching'][uuid]['history']):
-                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)

-            datastore.data['watching'][uuid].update(update_obj)
+            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(extra_update_obj)

            flash("Updated watch.")

@@ -610,17 +595,12 @@ def changedetection_app(config=None, datastore_o=None):
            if request.method == 'POST' and not form.validate():
                flash("An error occurred, please see below.", "error")

-            # Re #110 offer the default minutes
-            using_default_minutes = False
-            if form.minutes_between_check.data == None:
-                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
-                using_default_minutes = True
-
+            has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime
            output = render_template("edit.html",
                                     uuid=uuid,
                                     watch=datastore.data['watching'][uuid],
                                     form=form,
-                                     using_default_minutes=using_default_minutes,
+                                     has_empty_checktime=has_empty_checktime,
                                     current_base_url=datastore.data['settings']['application']['base_url'],
                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                     )
@@ -630,61 +610,39 @@ def changedetection_app(config=None, datastore_o=None):
    @app.route("/settings", methods=['GET', "POST"])
    @login_required
    def settings_page():
-
        from changedetectionio import content_fetcher, forms

-        form = forms.globalSettingsForm(request.form)
+        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=datastore.data['settings']
+                                        )

-        if request.method == 'GET':
-            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
-            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
-            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
-            form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
-            form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
-            form.render_anchor_tag_content.data = datastore.data['settings']['application']['render_anchor_tag_content']
-            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
-            form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
-            form.notification_title.data = datastore.data['settings']['application']['notification_title']
-            form.notification_body.data = datastore.data['settings']['application']['notification_body']
-            form.notification_format.data = datastore.data['settings']['application']['notification_format']
-            form.base_url.data = datastore.data['settings']['application']['base_url']
-            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
-
-        if request.method == 'POST' and form.data.get('removepassword_button') == True:
+        if request.method == 'POST':
            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
-            if not os.getenv("SALTED_PASS", False):
-                datastore.data['settings']['application']['password'] = False
-                flash("Password protection removed.", 'notice')
-                flask_login.logout_user()
-                return redirect(url_for('settings_page'))
+            if form.application.form.data.get('removepassword_button', False):
+                # SALTED_PASS means the password is "locked" to what we set in the Env var
+                if not os.getenv("SALTED_PASS", False):
+                    datastore.remove_password()
+                    flash("Password protection removed.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('settings_page'))

-        if request.method == 'POST' and form.validate():
-            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
-            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
-            datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
-            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
-            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
-            datastore.data['settings']['application']['notification_format'] = form.notification_format.data
-            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-            datastore.data['settings']['application']['base_url'] = form.base_url.data
-            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
-            datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
-            datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
-            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
-            datastore.data['settings']['application']['render_anchor_tag_content'] = form.render_anchor_tag_content.data
+            if form.validate():
+                datastore.data['settings']['application'].update(form.data['application'])
+                datastore.data['settings']['requests'].update(form.data['requests'])
+                datastore.needs_write = True

-            if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password:
-                datastore.data['settings']['application']['password'] = form.password.encrypted_password
-                flash("Password protection enabled.", 'notice')
-                flask_login.logout_user()
-                return redirect(url_for('index'))
+                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
+                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+                    datastore.needs_write = True
+                    flash("Password protection enabled.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('index'))

-            datastore.needs_write = True
-            flash("Settings updated.")
+                flash("Settings updated.")

-        if request.method == 'POST' and not form.validate():
-            flash("An error occurred, please see below.", "error")
+            else:
+                flash("An error occurred, please see below.", "error")

        output = render_template("settings.html",
                                 form=form,
@@ -1172,8 +1130,6 @@ def notification_runner():
                notification_debug_log = notification_debug_log[-100:]


-
-
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
    from changedetectionio import update_worker
@@ -1210,7 +1166,9 @@ def ticker_thread_check_time_launch_checks():

        # Check for watches outside of the time threshold to put in the thread queue.
        now = time.time()
-        max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
+
+        recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
+        recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60

        for uuid, watch in copied_datastore.data['watching'].items():

@@ -1219,18 +1177,14 @@ def ticker_thread_check_time_launch_checks():
                continue

            # If they supplied an individual entry minutes to threshold.
-            watch_minutes_between_check = watch.get('minutes_between_check', None)
-            if watch_minutes_between_check is not None:
-                # Cast to int just incase
-                max_time = int(watch_minutes_between_check) * 60
+            threshold = now
+            if watch.threshold_seconds:
+                threshold -= watch.threshold_seconds
            else:
-                # Default system wide.
-                max_time = max_system_wide
-
-            threshold = now - max_time
+                threshold -= recheck_time_system_seconds

            # Yeah, put it in the queue, it's more than time
-            if watch['last_checked'] <= threshold:
+            if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds):
                if not uuid in running_uuids and uuid not in update_q.queue:
                    update_q.put(uuid)

--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -25,6 +25,8 @@ from changedetectionio.notification import (
    valid_notification_formats,
 )

+from wtforms.fields import FormField
+
 valid_method = {
    'GET',
    'POST',
@@ -121,7 +123,6 @@ class ValidateContentFetcherIsReady(object):

    def __call__(self, form, field):
        import urllib3.exceptions
-
        from changedetectionio import content_fetcher

        # Better would be a radiohandler that keeps a reference to each class
@@ -297,6 +298,7 @@ class quickWatchForm(Form):
    url = fields.URLField('URL', validators=[validateURL()])
    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])

+# Common to a single watch and the global settings
 class commonSettingsForm(Form):

    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
@@ -342,19 +344,31 @@ class watchForm(commonSettingsForm):

        return result

-class globalSettingsForm(commonSettingsForm):
-    password = SaltyPasswordField()
+
+# datastore.data['settings']['requests']..
+class globalSettingsRequestForm(Form):
    minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
                                               [validators.NumberRange(min=1)])
-    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+# datastore.data['settings']['application']..
+class globalSettingsApplicationForm(commonSettingsForm):
+
    base_url = StringField('Base URL', validators=[validators.Optional()])
    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
    ignore_whitespace = BooleanField('Ignore whitespace')
-
-    render_anchor_tag_content = BooleanField('Render anchor tag content',
-                                             default=False)
-
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome')
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    password = SaltyPasswordField()
+
+
+class globalSettingsForm(Form):
+    # Define these as FormFields/"sub forms", this way it matches the JSON storage
+    # datastore.data['settings']['application']..
+    # datastore.data['settings']['requests']..
+
+    requests = FormField(globalSettingsRequestForm)
+    application = FormField(globalSettingsApplicationForm)
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
--- a/changedetectionio/model/App.py
+++ b/changedetectionio/model/App.py
@@ -0,0 +1,49 @@
+import collections
+import os
+
+import uuid as uuid_builder
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+class model(dict):
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update({
+            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
+            'watching': {},
+            'settings': {
+                'headers': {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
+                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
+                },
+                'requests': {
+                    'timeout': 15,  # Default 15 seconds
+                    # Default 3 hours
+                    'minutes_between_check': 3 * 60,  # Default 3 hours
+                    'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'base_url' : None,
+                    'extract_title_as_title': False,
+                    'fetch_backend': 'html_requests',
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
+                    'ignore_whitespace': False,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
+                    'schema_version' : 0
+                }
+            }
+        })
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -0,0 +1,60 @@
+import os
+
+import uuid as uuid_builder
+
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5))
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+
+class model(dict):
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update({
+            'url': None,
+            'tag': None,
+            'last_checked': 0,
+            'last_changed': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'newest_history_key': "",
+            'title': None,
+            'previous_md5': "",
+            'uuid': str(uuid_builder.uuid4()),
+            'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'css_filter': "",
+            'subtractive_selectors': [],
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
+            'extract_title_as_title': False,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            # Should be all None by default, so we use the system default in this case.
+            'minutes_between_check': None
+        })
+
+    @property
+    def has_empty_checktime(self):
+        if self.get('minutes_between_check', None):
+            return False
+        return True
+
+    @property
+    def threshold_seconds(self):
+        sec = self.get('minutes_between_check', None)
+        if sec:
+            sec = sec * 60
+        return sec
--- a/changedetectionio/model/init.py
+++ b/changedetectionio/model/init.py
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -8,11 +8,7 @@ from copy import deepcopy
 from os import mkdir, path, unlink
 from threading import Lock

-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-)
+from changedetectionio.model import Watch, App


 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -29,71 +25,10 @@ class ChangeDetectionStore:
        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
        self.stop_thread = False

-        self.__data = {
-            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
-            'watching': {},
-            'settings': {
-                'headers': {
-                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
-                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
-                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
-                },
-                'requests': {
-                    'timeout': 15,  # Default 15 seconds
-                    'minutes_between_check': 3 * 60,  # Default 3 hours
-                    'workers': 10  # Number of threads, lower is better for slow connections
-                },
-                'application': {
-                    'password': False,
-                    'base_url' : None,
-                    'extract_title_as_title': False,
-                    'fetch_backend': 'html_requests',
-                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
-                    'global_subtractive_selectors': [],
-                    'ignore_whitespace': False,
-                    'render_anchor_tag_content': False,
-                    'notification_urls': [], # Apprise URL list
-                    # Custom notification content
-                    'notification_title': default_notification_title,
-                    'notification_body': default_notification_body,
-                    'notification_format': default_notification_format,
-                    'real_browser_save_screenshot': True,
-                }
-            }
-        }
+        self.__data = App.model()

        # Base definition for all watchers
-        self.generic_definition = {
-            'url': None,
-            'tag': None,
-            'last_checked': 0,
-            'last_changed': 0,
-            'paused': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'newest_history_key': "",
-            'title': None,
-            # Re #110, so then if this is set to None, we know to use the default value instead
-            # Requires setting to None on submit if it's the same as the default
-            'minutes_between_check': None,
-            'previous_md5': "",
-            'uuid': str(uuid_builder.uuid4()),
-            'headers': {},  # Extra headers to send
-            'body': None,
-            'method': 'GET',
-            'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
-            # Custom notification content
-            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': default_notification_title,
-            'notification_body': default_notification_body,
-            'notification_format': default_notification_format,
-            'css_filter': "",
-            'subtractive_selectors': [],
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'fetch_backend': None,
-            'extract_title_as_title': False
-        }
+        self.generic_definition = Watch.model()

        if path.isfile('changedetectionio/source.txt'):
            with open('changedetectionio/source.txt') as f:
@@ -190,6 +125,10 @@ class ChangeDetectionStore:
        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
        self.needs_write = True

+    def remove_password(self):
+        self.__data['settings']['application']['password'] = False
+        self.needs_write = True
+
    def update_watch(self, uuid, update_obj):

        with self.lock:
--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -42,7 +42,7 @@
                    </div>
                    <div class="pure-control-group">
                        {{ render_field(form.minutes_between_check) }}
-                        {% if using_default_minutes %}
+                        {% if has_empty_checktime %}
                        <span class="pure-form-message-inline">Currently using the <a
                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
                        {% else %}
--- a/changedetectionio/templates/settings.html
+++ b/changedetectionio/templates/settings.html
@@ -28,15 +28,15 @@
            <div class="tab-pane-inner" id="general">
                <fieldset>
                    <div class="pure-control-group">
-                        {{ render_field(form.minutes_between_check) }}
+                        {{ render_field(form.requests.form.minutes_between_check) }}
                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                    </div>
                    <div class="pure-control-group">
                        {% if not hide_remove_pass %}
                            {% if current_user.is_authenticated %}
-                                {{ render_button(form.removepassword_button) }}
+                                {{ render_button(form.application.form.removepassword_button) }}
                            {% else %}
-                            {{ render_field(form.password) }}
+                            {{ render_field(form.application.form.password) }}
                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                            {% endif %}
                        {% else %}
@@ -44,7 +44,7 @@
                        {% endif %}
                    </div>
                    <div class="pure-control-group">
-                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
+                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                        class="m-d") }}
                        <span class="pure-form-message-inline">
                            Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
@@ -53,12 +53,12 @@
                    </div>

                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.extract_title_as_title) }}
+                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                    </div>

                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.real_browser_save_screenshot) }}
+                        {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                    </div>

@@ -68,14 +68,14 @@
            <div class="tab-pane-inner" id="notifications">
                <fieldset>
                    <div class="field-group">
-                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
+                        {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }}
                    </div>
                </fieldset>
            </div>

            <div class="tab-pane-inner" id="fetching">
                <div class="pure-control-group">
-                    {{ render_field(form.fetch_backend) }}
+                    {{ render_field(form.application.form.fetch_backend) }}
                    <span class="pure-form-message-inline">
                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
@@ -87,20 +87,20 @@
            <div class="tab-pane-inner" id="filters">

                    <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.ignore_whitespace) }}
+                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                    </span>
                    </fieldset>
                <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.render_anchor_tag_content) }}
+                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
                        <br/>
                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
                    </span>
                    </fieldset>
                    <fieldset class="pure-group">
-                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
+                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
@@ -112,7 +112,7 @@ nav
                      </span>
                    </fieldset>
                    <fieldset class="pure-group">
-                    {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                    ") }}
                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
--- a/changedetectionio/tests/conftest.py
+++ b/changedetectionio/tests/conftest.py
@@ -16,6 +16,7 @@ def cleanup(datastore_path):
    # Unlink test output files
    files = ['output.txt',
             'url-watches.json',
+             'secret.txt',
             'notification.txt',
             'count.txt',
             'endpoint-content.txt'
--- a/changedetectionio/tests/test_access_control.py
+++ b/changedetectionio/tests/test_access_control.py
@@ -1,5 +1,5 @@
 from flask import url_for
-
+from . util import live_server_setup

 def test_check_access_control(app, client):
    # Still doesnt work, but this is closer.
@@ -12,9 +12,9 @@ def test_check_access_control(app, client):
        # Enable password check.
        res = c.post(
            url_for("settings_page"),
-            data={"password": "foobar",
-                  "minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
+            data={"application-password": "foobar",
+                  "requests-minutes_between_check": 180,
+                  'application-fetch_backend': "html_requests"},
            follow_redirects=True
        )

@@ -49,74 +49,31 @@ def test_check_access_control(app, client):
        assert b"minutes_between_check" in res.data
        assert b"fetch_backend" in res.data

+        ##################################################
+        # Remove password button, and check that it worked
+        ##################################################
        res = c.post(
            url_for("settings_page"),
            data={
-                "minutes_between_check": 180,
-                "tag": "",
-                "headers": "",
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
+                "requests-minutes_between_check": 180,
+                "application-fetch_backend": "html_webdriver",
+                "application-removepassword_button": "Remove password"
            },
            follow_redirects=True,
        )
+        assert b"Password protection removed." in res.data
+        assert b"LOG OUT" not in res.data

-# There was a bug where saving the settings form would submit a blank password
-def test_check_access_control_no_blank_password(app, client):
-    # Still doesnt work, but this is closer.
-
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
-        assert b"Remove password" not in res.data
-
-        # Enable password check.
+        ############################################################
+        # Be sure a blank password doesnt setup password protection
+        ############################################################
        res = c.post(
            url_for("settings_page"),
-            data={"password": "",
-                  "minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
+            data={"application-password": "",
+                  "requests-minutes_between_check": 180,
+                  'application-fetch_backend': "html_requests"},
            follow_redirects=True
        )

-        assert b"Password protection enabled." not in res.data
-        assert b"Login" not in res.data
+        assert b"Password protection enabled" not in res.data

-
-# There was a bug where saving the settings form would submit a blank password
-def test_check_access_no_remote_access_to_remove_password(app, client):
-    # Still doesnt work, but this is closer.
-
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
-        assert b"Remove password" not in res.data
-
-        # Enable password check.
-        res = c.post(
-            url_for("settings_page"),
-            data={"password": "password",
-                  "minutes_between_check": 180,
-                  'fetch_backend': "html_requests"},
-            follow_redirects=True
-        )
-
-        assert b"Password protection enabled." in res.data
-        assert b"Login" in res.data
-
-        res = c.post(
-            url_for("settings_page"),
-            data={
-                "minutes_between_check": 180,
-                "tag": "",
-                "headers": "",
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
-            },
-            follow_redirects=True,
-        )
-        assert b"Password protection removed." not in res.data
-
-        res = c.get(url_for("index"),
-              follow_redirects=True)
-        assert b"watch-table-wrapper" not in res.data
--- a/changedetectionio/tests/test_backend.py
+++ b/changedetectionio/tests/test_backend.py
@@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
    # Enable auto pickup of <title> in settings
    res = client.post(
        url_for("settings_page"),
-        data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
+        data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"},
        follow_redirects=True
    )

--- a/changedetectionio/tests/test_ignore_text.py
+++ b/changedetectionio/tests/test_ignore_text.py
@@ -196,9 +196,9 @@ def test_check_global_ignore_text_functionality(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "global_ignore_text": ignore_text,
-            'fetch_backend': "html_requests"
+            "requests-minutes_between_check": 180,
+            "application-global_ignore_text": ignore_text,
+            'application-fetch_backend': "html_requests"
        },
        follow_redirects=True
    )
--- a/changedetectionio/tests/test_ignorehyperlinks.py
+++ b/changedetectionio/tests/test_ignorehyperlinks.py
@@ -55,9 +55,9 @@ def test_render_anchor_tag_content_true(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "render_anchor_tag_content": "true",
-            "fetch_backend": "html_requests",
+            "requests-minutes_between_check": 180,
+            "application-render_anchor_tag_content": "true",
+            "application-fetch_backend": "html_requests",
        },
        follow_redirects=True,
    )
@@ -116,9 +116,9 @@ def test_render_anchor_tag_content_false(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "render_anchor_tag_content": "false",
-            "fetch_backend": "html_requests",
+            "requests-minutes_between_check": 180,
+            "application-render_anchor_tag_content": "false",
+            "application-fetch_backend": "html_requests",
        },
        follow_redirects=True,
    )
@@ -175,8 +175,8 @@ def test_render_anchor_tag_content_default(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "fetch_backend": "html_requests",
+            "requests-minutes_between_check": 180,
+            "application-fetch_backend": "html_requests",
        },
        follow_redirects=True,
    )
--- a/changedetectionio/tests/test_ignorestatuscode.py
+++ b/changedetectionio/tests/test_ignorestatuscode.py
@@ -51,9 +51,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "ignore_status_codes": "y",
-            'fetch_backend': "html_requests"
+            "requests-minutes_between_check": 180,
+            "application-ignore_status_codes": "y",
+            'application-fetch_backend': "html_requests"
        },
        follow_redirects=True
    )
--- a/changedetectionio/tests/test_ignorewhitespace.py
+++ b/changedetectionio/tests/test_ignorewhitespace.py
@@ -61,9 +61,9 @@ def test_check_ignore_whitespace(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-            "minutes_between_check": 180,
-            "ignore_whitespace": "y",
-            'fetch_backend': "html_requests"
+            "requests-minutes_between_check": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests"
        },
        follow_redirects=True
    )
--- a/changedetectionio/tests/test_notification.py
+++ b/changedetectionio/tests/test_notification.py
@@ -195,11 +195,11 @@ def test_notification_validation(client, live_server):
    # Now adding a wrong token should give us an error
    res = client.post(
        url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_body": "Rubbish: {rubbish}\n",
-              "notification_format": "Text",
-              "notification_urls": "json://localhost/foobar",
-              "time_between_check": {'seconds': 180},
+        data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "application-notification_body": "Rubbish: {rubbish}\n",
+              "application-notification_format": "Text",
+              "application-notification_urls": "json://localhost/foobar",
+              "requests-minutes_between_check": 180,
              "fetch_backend": "html_requests"
              },
        follow_redirects=True
--- a/changedetectionio/tests/test_request.py
+++ b/changedetectionio/tests/test_request.py
@@ -84,9 +84,25 @@ def test_body_in_request(client, live_server):
    )
    assert b"1 Imported" in res.data

-    body_value = 'Test Body Value'
+    time.sleep(3)

-    # Add a properly formatted body with a proper method
+    # add the first 'version'
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "POST",
+              "fetch_backend": "html_requests",
+              "body": "something something"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    time.sleep(3)
+
+    # Now the change which should trigger a change
+    body_value = 'Test Body Value'
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={
--- a/changedetectionio/tests/test_watch_fields_storage.py
+++ b/changedetectionio/tests/test_watch_fields_storage.py
@@ -56,8 +56,8 @@ def test_check_recheck_global_setting(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-               "minutes_between_check": 1566,
-               'fetch_backend': "html_requests"
+               "requests-minutes_between_check": 1566,
+               'application-fetch_backend': "html_requests"
               },
        follow_redirects=True
    )
@@ -88,8 +88,8 @@ def test_check_recheck_global_setting(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-               "minutes_between_check": 222,
-                'fetch_backend': "html_requests"
+               "requests-minutes_between_check": 222,
+                'application-fetch_backend': "html_requests"
               },
        follow_redirects=True
    )
@@ -124,8 +124,8 @@ def test_check_recheck_global_setting(client, live_server):
    res = client.post(
        url_for("settings_page"),
        data={
-               "minutes_between_check": 666,
-                'fetch_backend': "html_requests"
+               "requests-minutes_between_check": 666,
+                'application-fetch_backend': "html_requests"
               },
        follow_redirects=True
    )
--- a/changedetectionio/tests/util.py
+++ b/changedetectionio/tests/util.py
@@ -85,6 +85,7 @@ def live_server_setup(live_server):
    # Just return the body in the request
    @live_server.app.route('/test-body', methods=['POST', 'GET'])
    def test_body():
+        print ("TEST-BODY GOT", request.data, "returning")
        return request.data

    # Just return the verb in the request