0.33

* Re #81 - Regex support
* minor cleanup

Dockerfile

@@ -1,29 +1,56 @@
-FROM python:3.8-slim
+# pip dependencies install stage
+FROM python:3.8-slim as builder
 
 # rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
-COPY requirements.txt /tmp/requirements.txt
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+    
+RUN mkdir /install
+WORKDIR /install
 
-RUN apt-get update && apt-get install -y libssl-dev libffi-dev gcc libc-dev libxslt-dev zlib1g-dev rustc g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
-# Update pip, install requirements, remove rust and dev packages that are no longer needed.
-RUN pip3 install --upgrade pip && pip3 install --no-cache-dir -r /tmp/requirements.txt && apt-get remove rustc *-dev --purge -y
+COPY requirements.txt /requirements.txt
+
+RUN pip install --target=/dependencies -r /requirements.txt
+
+# Final image stage
+FROM python:3.8-slim
+
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+# Re #93, #73, excluding rustc (adds another 430Mb~)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
+ENV PYTHONUNBUFFERED=1
 
-RUN [ ! -d "/app" ] && mkdir /app
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
+# Copy modules over to the final image and add their dir to PYTHONPATH
+COPY --from=builder /dependencies /usr/local
+ENV PYTHONPATH=/usr/local
+
 # The actual flask app
 COPY backend /app/backend
-
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
 
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
 CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
-
-
-

README.md

@@ -103,6 +103,10 @@ This proxy support also extends to the notifications https://github.com/caronc/a
 - Wont work with Cloudfare type "Please turn on javascript" protected pages
 - You can use the 'headers' section to monitor password protected web page changes
 
+### RaspberriPi support?
+
+RaspberriPi and linux/arm/v6 linux/arm/v7 devices are supported! 
+
 
 ### Support us
 

backend/__init__.py

@@ -6,7 +6,6 @@
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
-# @todo make tables responsive!
 # @todo fetch title into json
 # https://distill.io/features
 # proxy per check
@@ -23,7 +22,7 @@ from threading import Event
 
 import queue
 
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for
+from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for, flash
 
 from feedgen.feed import FeedGenerator
 from flask import make_response
@@ -36,7 +35,6 @@ datastore = None
 running_update_threads = []
 ticker_thread = None
 
-messages = []
 extra_stylesheets = []
 
 update_q = queue.Queue()
@@ -54,10 +52,40 @@ app.config['NEW_VERSION_AVAILABLE'] = False
 
 app.config['LOGIN_DISABLED'] = False
 
+#app.config["EXPLAIN_TEMPLATE_LOADING"] = True
+
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
 
+def init_app_secret(datastore_path):
+    secret = ""
+
+    path = "{}/secret.txt".format(datastore_path)
+
+    try:
+        with open(path, "r") as f:
+            secret = f.read()
+
+    except FileNotFoundError:
+        import secrets
+        with open(path, "w") as f:
+            secret = secrets.token_hex(32)
+            f.write(secret)
+
+    return secret
+
+# Remember python is by reference
+# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
+def populate_form_from_watch(form, watch):
+    for i in form.__dict__.keys():
+        if i[0] != '_':
+            p = getattr(form, i)
+            if hasattr(p, 'data') and i in watch:
+                if not p.data:
+                    setattr(p, "data", watch[i])
+
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -125,7 +153,7 @@ class User(flask_login.UserMixin):
 
     pass
 
-def changedetection_app(conig=None, datastore_o=None):
+def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
@@ -134,7 +162,7 @@ def changedetection_app(conig=None, datastore_o=None):
 
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
-
+    app.secret_key = init_app_secret(config['datastore_path'])
 
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
@@ -161,12 +189,8 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route('/login', methods=['GET', 'POST'])
     def login():
 
-        global messages
-
         if request.method == 'GET':
-            output = render_template("login.html", messages=messages)
-            # Show messages but once.
-            messages = []
+            output = render_template("login.html")
             return output
 
         user = User()
@@ -182,7 +206,7 @@ def changedetection_app(conig=None, datastore_o=None):
             return redirect(next or url_for('index'))
 
         else:
-            messages.append({'class': 'error', 'message': 'Incorrect password'})
+            flash('Incorrect password', 'error')
 
         return redirect(url_for('login'))
 
@@ -194,7 +218,6 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_required
     def index():
-        global messages
         limit_tag = request.args.get('tag')
 
         pause_uuid = request.args.get('pause')
@@ -254,42 +277,57 @@ def changedetection_app(conig=None, datastore_o=None):
         else:
             output = render_template("watch-overview.html",
                                      watches=sorted_watches,
-                                     messages=messages,
                                      tags=existing_tags,
                                      active_tag=limit_tag,
                                      has_unviewed=datastore.data['has_unviewed'])
 
-            # Show messages but once.
-            messages = []
-
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
     @login_required
     def scrub_page():
-        from pathlib import Path
 
-        global messages
+        import re
 
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
-            limit_timestamp = int(request.form.get('limit_date'))
+            limit_date = request.form.get('limit_date')
+
+            try:
+                limit_date = limit_date.replace('T', ' ')
+                # I noticed chrome will show '/' but actually submit '-'
+                limit_date = limit_date.replace('-', '/')
+                # In the case that :ss seconds are supplied
+                limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+
+                str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                limit_timestamp = int(str_to_dt.timestamp())
+
+                if limit_timestamp > time.time():
+                    flash("Timestamp is in the future, cannot continue.", 'error')
+                    return redirect(url_for('scrub_page'))
+
+            except ValueError:
+                flash('Incorrect date format, cannot continue.', 'error')
+                return redirect(url_for('scrub_page'))
 
             if confirmtext == 'scrub':
-
+                changes_removed = 0
                 for uuid, watch in datastore.data['watching'].items():
-                    if len(str(limit_timestamp)) == 10:
-                        datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
+                    if limit_timestamp:
+                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
                     else:
-                        datastore.scrub_watch(uuid)
+                        changes_removed += datastore.scrub_watch(uuid)
 
-                messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
+                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
             else:
-                messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
+                flash('Incorrect confirmation text.', 'error')
 
             return redirect(url_for('index'))
 
-        return render_template("scrub.html")
+        output =  render_template("scrub.html")
+        return output
+
 
     # If they edited an existing watch, we need to know to reset the current/previous md5 to include
     # the excluded text.
@@ -319,204 +357,144 @@ def changedetection_app(conig=None, datastore_o=None):
 
         return datastore.data['watching'][uuid]['previous_md5']
 
+
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_required
     def edit_page(uuid):
-        global messages
-        import validators
+        from backend import forms
+        form = forms.watchForm(request.form)
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        if request.method == 'POST':
+        if request.method == 'GET':
+            if not uuid in datastore.data['watching']:
+                flash("No watch with the UUID %s found." % (uuid), "error")
+                return redirect(url_for('index'))
 
-            url = request.form.get('url').strip()
-            tag = request.form.get('tag').strip()
+            populate_form_from_watch(form, datastore.data['watching'][uuid])
 
-            minutes_recheck = request.form.get('minutes')
-            if minutes_recheck:
-                minutes = int(minutes_recheck.strip())
-                if minutes >= 1:
-                    datastore.data['watching'][uuid]['minutes_between_check'] = minutes
-                else:
-                    messages.append(
-                        {'class': 'error', 'message': "Must be atleast 1 minute."})
-
-
-
-            # Extra headers
-            form_headers = request.form.get('headers').strip().split("\n")
-            extra_headers = {}
-            if form_headers:
-                for header in form_headers:
-                    if len(header):
-                        parts = header.split(':', 1)
-                        if len(parts) == 2:
-                            extra_headers.update({parts[0].strip(): parts[1].strip()})
-
-            update_obj = {'url': url,
-                          'tag': tag,
-                          'headers': extra_headers
+        if request.method == 'POST' and form.validate():
+            update_obj = {'url': form.url.data.strip(),
+                          'tag': form.tag.data.strip(),
+                          'headers': form.headers.data
                           }
 
             # Notification URLs
-            form_notification_text = request.form.get('notification_urls')
-            notification_urls = []
-            if form_notification_text:
-                for text in form_notification_text.strip().split("\n"):
-                    text = text.strip()
-                    if len(text):
-                        notification_urls.append(text)
-
-            datastore.data['watching'][uuid]['notification_urls'] = notification_urls
+            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
 
             # Ignore text
-            form_ignore_text = request.form.get('ignore-text')
-            ignore_text = []
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form_ignore_text:
-                for text in form_ignore_text.strip().split("\n"):
-                    text = text.strip()
-                    if len(text):
-                        ignore_text.append(text)
-
-                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
-
-                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
                 if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
 
-            # CSS Filter
-            css_filter = request.form.get('css_filter')
-            if css_filter:
-                datastore.data['watching'][uuid]['css_filter'] = css_filter.strip()
+            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
 
-                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                 if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
 
-            validators.url(url)  # @todo switch to prop/attr/observer
             datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
+            flash("Updated watch.")
 
-            messages.append({'class': 'ok', 'message': 'Updated watch.'})
+            # Queue the watch for immediate recheck
+            update_q.put(uuid)
 
-            trigger_n = request.form.get('trigger-test-notification')
-            if trigger_n:
-                n_object = {'watch_url': url,
-                            'notification_urls': notification_urls}
+            if form.trigger_check.data:
+                n_object = {'watch_url': form.url.data.strip(),
+                            'notification_urls': form.notification_urls.data}
                 notification_q.put(n_object)
 
-                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
+                flash('Notifications queued.')
 
             return redirect(url_for('index'))
 
         else:
-            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form)
 
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
     @login_required
     def settings_page():
-        global messages
+
+        from backend import forms
+        form = forms.globalSettingsForm(request.form)
 
         if request.method == 'GET':
-            if request.values.get('notification-test'):
-                url_count = len(datastore.data['settings']['application']['notification_urls'])
-                if url_count:
-                    import apprise
-                    apobj = apprise.Apprise()
-                    apobj.debug = True
+            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'] / 60)
+            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
 
-                    # Add each notification
-                    for n in datastore.data['settings']['application']['notification_urls']:
-                        apobj.add(n)
-                    outcome = apobj.notify(
-                        body='Hello from the worlds best and simplest web page change detection and monitoring service!',
-                        title='Changedetection.io Notification Test',
-                    )
-
-                    if outcome:
-                        messages.append(
-                            {'class': 'notice', 'message': "{} Notification URLs reached.".format(url_count)})
-                    else:
-                        messages.append(
-                            {'class': 'error', 'message': "One or more Notification URLs failed"})
-
-                return redirect(url_for('settings_page'))
-
-            if request.values.get('removepassword'):
+            # Password unset is a GET
+            if request.values.get('removepassword') == 'true':
                 from pathlib import Path
-
                 datastore.data['settings']['application']['password'] = False
-                messages.append({'class': 'notice', 'message': "Password protection removed."})
+                flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
 
-                return redirect(url_for('settings_page'))
+        if request.method == 'POST' and form.validate():
 
-        if request.method == 'POST':
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data * 60
 
-            password = request.values.get('password')
-            if password:
-                import hashlib
-                import base64
-                import secrets
+            if len(form.notification_urls.data):
+                import apprise
+                apobj = apprise.Apprise()
+                apobj.debug = True
 
-                # Make a new salt on every new password and store it with the password
-                salt = secrets.token_bytes(32)
+                # Add each notification
+                for n in datastore.data['settings']['application']['notification_urls']:
+                    apobj.add(n)
+                outcome = apobj.notify(
+                    body='Hello from the worlds best and simplest web page change detection and monitoring service!',
+                    title='Changedetection.io Notification Test',
+                )
 
-                key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
-                store = base64.b64encode(salt + key).decode('ascii')
-                datastore.data['settings']['application']['password'] = store
-                messages.append({'class': 'notice', 'message': "Password protection enabled."})
+                if outcome:
+                    flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
+                else:
+                    flash("One or more Notification URLs failed", 'error')
+
+
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.needs_write = True
+
+            if form.trigger_check.data:
+                n_object = {'watch_url': "Test from changedetection.io!",
+                            'notification_urls': form.notification_urls.data}
+                notification_q.put(n_object)
+                flash('Notifications queued.')
+
+            if form.password.encrypted_password:
+                datastore.data['settings']['application']['password'] = form.password.encrypted_password
+                flash("Password protection enabled.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('index'))
 
-            try:
-                minutes = int(request.values.get('minutes').strip())
-            except ValueError:
-                messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
+            flash("Settings updated.")
 
-            else:
-                if minutes >= 1:
-                    datastore.data['settings']['requests']['minutes_between_check'] = minutes
-                    datastore.needs_write = True
-                else:
-                    messages.append(
-                        {'class': 'error', 'message': "Must be atleast 1 minute."})
-
-            # 'validators' package doesnt work because its often a non-stanadard protocol. :(
-            datastore.data['settings']['application']['notification_urls'] = []
-            trigger_n = request.form.get('trigger-test-notification')
-
-            for n in request.values.get('notification_urls').strip().split("\n"):
-                url = n.strip()
-                datastore.data['settings']['application']['notification_urls'].append(url)
-                datastore.needs_write = True
-
-            if trigger_n:
-                n_object = {'watch_url': "Test from changedetection.io!",
-                            'notification_urls': datastore.data['settings']['application']['notification_urls']}
-                notification_q.put(n_object)
-
-                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
-
-        output = render_template("settings.html", messages=messages,
-                                 minutes=datastore.data['settings']['requests']['minutes_between_check'],
-                                 notification_urls="\r\n".join(
-                                     datastore.data['settings']['application']['notification_urls']))
-        messages = []
+        if request.method == 'POST' and not form.validate():
+            flash("An error occurred, please see below.", "error")
 
+        output = render_template("settings.html", form=form)
         return output
 
     @app.route("/import", methods=['GET', "POST"])
     @login_required
     def import_page():
         import validators
-        global messages
         remaining_urls = []
 
         good = 0
@@ -534,7 +512,7 @@ def changedetection_app(conig=None, datastore_o=None):
                     if len(url):
                         remaining_urls.append(url)
 
-            messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
+            flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
 
             if len(remaining_urls) == 0:
                 # Looking good, redirect to index.
@@ -542,11 +520,8 @@ def changedetection_app(conig=None, datastore_o=None):
 
         # Could be some remaining, or we could be on GET
         output = render_template("import.html",
-                                 messages=messages,
                                  remaining="\n".join(remaining_urls)
                                  )
-        messages = []
-
         return output
 
     # Clear all statuses, so we do not see the 'unviewed' class
@@ -558,23 +533,22 @@ def changedetection_app(conig=None, datastore_o=None):
         for watch_uuid, watch in datastore.data['watching'].items():
             datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
 
-        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
+        flash("Cleared all statuses.")
         return redirect(url_for('index'))
 
     @app.route("/diff/<string:uuid>", methods=['GET'])
     @login_required
     def diff_history_page(uuid):
-        global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/css/diff.css']
+        extra_stylesheets = ['/static/styles/diff.css']
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
         dates = list(watch['history'].keys())
@@ -584,8 +558,7 @@ def changedetection_app(conig=None, datastore_o=None):
         dates = [str(i) for i in dates]
 
         if len(dates) < 2:
-            messages.append(
-                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
+            flash("Not enough saved change detection snapshots to produce a report.", "error")
             return redirect(url_for('index'))
 
         # Save the current newest history as the most recently viewed
@@ -607,7 +580,6 @@ def changedetection_app(conig=None, datastore_o=None):
             previous_version_file_contents = f.read()
 
         output = render_template("diff.html", watch_a=watch,
-                                 messages=messages,
                                  newest=newest_version_file_contents,
                                  previous=previous_version_file_contents,
                                  extra_stylesheets=extra_stylesheets,
@@ -621,18 +593,17 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route("/preview/<string:uuid>", methods=['GET'])
     @login_required
     def preview_page(uuid):
-        global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/css/diff.css']
+        extra_stylesheets = ['/static/styles/diff.css']
 
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
         print(watch)
@@ -717,11 +688,10 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route("/api/add", methods=['POST'])
     @login_required
     def api_watch_add():
-        global messages
 
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
-            messages.append({'class': 'error', 'message': 'The URL {} already exists'.format(url)})
+            flash('The URL {} already exists'.format(url), "error")
             return redirect(url_for('index'))
 
         # @todo add_watch should throw a custom Exception for validation etc
@@ -729,16 +699,16 @@ def changedetection_app(conig=None, datastore_o=None):
         # Straight into the queue.
         update_q.put(new_uuid)
 
-        messages.append({'class': 'ok', 'message': 'Watch added.'})
+        flash("Watch added.")
         return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
-        global messages
+
         uuid = request.args.get('uuid')
         datastore.delete(uuid)
-        messages.append({'class': 'ok', 'message': 'Deleted.'})
+        flash('Deleted.')
 
         return redirect(url_for('index'))
 
@@ -746,8 +716,6 @@ def changedetection_app(conig=None, datastore_o=None):
     @login_required
     def api_watch_checknow():
 
-        global messages
-
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
         i = 0
@@ -778,8 +746,7 @@ def changedetection_app(conig=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(watch_uuid)
                     i += 1
-
-        messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
+        flash("{} watches are rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
     # @todo handle ctrl break
@@ -866,28 +833,31 @@ def ticker_thread_check_time_launch_checks():
 
     while not app.config.exit.is_set():
 
+        # Get a list of watches by UUID that are currently fetching data
         running_uuids = []
         for t in running_update_threads:
             if t.current_uuid:
                 running_uuids.append(t.current_uuid)
 
-        # Look at the dataset, find a stale watch to process
-
-        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
-
-        threshold = time.time() - (minutes * 60)
+        # Check for watches outside of the time threshold to put in the thread queue.
         for uuid, watch in datastore.data['watching'].items():
 
-            # If they supplied an individual entry minutes to recheck and its not the same as the default
-            if 'minutes_between_check' in watch and minutes != watch['minutes_between_check']:
-                threshold = time.time() - (watch['minutes_between_check'] * 60)
+            # If they supplied an individual entry minutes to threshold.
+            if 'minutes_between_check' in watch:
+                max_time = watch['minutes_between_check'] * 60
+            else:
+                # Default system wide.
+                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
 
+            threshold = time.time() - max_time
+
+            # Yeah, put it in the queue, it's more than time.
             if not watch['paused'] and watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
-            time.sleep(0.1)
+        # Wait a few seconds before checking the list again
+        time.sleep(3)
 
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

backend/fetch_site_status.py

@@ -13,18 +13,34 @@ class perform_site_check():
         self.datastore = datastore
 
     def strip_ignore_text(self, content, list_ignore_text):
+        import re
         ignore = []
+        ignore_regex = []
         for k in list_ignore_text:
-            ignore.append(k.encode('utf8'))
+
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
 
         output = []
         for line in content.splitlines():
-            line = line.encode('utf8')
 
             # Always ignore blank lines in this mode. (when this function gets called)
             if len(line.strip()):
-                if not any(skip_text in line for skip_text in ignore):
-                    output.append(line)
+                regex_matches = False
+
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
+
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
 
         return "\n".encode('utf8').join(output)
 
@@ -73,7 +89,7 @@ class perform_site_check():
                 soup = BeautifulSoup(r.content, "html.parser")
                 stripped_text_from_html = ""
                 for item in soup.select(css_filter):
-                    text = str(item.get_text())+"\n"
+                    text = str(item.get_text()).strip() + '\n'
                     stripped_text_from_html += text
 
             else:

backend/forms.py

@@ -0,0 +1,130 @@
+from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+    Field
+from wtforms import widgets
+from wtforms.validators import ValidationError
+from wtforms.fields import html5
+
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            return "\r\n".join(self.data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            self.data = [x.strip() for x in cleaned]
+            p = 1
+        else:
+            self.data = []
+
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import hashlib
+        import base64
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            self.encrypted_password = self.build_password(valuelist[0])
+            self.data = []
+        else:
+            self.data = []
+
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':')
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import re
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+
+class watchForm(Form):
+    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
+    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
+
+    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
+    css_filter = StringField('CSS Filter')
+
+    ignore_text = StringListField('Ignore Text', [ListRegex()])
+    notification_urls = StringListField('Notification URL List')
+    headers = StringDictKeyValue('Request Headers')
+    trigger_check = BooleanField('Send test notification on save')
+
+
+class globalSettingsForm(Form):
+
+    password = SaltyPasswordField()
+    remove_password = BooleanField('Remove password')
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.NumberRange(min=1)])
+
+    notification_urls = StringListField('Notification URL List')
+    trigger_check = BooleanField('Send test notification on save')

backend/static/css/styles.css

@@ -1,277 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- */
-body {
-  color: #333;
-  background: #262626;
-}
-
-.pure-table-even {
-  background: #fff;
-}
-/* Some styles from https://css-tricks.com/ */
-
-a {
-  text-decoration: none;
-  color: #1b98f8;
-}
-a.github-link {
-  color: #fff;
-}
-
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-
-  align-items: center;
-}
-
-section.content {
-
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.pure-table.watch-table td {
-  font-size: 80%;
-}
-
-/* table related */
-.watch-table {
-  width: 100%;
-
-}
-
-.watch-table tr.unviewed {
-    font-weight: bold;
-}
-
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap;
-}
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center;
-}
-
-.watch-table .error {
-  color: #a00;
-}
-
-.watch-table td {
-  white-space: nowrap;
-}
-
-.watch-table td.title-col {
-  word-break: break-all;
-  white-space: normal;
-}
-.watch-table th {
-  white-space: nowrap;
-}
-
-.watch-table .title-col a[target="_blank"]::after, .current-diff-url::after {
-  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-  margin: 0 3px 0 5px;
-}
-
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px;
-}
-#post-list-buttons li {
-  display: inline-block;
-}
-
-#post-list-buttons a {
-   border-top-left-radius: initial;
-   border-top-right-radius: initial;
-   border-bottom-left-radius: 5px;
-   border-bottom-right-radius: 5px;
-}
-
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
-}
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1;
-}
-body::after {
-  opacity: 0.91;
-}
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-body:before {
-  background-size: cover
-}
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
-}
-
-
-.button-small {
-  font-size: 85%;
-}
-
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block;
-}
-
-.edit-form {
-   background: #fff;
-   padding: 2em;
-   margin: 1em;
-   border-radius: 5px;
-}
-.button-secondary {
-    color: white;
-    border-radius: 4px;
-    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
-}
-
-.button-success {
-    background: rgb(28, 184, 65);
-    /* this is a green */
-}
-
-.button-tag {
-    background: rgb(99, 99, 99);
-    color: #fff;
-    font-size: 65%;
-    border-bottom-left-radius: initial;
-    border-bottom-right-radius: initial;
-}
-.button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold;
-}
-.button-error {
-    background: rgb(202, 60, 60);
-    /* this is a maroon */
-}
-
-.button-warning {
-    background: rgb(223, 117, 20);
-    /* this is an orange */
-}
-
-.button-secondary {
-    background: rgb(66, 184, 221);
-    /* this is a light blue */
-}
-
-
-.button-cancel {
-    background: rgb(200, 200, 200);
-    /* this is a green */
-}
-
-.messages {
-    padding: 1em;
-    background: rgba(255,255,255,.2);
-    border-radius: 10px;
-    color: #fff;
-    font-weight: bold;
-}
-
-.pure-form label  {
-  font-weight: bold;
-}
-
-#new-watch-form {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-}
-#new-watch-form legend {
-    color: #fff;
-}
-
-#diff-col {
-    padding-left:40px;
-}
-#diff-jump {
-    position: fixed;
-    left: 0px;
-    top: 80px;
-    background: #fff;
-    padding: 10px;
-    border-top-right-radius: 5px;
-    border-bottom-right-radius: 5px;
-    box-shadow: 5px 0 5px -2px #888;
-}
-
-#diff-jump a {
-    color: #1b98f8;
-    cursor: grabbing;
-    -moz-user-select: none;
-    -webkit-user-select: none;
-    -ms-user-select:none;
-    user-select:none;
-    -o-user-select:none;
-}
-
-footer {
-    padding: 10px;
-    background: #fff;
-    color: #444;
-    text-align: center;
-}
-
-#feed-icon {
-    vertical-align: middle;
-}
-
-#version {
-    position: absolute;
-    top: 80px;
-    right: 0px;
-    font-size: 8px;
-    background: #fff;
-    padding: 10px;
-}
-
-#new-version-text a{
-    color: #e07171;
-}
-
-.paused-state.state-False img {
-    opacity: 0.2;
-}
-
-
-.paused-state.state-False:hover img{
-    opacity: 0.8;
-}

backend/static/styles/.gitignore

@@ -0,0 +1 @@
+node_modules

backend/static/styles/diff.css

@@ -0,0 +1,55 @@
+#diff-ui {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  font-size: 9px; }
+  #diff-ui table {
+    table-layout: fixed;
+    width: 100%; }
+  #diff-ui td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left;
+    white-space: pre-wrap; }
+
+h1 {
+  display: inline;
+  font-size: 100%; }
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7; }
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none; }
+
+#result {
+  white-space: pre-wrap; }
+
+#settings {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%; }
+  #settings label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal; }
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em; }
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */ } }

backend/static/styles/diff.scss

@@ -1,15 +1,24 @@
-table {
-	table-layout: fixed;
-	width: 100%;
-}
-td {
-	width: 33%;
-	padding: 3px 4px;
-	border: 1px solid transparent;
-	vertical-align: top;
-	font: 1em monospace;
-	text-align: left;
-	white-space: pre-wrap;
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin: 1em;
+    border-radius: 5px;
+    font-size: 9px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+        white-space: pre-wrap;
+    }
+
 }
 h1 {
 	display: inline;
@@ -33,16 +42,16 @@ ins {
 
 #settings {
     background: rgba(0,0,0,.05);
-        padding: 1em;
+    padding: 1em;
     border-radius: 10px;
     margin-bottom: 1em;
     color: #fff;
     font-size: 80%;
-}
-#settings label {
-	margin-left: 1em;
-	display: inline-block;
-	font-weight: normal;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
 }
 
 .source {
@@ -55,12 +64,4 @@ ins {
 	body {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
-}
-
-#diff-ui {
-    background: #fff;
-    padding: 2em;
-    margin: 1em;
-    border-radius: 5px;
-    font-size: 9px;
 }

backend/static/styles/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "changedetection.io-theme",
+  "version": "0.0.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "scss": "node-sass --watch styles.scss diff.scss -o ."
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-sass": "^6.0.0"
+  }
+}

backend/static/styles/styles.css

@@ -0,0 +1,313 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626; }
+
+.pure-table-even {
+  background: #fff; }
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8; }
+
+a.github-link {
+  color: #fff; }
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center; }
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center; }
+
+/* table related */
+.watch-table {
+  width: 100%; }
+  .watch-table tr.unviewed {
+    font-weight: bold; }
+  .watch-table .error {
+    color: #a00; }
+  .watch-table td {
+    font-size: 80%;
+    white-space: nowrap; }
+  .watch-table td.title-col {
+    word-break: break-all;
+    white-space: normal; }
+  .watch-table th {
+    white-space: nowrap; }
+  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px; }
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap; }
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center; }
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px; }
+  #post-list-buttons li {
+    display: inline-block; }
+  #post-list-buttons a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px; }
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1; }
+
+body::after {
+  opacity: 0.91; }
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png); }
+
+body:before {
+  background-size: cover; }
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
+
+.button-small {
+  font-size: 85%; }
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block; }
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px; }
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
+
+.button-success {
+  background: #1cb841;
+  /* this is a green */ }
+
+.button-tag {
+  background: #636363;
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial; }
+  .button-tag.active {
+    background: #9c9c9c;
+    font-weight: bold; }
+
+.button-error {
+  background: #ca3c3c;
+  /* this is a maroon */ }
+
+.button-warning {
+  background: #df7514;
+  /* this is an orange */ }
+
+.button-secondary {
+  background: #42b8dd;
+  /* this is a light blue */ }
+
+.button-cancel {
+  background: #c8c8c8;
+  /* this is a green */ }
+
+.messages li {
+  list-style: none;
+  padding: 1em;
+  border-radius: 10px;
+  color: #fff;
+  font-weight: bold; }
+  .messages li.message {
+    background: rgba(255, 255, 255, 0.2); }
+  .messages li.error {
+    background: rgba(255, 1, 1, 0.5); }
+  .messages li.notice {
+    background: rgba(255, 255, 255, 0.5); }
+
+#new-watch-form {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em; }
+
+#new-watch-form legend {
+  color: #fff; }
+
+#new-watch-form input {
+  width: auto !important; }
+
+#diff-col {
+  padding-left: 40px; }
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 80px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888; }
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none; }
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center; }
+
+#feed-icon {
+  vertical-align: middle; }
+
+#version {
+  position: absolute;
+  top: 80px;
+  right: 0px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px; }
+
+#new-version-text a {
+  color: #e07171; }
+
+.paused-state.state-False img {
+  opacity: 0.2; }
+
+.paused-state.state-False:hover img {
+  opacity: 0.8; }
+
+.monospaced-textarea textarea {
+  width: 100%;
+  font-family: monospace;
+  white-space: pre;
+  overflow-wrap: normal;
+  overflow-x: scroll; }
+
+.pure-form {
+  /* The input fields with errors */
+  /* The list of errors */ }
+  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
+    padding-bottom: 1em; }
+    .pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
+      margin: 0px; }
+  .pure-form .error input {
+    background-color: #ffebeb; }
+  .pure-form ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box; }
+    .pure-form ul.errors li {
+      margin-left: 1em;
+      color: #dd0000; }
+  .pure-form label {
+    font-weight: bold; }
+  .pure-form input[type=url] {
+    width: 100%; }
+  .pure-form textarea {
+    width: 100%;
+    font-size: 14px; }
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%; }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0.5em; }
+  #nav-menu {
+    overflow-x: scroll; } }
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    /* Force table to not be like tables anymore */
+    /* Hide table headers (but not display: none;, for accessibility) */ }
+    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
+      display: block; }
+    .watch-table .last-checked::before {
+      color: #555;
+      content: "Last Checked "; }
+    .watch-table .last-changed::before {
+      color: #555;
+      content: "Last Changed "; }
+    .watch-table td.inline {
+      display: inline-block; }
+    .watch-table thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px; }
+    .watch-table .pure-table td, .watch-table .pure-table th {
+      border: none; }
+    .watch-table td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee; }
+      .watch-table td:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap; }
+    .watch-table.pure-table-striped tr {
+      background-color: #fff; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) {
+      background-color: #eee; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
+      background-color: inherit; } }

backend/static/styles/styles.scss

@@ -0,0 +1,439 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626;
+}
+
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    font-size: 80%;
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+}
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png);
+}
+
+body:before {
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+
+.button-small {
+  font-size: 85%;
+}
+
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+}
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+}
+
+#new-watch-form legend {
+  color: #fff;
+}
+
+#new-watch-form input {
+  width: auto !important;
+}
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 80px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+}
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none;
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#version {
+  position: absolute;
+  top: 80px;
+  right: 0px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px;
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.paused-state {
+  &.state-False img {
+    opacity: 0.2;
+  }
+
+  &.state-False:hover img {
+    opacity: 0.8;
+  }
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        dd {
+            margin: 0px;
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+   /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  input[type=url] {
+    width: 100%;
+  }
+
+  textarea {
+    width: 100%;
+    font-size: 14px;
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0.5em;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+
+}
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px),
+(min-device-width: 768px) and (max-device-width: 1024px) {
+
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+

backend/store.py

@@ -113,7 +113,7 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
-        self.__data['version_tag'] = "0.30"
+        self.__data['version_tag'] = "0.33"
 
         if not 'app_guid' in self.__data:
             import sys
@@ -241,18 +241,20 @@ class ChangeDetectionStore:
         import hashlib
         del_timestamps = []
 
+        changes_removed = 0
+
         for timestamp, path in self.data['watching'][uuid]['history'].items():
             if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
                 self.unlink_history_file(path)
                 del_timestamps.append(timestamp)
-
-
+                changes_removed += 1
 
                 if not limit_timestamp:
                     self.data['watching'][uuid]['last_checked'] = 0
                     self.data['watching'][uuid]['last_changed'] = 0
                     self.data['watching'][uuid]['previous_md5'] = 0
 
+
         for timestamp in del_timestamps:
             del self.data['watching'][uuid]['history'][str(timestamp)]
 
@@ -272,9 +274,8 @@ class ChangeDetectionStore:
                         self.data['watching'][uuid]['previous_md5'] = False
                         pass
 
-
         self.needs_write = True
-
+        return changes_removed
 
     def add_watch(self, url, tag):
         with self.lock:
@@ -346,3 +347,19 @@ class ChangeDetectionStore:
                 self.sync_to_json()
             time.sleep(3)
 
+    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+    # This usually is not used, but can be handy.
+    def remove_unused_snapshots(self):
+        print ("Removing snapshots from datastore that are not in the index..")
+
+        index=[]
+        for uuid in self.data['watching']:
+            for id in self.data['watching'][uuid]['history']:
+                index.append(self.data['watching'][uuid]['history'][str(id)])
+
+        import pathlib
+        # Only in the sub-directories
+        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
+            if not str(item) in index:
+                print ("Removing",item)
+                os.unlink(item)

backend/templates/_helpers.jinja

@@ -0,0 +1,12 @@
+{% macro render_field(field) %}
+  <dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
+  <dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </dd>
+{% endmacro %}

backend/templates/base.html

@@ -5,8 +5,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
     <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/css/styles.css?ver=1000">
+    <link rel="stylesheet" href="/static/styles/pure-min.css">
+    <link rel="stylesheet" href="/static/styles/styles.css?ver=1000">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
         <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -16,7 +16,8 @@
 <body>
 
 <div class="header">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
             <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
         {% else %}
@@ -67,15 +68,15 @@
         {% block header %}{% endblock %}
     </header>
 
-    {% if messages %}
-    <div class="messages">
-        {% for message in messages %}
-        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
+    {% with messages = get_flashed_messages(with_categories=true) %}
+      {% if messages %}
+        <ul class=messages>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
-    </div>
-    {% endif %}
-
-
+        </ul>
+      {% endif %}
+    {% endwith %}
     {% block content %}
 
     {% endblock %}

backend/templates/diff.html

@@ -36,7 +36,6 @@
     <a onclick="next_diff();">Jump</a>
 </div>
 <div id="diff-ui">
-
     <table>
         <tbody>
         <tr>

backend/templates/edit.html

@@ -1,98 +1,64 @@
 {% extends 'base.html' %}
-
 {% block content %}
-<div class="edit-form">
-
-
+{% from '_helpers.jinja' import render_field %}
+<div class="edit-form monospaced-textarea">
     <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="url">URL</label>
-                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
-                       size="50"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                {{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
             </div>
             <div class="pure-control-group">
-                <label for="tag">Tag</label>
-                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
+                {{ render_field(form.tag, size=10) }}
             </div>
-            </br>
             <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" name="minutes" value="{{watch.minutes_between_check}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
+                {{ render_field(form.minutes_between_check, size=5) }}
             </div>
-            </br>
             <div class="pure-control-group">
-                <label for="minutes">CSS Filter</label>
-                <input type="text" id="css_filter" name="css_filter" value="{{watch.css_filter}}"
-                       size="25"/>
-                <span class="pure-form-message-inline">Limit text to this CSS rule, all matching CSS is included.</span>
+                {{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
+                <span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
+                    Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
+                    Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
+                </span>
             </div>
             <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
-                <label for="ignore-text">Ignore text</label>
-
-                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
-{% endfor %}</textarea>
-                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
+                {{ render_field(form.ignore_text, rows=5,  placeholder="Some text to ignore in a line
+/some.regex\d{2}/   for case-INsensitive regex
+") }}
+                <span class="pure-form-message-inline">
+                    Each line processed separately, any line matching will be ignored.<br/>
+                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                </span>
 
             </fieldset>
 
-            <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
-                <label for="headers">Extra request headers</label>
-
-                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
+                {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
-User-Agent: wonderbra 1.0"
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
-{% endfor %}</textarea>
-                <br/>
-
+User-Agent: wonderbra 1.0") }}
             </fieldset>
-            <div class="pure-control-group">
-                <label for="tag">Notification URLs</label>
-                <textarea id="notification_urls" name="notification_urls" class="pure-input-1-2" placeholder=""
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for value in watch.notification_urls %}{{ value }}
-{% endfor %}</textarea>
-                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</a> </span>
-                <br/>
-                <div class="pure-controls">
-                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
-                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
 
-                </div>
+            <div class="pure-control-group">
+                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+") }}
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+            </div>
+
+            <div class="pure-controls">
+                {{ render_field(form.trigger_check, rows=5) }}
             </div>
-            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
-
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Cancel</a>
                 <a href="/api/delete?uuid={{uuid}}"
                    class="pure-button button-small button-error ">Delete</a>
-
             </div>
-
-
         </fieldset>
     </form>
 

backend/templates/scrub.html

@@ -2,34 +2,25 @@
 
 {% block content %}
 <div class="edit-form">
-
-
     <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
                 You may like to use the <strong>BACKUP</strong> link first.<br/>
-
-                Type in the word <strong>scrub</strong> to confirm that you understand!
-                <br/>
-
             </div>
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Confirm text</label><br/>
+                <label for="confirmtext">Confirmation text</label>
                 <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
             </div>
-
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Limit delete history including and after date</label><br/>
-                <input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
-                <br/>
-
-
+                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
+                <input type="datetime-local" id="limit_date" name="limit_date"  />
+                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
             </div>
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>
@@ -37,12 +28,8 @@
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Cancel</a>
             </div>
-
-
         </fieldset>
     </form>
-
-
 </div>
 
 {% endblock %}

backend/templates/settings.html

@@ -1,42 +1,28 @@
 {% extends 'base.html' %}
 
 {% block content %}
+{% from '_helpers.jinja' import render_field %}
+
 <div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
+    <form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span><br/>
-                <span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
+                {{ render_field(form.minutes_between_check, size=5) }}
             </div>
-            <br/>
-            <hr>
             <div class="pure-control-group">
-                <label for="minutes">Password protection</label>
-                <input type="password" id="password" name="password" size="15"/>
                 {% if current_user.is_authenticated %}
                     <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
+                {% else %}
+                    {{ render_field(form.password, size=10) }}
                 {% endif %}
             </div>
-
-            <br/>
-            <hr>
-
             <div class="pure-control-group">
-                <label for="minutes">Global notification settings</label><br/>
-                Notification URLs <a href="https://github.com/caronc/apprise"> see Apprise examples</a>.
-                <textarea style="overflow-wrap: normal;  overflow-x: scroll;" id="notification_urls" name="notification_urls" cols="80"
-                          rows="6" wrap=off placeholder="Example:
-
-Gitter - gitter://token/room
+                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
 Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
 AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
 SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-">{{notification_urls}}</textarea>
+") }}
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
             </div>
                 <div class="pure-controls">
                     <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
@@ -52,7 +38,7 @@ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
 
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
+                <a href="/scrub" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
             </div>
 
 

backend/templates/watch-overview.html

@@ -44,9 +44,9 @@
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
+                <td class="title-col inline">{{watch.title if watch.title is not none else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
@@ -55,8 +55,8 @@
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
                 </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{% if watch.history|length >= 2 and watch.last_changed %}
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
                     Not yet

backend/tests/test_ignore_regex_text.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_regex_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    but sometimes we want to remove the lines.
+    
+    but 1 lines
+    but including 1234 lines
+    igNORe-cAse text we dont want to keep    
+    but not always."""
+
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"but 1 lines" in stripped_content
+    assert b"igNORe-cAse text" not in stripped_content
+    assert b"but 1234 lines" not in stripped_content
+

backend/tests/test_ignore_text.py

@@ -82,7 +82,7 @@ def set_modified_ignore_response():
 def test_check_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
     set_original_ignore_response()
 
     # Give the endpoint time to spin up
@@ -107,7 +107,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        data={"ignore_text": ignore_text, "url": test_url},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetection.py

@@ -12,35 +12,16 @@ import backend
 
 from backend import store
 
-
-def init_app_secret(datastore_path):
-    secret = ""
-
-    path = "{}/secret.txt".format(datastore_path)
-
-    try:
-        with open(path, "r") as f:
-            secret = f.read()
-
-    except FileNotFoundError:
-
-        import secrets
-        with open(path, "w") as f:
-            secret = secrets.token_hex(32)
-            f.write(secret)
-
-    return secret
-
-
 def main(argv):
     ssl_mode = False
     port = 5000
+    do_cleanup = False
 
     # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
     datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
-        opts, args = getopt.getopt(argv, "sd:p:", "purge")
+        opts, args = getopt.getopt(argv, "csd:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port] -d [datastore path]')
         sys.exit(2)
@@ -60,14 +41,22 @@ def main(argv):
         if opt == '-d':
             datastore_path = arg
 
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
     app = backend.changedetection_app(app_config, datastore)
 
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
     app.config['datastore_path'] = datastore_path
-    app.secret_key = init_app_secret(app_config['datastore_path'])
+
 
     @app.context_processor
     def inject_version():

docker-compose.yml

@@ -6,8 +6,9 @@ services:
       hostname: changedetection.io
       volumes:
         - changedetection-data:/datastore
-
   #    environment:
+  #      - PUID=1000
+  #      - PGID=1000
   #        Proxy support example.
   #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
   #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"

requirements.txt

@@ -11,6 +11,9 @@ feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
 urllib3
+wtforms ~= 2.3.3
+
+
 
 # Notification library
 apprise ~= 0.9