tag 0.38.2

Re #156 - You can specify the port here too, to be sure

.github/workflows/codeql-analysis.yml

@@ -12,11 +12,6 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
   schedule:
     - cron: '27 9 * * 4'
 

.github/workflows/image-tag.yml

@@ -0,0 +1,91 @@
+name: Test, build and push tagged release to Docker Hub
+
+on:
+  push:
+    tags:
+      - '*.*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+
+#        with:
+#          tagRegex: "foobar-(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+#          tagRegexGroup: 1 # Optional. Default is 1.
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > backend/source.txt
+          echo ${{ github.ref }} > backend/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: tag
+        run : echo ${{ github.event.release.tag_name }}
+
+      - name: Build and push tagged version
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ steps.tagName.outputs.tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+        env:
+            SOURCE_NAME: ${{ steps.branch_name.outputs.SOURCE_NAME }}
+            SOURCE_BRANCH: ${{ steps.branch_name.outputs.SOURCE_BRANCH }}
+            SOURCE_TAG: ${{ steps.branch_name.outputs.SOURCE_TAG }}
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/image.yml

@@ -2,7 +2,7 @@ name: Test, build and push to Docker Hub
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, arm-build ]
 
 jobs:
   build:
@@ -27,9 +27,17 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > backend/source.txt
+          echo ${{ github.ref }} > backend/tag.txt
+
       - name: Test with pytest
         run: |
-          cd backend; pytest
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -49,6 +57,7 @@ jobs:
           install: true
           version: latest
           driver-opts: image=moby/buildkit:master
+
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v2
@@ -60,18 +69,20 @@ jobs:
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
           #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+#          platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+# failed: Cache service responded with 503
+#      - name: Cache Docker layers
+#        uses: actions/cache@v2
+#        with:
+#          path: /tmp/.buildx-cache
+#          key: ${{ runner.os }}-buildx-${{ github.sha }}
+#          restore-keys: |
+#            ${{ runner.os }}-buildx-
 
 

.github/workflows/test-only.yml

@@ -1,10 +1,7 @@
-name: Test only
+name: ChangeDetection.io Test
 
-on:
-  push:
-    branches:
-      - /refs/heads/*
-      - !master
+# Triggers the workflow on push or pull request events
+on: [push, pull_request]
 
 jobs:
   build:
@@ -31,5 +28,6 @@ jobs:
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Test with pytest
         run: |
-          cd backend; pytest
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
 

Dockerfile

@@ -1,30 +1,56 @@
+# pip dependencies install stage
+FROM python:3.8-slim as builder
+
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --target=/dependencies -r /requirements.txt
+
+# Final image stage
 FROM python:3.8-slim
-COPY requirements.txt /tmp/requirements.txt
-RUN apt-get update && apt-get install -y gcc libc-dev libxslt-dev zlib1g-dev g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
-RUN pip3 install --no-cache-dir -r /tmp/requirements.txt
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
+# Re #93, #73, excluding rustc (adds another 430Mb~)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
+ENV PYTHONUNBUFFERED=1
 
-RUN [ ! -d "/app" ] && mkdir /app
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
+# Copy modules over to the final image and add their dir to PYTHONPATH
+COPY --from=builder /dependencies /usr/local
+ENV PYTHONPATH=/usr/local
+
 # The actual flask app
 COPY backend /app/backend
-
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
 
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
-# Attempt to store the triggered commit
-ARG SOURCE_COMMIT
-ARG SOURCE_BRANCH
-RUN echo "commit: $SOURCE_COMMIT branch: $SOURCE_BRANCH" >/source.txt
-
 CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
-
-
-

README.md

@@ -1,5 +1,5 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/image.yml/badge.svg?branch=master)
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
   <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
 </a>
@@ -25,9 +25,23 @@ Know when ...
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
+- COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
 
+_Need an actual Chrome runner with Javascript support? see the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome support changedetection.io branch!</a>_
 
 **Get monitoring now! super simple, one command!**
+Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
+
+With one docker-compose command
+
+```bash
+docker-compose up -d
+```
+
+or
+
 
 ```bash
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
@@ -50,7 +64,96 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
 
 Examining differences in content.
 
-
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
+### Notifications
+
+ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
+Simply set one or more notification URL's in the _[edit]_ tab of that watch.
+
+Just some examples
+
+    discord://webhook_id/webhook_token
+    flock://app_token/g:channel_id
+    gitter://token/room
+    gchat://workspace/key/token
+    msteams://TokenA/TokenB/TokenC/
+    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    rocket://user:password@hostname/#Channel
+    mailto://user:pass@example.com?to=receivingAddress@example.com
+    json://someserver.com/custom-api
+    syslog://
+ 
+<a href="https://github.com/caronc/apprise">And everything else in this list!</a>
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
+
+Now you can also customise your notification content!
+
+### JSON API Monitoring
+
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+
+![image](https://user-images.githubusercontent.com/275001/125165842-0ce01980-e1dc-11eb-9e73-d8137dd162dc.png)
+
+This will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results
+
+![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
+
+#### Parse JSON embedded in HTML!
+
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+
+```
+<html>
+...
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+</script>
+```  
+
+`json:$.price` would give `23.50`, or you can extract the whole structure
+
+### Proxy
+
+A proxy for ChangeDetection.io can be configured by setting environment the 
+`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
+
+`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
+
+as `docker run` with `-e`
+
+```
+docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+With `docker-compose`, see the `Proxy support example` in <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a>.
+
+For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies
+
+This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
+
+### Notes
+
+- ~~Does not yet support Javascript~~
+- ~~Wont work with Cloudfare type "Please turn on javascript" protected pages~~
+- You can use the 'headers' section to monitor password protected web page changes
+
+See the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome browser support!</a>
+
+### RaspberriPi support?
+
+RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

backend/README-pytest.md

@@ -1 +0,0 @@
-Note: run `pytest` from this directory.

backend/__init__.py

@@ -6,7 +6,6 @@
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
-# @todo make tables responsive!
 # @todo fetch title into json
 # https://distill.io/features
 # proxy per check
@@ -23,7 +22,7 @@ from threading import Event
 
 import queue
 
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for
+from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for, flash
 
 from feedgen.feed import FeedGenerator
 from flask import make_response
@@ -36,12 +35,13 @@ datastore = None
 running_update_threads = []
 ticker_thread = None
 
-messages = []
 extra_stylesheets = []
 
 update_q = queue.Queue()
 
-app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
+notification_q = queue.Queue()
+
+app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -52,10 +52,40 @@ app.config['NEW_VERSION_AVAILABLE'] = False
 
 app.config['LOGIN_DISABLED'] = False
 
+#app.config["EXPLAIN_TEMPLATE_LOADING"] = True
+
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
 
+def init_app_secret(datastore_path):
+    secret = ""
+
+    path = "{}/secret.txt".format(datastore_path)
+
+    try:
+        with open(path, "r") as f:
+            secret = f.read()
+
+    except FileNotFoundError:
+        import secrets
+        with open(path, "w") as f:
+            secret = secrets.token_hex(32)
+            f.write(secret)
+
+    return secret
+
+# Remember python is by reference
+# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
+def populate_form_from_watch(form, watch):
+    for i in form.__dict__.keys():
+        if i[0] != '_':
+            p = getattr(form, i)
+            if hasattr(p, 'data') and i in watch:
+                if not p.data:
+                    setattr(p, "data", watch[i])
+
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -92,6 +122,7 @@ class User(flask_login.UserMixin):
     def get_user(self, email="defaultuser@changedetection.io"):
         return self
     def is_authenticated(self):
+
         return True
     def is_active(self):
         return True
@@ -122,7 +153,7 @@ class User(flask_login.UserMixin):
 
     pass
 
-def changedetection_app(conig=None, datastore_o=None):
+def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
@@ -131,7 +162,7 @@ def changedetection_app(conig=None, datastore_o=None):
 
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
-
+    app.secret_key = init_app_secret(config['datastore_path'])
 
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
@@ -158,12 +189,12 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route('/login', methods=['GET', 'POST'])
     def login():
 
-        global messages
+        if not datastore.data['settings']['application']['password']:
+            flash("Login not required, no password enabled.", "notice")
+            return redirect(url_for('index'))
 
         if request.method == 'GET':
-            output = render_template("login.html", messages=messages)
-            # Show messages but once.
-            messages = []
+            output = render_template("login.html")
             return output
 
         user = User()
@@ -179,7 +210,7 @@ def changedetection_app(conig=None, datastore_o=None):
             return redirect(next or url_for('index'))
 
         else:
-            messages.append({'class': 'error', 'message': 'Incorrect password'})
+            flash('Incorrect password', 'error')
 
         return redirect(url_for('login'))
 
@@ -191,9 +222,7 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_required
     def index():
-        global messages
         limit_tag = request.args.get('tag')
-
         pause_uuid = request.args.get('pause')
 
         if pause_uuid:
@@ -249,44 +278,66 @@ def changedetection_app(conig=None, datastore_o=None):
             return response
 
         else:
+            from backend import forms
+            form = forms.quickWatchForm(request.form)
+
             output = render_template("watch-overview.html",
+                                     form=form,
                                      watches=sorted_watches,
-                                     messages=messages,
                                      tags=existing_tags,
                                      active_tag=limit_tag,
                                      has_unviewed=datastore.data['has_unviewed'])
 
-            # Show messages but once.
-            messages = []
-
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
     @login_required
     def scrub_page():
-        from pathlib import Path
 
-        global messages
+        import re
 
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
-            limit_timestamp = int(request.form.get('limit_date'))
+            limit_date = request.form.get('limit_date')
+            limit_timestamp = 0
+
+            # Re #149 - allow empty/0 timestamp limit
+            if len(limit_date):
+                try:
+                    limit_date = limit_date.replace('T', ' ')
+                    # I noticed chrome will show '/' but actually submit '-'
+                    limit_date = limit_date.replace('-', '/')
+                    # In the case that :ss seconds are supplied
+                    limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+
+                    str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                    limit_timestamp = int(str_to_dt.timestamp())
+
+                    if limit_timestamp > time.time():
+                        flash("Timestamp is in the future, cannot continue.", 'error')
+                        return redirect(url_for('scrub_page'))
+
+                except ValueError:
+                    flash('Incorrect date format, cannot continue.', 'error')
+                    return redirect(url_for('scrub_page'))
 
             if confirmtext == 'scrub':
-
+                changes_removed = 0
                 for uuid, watch in datastore.data['watching'].items():
-                    if len(str(limit_timestamp)) == 10:
-                        datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
+                    if limit_timestamp:
+                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
                     else:
-                        datastore.scrub_watch(uuid)
+                        changes_removed += datastore.scrub_watch(uuid)
 
-                messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
+                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
             else:
-                messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
+                flash('Incorrect confirmation text.', 'error')
 
             return redirect(url_for('index'))
 
-        return render_template("scrub.html")
+        output =  render_template("scrub.html")
+        return output
+
 
     # If they edited an existing watch, we need to know to reset the current/previous md5 to include
     # the excluded text.
@@ -316,117 +367,151 @@ def changedetection_app(conig=None, datastore_o=None):
 
         return datastore.data['watching'][uuid]['previous_md5']
 
+
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_required
     def edit_page(uuid):
-        global messages
-        import validators
+        from backend import forms
+        form = forms.watchForm(request.form)
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        if request.method == 'POST':
+        if request.method == 'GET':
+            if not uuid in datastore.data['watching']:
+                flash("No watch with the UUID %s found." % (uuid), "error")
+                return redirect(url_for('index'))
 
-            url = request.form.get('url').strip()
-            tag = request.form.get('tag').strip()
+            populate_form_from_watch(form, datastore.data['watching'][uuid])
 
-            # Extra headers
-            form_headers = request.form.get('headers').strip().split("\n")
-            extra_headers = {}
-            if form_headers:
-                for header in form_headers:
-                    if len(header):
-                        parts = header.split(':', 1)
-                        if len(parts) == 2:
-                            extra_headers.update({parts[0].strip(): parts[1].strip()})
+        if request.method == 'POST' and form.validate():
 
-            update_obj = {'url': url,
-                          'tag': tag,
-                          'headers': extra_headers
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
+                form.minutes_between_check.data = None
+
+            update_obj = {'url': form.url.data.strip(),
+                          'minutes_between_check': form.minutes_between_check.data,
+                          'tag': form.tag.data.strip(),
+                          'title': form.title.data.strip(),
+                          'headers': form.headers.data
                           }
 
+            # Notification URLs
+            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
+
             # Ignore text
-            form_ignore_text = request.form.get('ignore-text').strip()
-            ignore_text = []
-            if len(form_ignore_text):
-                for text in form_ignore_text.split("\n"):
-                    text = text.strip()
-                    if len(text):
-                        ignore_text.append(text)
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
 
-                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
-
-                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            if form_ignore_text:
                 if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-            validators.url(url)  # @todo switch to prop/attr/observer
+
+            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
+
             datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
+            flash("Updated watch.")
 
-            messages.append({'class': 'ok', 'message': 'Updated watch.'})
+            # Queue the watch for immediate recheck
+            update_q.put(uuid)
 
-            return redirect(url_for('index'))
+            if form.trigger_check.data:
+                n_object = {'watch_url': form.url.data.strip(),
+                            'notification_urls': form.notification_urls.data,
+                            'uuid': uuid}
+                notification_q.put(n_object)
+
+                flash('Notifications queued.')
+
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('diff_history_page', uuid=uuid))
+            else:
+                return redirect(url_for('index'))
 
         else:
-            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            # Re #110 offer the default minutes
+            using_default_minutes = False
+            if form.minutes_between_check.data == None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
+                using_default_minutes = True
+
+            output = render_template("edit.html",
+                                     uuid=uuid,
+                                     watch=datastore.data['watching'][uuid],
+                                     form=form,
+                                     using_default_minutes=using_default_minutes
+                                     )
 
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
     @login_required
     def settings_page():
-        global messages
+
+        from backend import forms
+        form = forms.globalSettingsForm(request.form)
 
         if request.method == 'GET':
-            if request.values.get('removepassword'):
+            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
+            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+            form.notification_title.data = datastore.data['settings']['application']['notification_title']
+            form.notification_body.data = datastore.data['settings']['application']['notification_body']
+
+            # Password unset is a GET
+            if request.values.get('removepassword') == 'yes':
                 from pathlib import Path
-
                 datastore.data['settings']['application']['password'] = False
-                messages.append({'class': 'notice', 'message': "Password protection removed."})
+                flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
-
                 return redirect(url_for('settings_page'))
 
-        if request.method == 'POST':
+        if request.method == 'POST' and form.validate():
 
-            password = request.values.get('password')
-            if password:
-                import hashlib
-                import base64
-                import secrets
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
+            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
+            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
 
-                # Make a new salt on every new password and store it with the password
-                salt = secrets.token_bytes(32)
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.needs_write = True
 
-                key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
-                store = base64.b64encode(salt + key).decode('ascii')
-                datastore.data['settings']['application']['password'] = store
-                messages.append({'class': 'notice', 'message': "Password protection enabled."})
+            if form.trigger_check.data and len(form.notification_urls.data):
+                n_object = {'watch_url': "Test from changedetection.io!",
+                            'notification_urls': form.notification_urls.data}
+                notification_q.put(n_object)
+                flash('Notifications queued.')
+
+            if form.password.encrypted_password:
+                datastore.data['settings']['application']['password'] = form.password.encrypted_password
+                flash("Password protection enabled.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('index'))
 
-            try:
-                minutes = int(request.values.get('minutes').strip())
-            except ValueError:
-                messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
+            flash("Settings updated.")
 
-            else:
-                if minutes >= 5:
-                    datastore.data['settings']['requests']['minutes_between_check'] = minutes
-                    datastore.needs_write = True
+        if request.method == 'POST' and not form.validate():
+            flash("An error occurred, please see below.", "error")
 
-                    messages.append({'class': 'ok', 'message': "Updated"})
-                else:
-                    messages.append(
-                        {'class': 'error', 'message': "Must be atleast 5 minutes."})
-
-
-
-        output = render_template("settings.html", messages=messages,
-                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
-        messages = []
+        # Same as notification.py
+        base_url = os.getenv('BASE_URL', '').strip('"')
+        output = render_template("settings.html", form=form, base_url=base_url)
 
         return output
 
@@ -434,7 +519,6 @@ def changedetection_app(conig=None, datastore_o=None):
     @login_required
     def import_page():
         import validators
-        global messages
         remaining_urls = []
 
         good = 0
@@ -452,7 +536,7 @@ def changedetection_app(conig=None, datastore_o=None):
                     if len(url):
                         remaining_urls.append(url)
 
-            messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
+            flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
 
             if len(remaining_urls) == 0:
                 # Looking good, redirect to index.
@@ -460,11 +544,8 @@ def changedetection_app(conig=None, datastore_o=None):
 
         # Could be some remaining, or we could be on GET
         output = render_template("import.html",
-                                 messages=messages,
                                  remaining="\n".join(remaining_urls)
                                  )
-        messages = []
-
         return output
 
     # Clear all statuses, so we do not see the 'unviewed' class
@@ -476,23 +557,22 @@ def changedetection_app(conig=None, datastore_o=None):
         for watch_uuid, watch in datastore.data['watching'].items():
             datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
 
-        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
+        flash("Cleared all statuses.")
         return redirect(url_for('index'))
 
     @app.route("/diff/<string:uuid>", methods=['GET'])
     @login_required
     def diff_history_page(uuid):
-        global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/css/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
         dates = list(watch['history'].keys())
@@ -502,8 +582,7 @@ def changedetection_app(conig=None, datastore_o=None):
         dates = [str(i) for i in dates]
 
         if len(dates) < 2:
-            messages.append(
-                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
+            flash("Not enough saved change detection snapshots to produce a report.", "error")
             return redirect(url_for('index'))
 
         # Save the current newest history as the most recently viewed
@@ -525,39 +604,44 @@ def changedetection_app(conig=None, datastore_o=None):
             previous_version_file_contents = f.read()
 
         output = render_template("diff.html", watch_a=watch,
-                                 messages=messages,
                                  newest=newest_version_file_contents,
                                  previous=previous_version_file_contents,
                                  extra_stylesheets=extra_stylesheets,
                                  versions=dates[1:],
+                                 uuid=uuid,
                                  newest_version_timestamp=dates[0],
                                  current_previous_version=str(previous_version),
-                                 current_diff_url=watch['url'])
+                                 current_diff_url=watch['url'],
+                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
+                                 left_sticky= True )
 
         return output
 
     @app.route("/preview/<string:uuid>", methods=['GET'])
     @login_required
     def preview_page(uuid):
-        global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/css/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
-        print(watch)
-        with open(list(watch['history'].values())[-1], 'r') as f:
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
             content = f.readlines()
 
-        output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets)
+        output = render_template("preview.html",
+                                 content=content,
+                                 extra_stylesheets=extra_stylesheets,
+                                 current_diff_url=watch['url'],
+                                 uuid=uuid)
         return output
 
 
@@ -573,6 +657,10 @@ def changedetection_app(conig=None, datastore_o=None):
         import zipfile
         from pathlib import Path
 
+        # Remove any existing backup file, for now we just keep one file
+        for previous_backup_filename in Path(app.config['datastore_path']).rglob('changedetection-backup-*.zip'):
+            os.unlink(previous_backup_filename)
+
         # create a ZipFile object
         backupname = "changedetection-backup-{}.zip".format(int(time.time()))
 
@@ -602,7 +690,20 @@ def changedetection_app(conig=None, datastore_o=None):
                                  compress_type=zipfile.ZIP_DEFLATED,
                                  compresslevel=8)
 
-        return send_from_directory(app.config['datastore_path'], backupname)
+            # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+            list_file = os.path.join(app.config['datastore_path'], "url-list.txt")
+            with open(list_file, "w") as f:
+                for uuid in datastore.data['watching']:
+                    url = datastore.data['watching'][uuid]['url']
+                    f.write("{}\r\n".format(url))
+
+            # Add it to the Zip
+            zipObj.write(list_file,
+                         arcname="url-list.txt",
+                         compress_type=zipfile.ZIP_DEFLATED,
+                         compresslevel=8)
+
+        return send_from_directory(app.config['datastore_path'], backupname, as_attachment=True)
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
@@ -618,28 +719,34 @@ def changedetection_app(conig=None, datastore_o=None):
     @app.route("/api/add", methods=['POST'])
     @login_required
     def api_watch_add():
-        global messages
+        from backend import forms
+        form = forms.quickWatchForm(request.form)
 
-        url = request.form.get('url').strip()
-        if datastore.url_exists(url):
-            messages.append({'class': 'error', 'message': 'The URL {} already exists'.format(url)})
+        if form.validate():
+
+            url = request.form.get('url').strip()
+            if datastore.url_exists(url):
+                flash('The URL {} already exists'.format(url), "error")
+                return redirect(url_for('index'))
+
+            # @todo add_watch should throw a custom Exception for validation etc
+            new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+            # Straight into the queue.
+            update_q.put(new_uuid)
+
+            flash("Watch added.")
+            return redirect(url_for('index'))
+        else:
+            flash("Error")
             return redirect(url_for('index'))
-
-        # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-        # Straight into the queue.
-        update_q.put(new_uuid)
-
-        messages.append({'class': 'ok', 'message': 'Watch added.'})
-        return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
-        global messages
+
         uuid = request.args.get('uuid')
         datastore.delete(uuid)
-        messages.append({'class': 'ok', 'message': 'Deleted.'})
+        flash('Deleted.')
 
         return redirect(url_for('index'))
 
@@ -647,8 +754,6 @@ def changedetection_app(conig=None, datastore_o=None):
     @login_required
     def api_watch_checknow():
 
-        global messages
-
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
         i = 0
@@ -679,13 +784,14 @@ def changedetection_app(conig=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(watch_uuid)
                     i += 1
-
-        messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
+        flash("{} watches are rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
+    threading.Thread(target=notification_runner).start()
+
     # Check for new release version
     threading.Thread(target=check_for_new_version).start()
     return app
@@ -702,7 +808,9 @@ def check_for_new_version():
         try:
             r = requests.post("https://changedetection.io/check-ver.php",
                               data={'version': datastore.data['version_tag'],
-                                    'app_guid': datastore.data['app_guid']},
+                                    'app_guid': datastore.data['app_guid'],
+                                    'watch_count': len(datastore.data['watching'])
+                                    },
 
                               verify=False)
         except:
@@ -717,74 +825,62 @@ def check_for_new_version():
         # Check daily
         app.config.exit.wait(86400)
 
+def notification_runner():
+    while not app.config.exit.is_set():
+        try:
+            # At the moment only one thread runs (single runner)
+            n_object = notification_q.get(block=False)
+        except queue.Empty:
+            time.sleep(1)
 
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-class Worker(threading.Thread):
-    current_uuid = None
-
-    def __init__(self, q, *args, **kwargs):
-        self.q = q
-        super().__init__(*args, **kwargs)
-
-    def run(self):
-        from backend import fetch_site_status
-
-        update_handler = fetch_site_status.perform_site_check(datastore=datastore)
-
-        while not app.config.exit.is_set():
-
+        else:
+            # Process notifications
             try:
-                uuid = self.q.get(block=False)
-            except queue.Empty:
-                pass
+                from backend import notification
+                notification.process_notification(n_object, datastore)
 
-            else:
-                self.current_uuid = uuid
+            except Exception as e:
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'], e))
 
-                if uuid in list(datastore.data['watching'].keys()):
-                    try:
-                        changed_detected, result, contents = update_handler.run(uuid)
-
-                    except PermissionError as s:
-                        app.logger.error("File permission error updating", uuid, str(s))
-                    else:
-                        if result:
-                            datastore.update_watch(uuid=uuid, update_obj=result)
-                            if changed_detected:
-                                # A change was detected
-                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
-
-                self.current_uuid = None  # Done
-                self.q.task_done()
-
-            app.config.exit.wait(1)
 
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+    from backend import update_worker
+
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
-        new_worker = Worker(update_q)
+        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
         running_update_threads.append(new_worker)
         new_worker.start()
 
     while not app.config.exit.is_set():
 
+        # Get a list of watches by UUID that are currently fetching data
         running_uuids = []
         for t in running_update_threads:
             if t.current_uuid:
                 running_uuids.append(t.current_uuid)
 
-        # Look at the dataset, find a stale watch to process
-
-        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
-
-        threshold = time.time() - (minutes * 60)
+        # Check for watches outside of the time threshold to put in the thread queue.
         for uuid, watch in datastore.data['watching'].items():
+
+            # If they supplied an individual entry minutes to threshold.
+            if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
+                max_time = watch['minutes_between_check'] * 60
+            else:
+                # Default system wide.
+                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
+
+            threshold = time.time() - max_time
+
+            # Yeah, put it in the queue, it's more than time.
             if not watch['paused'] and watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
+        # Wait a few seconds before checking the list again
+        time.sleep(3)
+
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

backend/fetch_site_status.py

@@ -3,8 +3,11 @@ import requests
 import hashlib
 from inscriptis import get_text
 import urllib3
+from . import html_tools
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+
 # Some common stuff here that can be moved to a base class
 class perform_site_check():
 
@@ -13,18 +16,34 @@ class perform_site_check():
         self.datastore = datastore
 
     def strip_ignore_text(self, content, list_ignore_text):
+        import re
         ignore = []
+        ignore_regex = []
         for k in list_ignore_text:
-            ignore.append(k.encode('utf8'))
+
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
 
         output = []
         for line in content.splitlines():
-            line = line.encode('utf8')
 
             # Always ignore blank lines in this mode. (when this function gets called)
             if len(line.strip()):
-                if not any(skip_text in line for skip_text in ignore):
-                    output.append(line)
+                regex_matches = False
+
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
+
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
 
         return "\n".encode('utf8').join(output)
 
@@ -32,6 +51,7 @@ class perform_site_check():
 
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
+
         stripped_text_from_html = False
         changed_detected = False
 
@@ -43,7 +63,7 @@ class perform_site_check():
         extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers']
+        request_headers = self.datastore.data['settings']['headers'].copy()
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
@@ -66,25 +86,38 @@ class perform_site_check():
                              timeout=timeout,
                              verify=False)
 
-            stripped_text_from_html = get_text(r.text)
+            html = r.text
+
+            is_html = True
+            css_filter_rule = self.datastore.data['watching'][uuid]['css_filter']
+            if css_filter_rule and len(css_filter_rule.strip()):
+                if 'json:' in css_filter_rule:
+                    stripped_text_from_html = html_tools.extract_json_as_string(html, css_filter_rule)
+                    is_html = False
+                else:
+                    # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                    html = html_tools.css_filter(css_filter=css_filter_rule, html_content=r.content)
+
+            if is_html:
+                stripped_text_from_html = get_text(html)
 
         # Usually from networkIO/requests level
         except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
             update_obj["last_error"] = str(e)
-
             print(str(e))
 
         except requests.exceptions.MissingSchema:
             print("Skipping {} due to missing schema/bad url".format(uuid))
 
         # Usually from html2text level
-        except UnicodeDecodeError as e:
-
+        except Exception as e:
+            #        except UnicodeDecodeError as e:
             update_obj["last_error"] = str(e)
             print(str(e))
             # figure out how to deal with this cleaner..
             # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
 
+
         else:
             # We rely on the actual text in the html output.. many sites have random script vars etc,
             # in the future we'll implement other mechanisms.
@@ -115,4 +148,10 @@ class perform_site_check():
 
                 update_obj["previous_md5"] = fetched_md5
 
+            # Extract title as title
+            if self.datastore.data['settings']['application']['extract_title_as_title']:
+                if not self.datastore.data['watching'][uuid]['title'] or not len(self.datastore.data['watching'][uuid]['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=html)
+
+
         return changed_detected, update_obj, stripped_text_from_html

backend/forms.py

@@ -0,0 +1,159 @@
+from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+    Field
+from wtforms import widgets
+from wtforms.validators import ValidationError
+from wtforms.fields import html5
+
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            return "\r\n".join(self.data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            self.data = [x.strip() for x in cleaned]
+            p = 1
+        else:
+            self.data = []
+
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import hashlib
+        import base64
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Be really sure it's non-zero in length
+            if len(valuelist[0].strip()) > 0:
+                self.encrypted_password = self.build_password(valuelist[0])
+                self.data = ""
+        else:
+            self.data = False
+
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':', 1)
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ValidateListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import re
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+class ValidateCSSJSONInput(object):
+    """
+    Filter validation
+    @todo CSS validator ;)
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if 'json:' in field.data:
+            from jsonpath_ng.exceptions import JsonPathParserError
+            from jsonpath_ng import jsonpath, parse
+
+            input = field.data.replace('json:', '')
+
+            try:
+                parse(input)
+            except JsonPathParserError as e:
+                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                raise ValidationError(message % (input, str(e)))
+
+class quickWatchForm(Form):
+    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
+    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
+
+    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
+
+class watchForm(quickWatchForm):
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
+    css_filter = StringField('CSS/JSON Filter', [ValidateCSSJSONInput()])
+    title = StringField('Title')
+
+    ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    notification_urls = StringListField('Notification URL List')
+    headers = StringDictKeyValue('Request Headers')
+    trigger_check = BooleanField('Send test notification on save')
+
+
+class globalSettingsForm(Form):
+
+    password = SaltyPasswordField()
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.NumberRange(min=1)])
+
+    notification_urls = StringListField('Notification URL List')
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+    trigger_check = BooleanField('Send test notification on save')
+
+    notification_title = StringField('Notification Title')
+    notification_body = TextAreaField('Notification Body')

backend/html_tools.py

@@ -0,0 +1,90 @@
+import json
+from bs4 import BeautifulSoup
+from jsonpath_ng import parse
+
+
+class JSONNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+
+#
+def _parse_json(json_data, jsonpath_filter):
+    s=[]
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
+    match = jsonpath_expression.find(json_data)
+
+    # More than one result, we will return it as a JSON list.
+    if len(match) > 1:
+        for i in match:
+            s.append(i.value)
+
+    # Single value, use just the value, as it could be later used in a token in notifications.
+    if len(match) == 1:
+        s = match[0].value
+
+    if not s:
+        raise JSONNotFound("No Matching JSON could be found for the rule {}".format(jsonpath_filter.replace('json:', '')))
+
+    stripped_text_from_html = json.dumps(s, indent=4)
+
+    return stripped_text_from_html
+
+def extract_json_as_string(content, jsonpath_filter):
+
+    stripped_text_from_html = False
+
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    try:
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
+    except json.JSONDecodeError:
+
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
+        s = []
+        soup = BeautifulSoup(content, 'html.parser')
+        bs_result = soup.findAll('script')
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
+        for result in bs_result:
+            # Skip empty tags, and things that dont even look like JSON
+            if not result.string or not '{' in result.string:
+                continue
+                
+            try:
+                json_data = json.loads(result.string)
+            except json.JSONDecodeError:
+                # Just skip it
+                continue
+            else:
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
+                if stripped_text_from_html:
+                    break
+
+    if not stripped_text_from_html:
+        raise JSONNotFound("No JSON matching the rule '%s' found" % jsonpath_filter.replace('json:',''))
+
+    return stripped_text_from_html

backend/notification.py

@@ -0,0 +1,57 @@
+import os
+import apprise
+
+def process_notification(n_object, datastore):
+    apobj = apprise.Apprise()
+    for url in n_object['notification_urls']:
+        print (">> Process Notification: AppRise notifying {}".format(url.strip()))
+        apobj.add(url.strip())
+
+    # Get the notification body from datastore
+    n_body = datastore.data['settings']['application']['notification_body']
+    # Get the notification title from the datastore
+    n_title = datastore.data['settings']['application']['notification_title']
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object)
+    raw_notification_text = [n_body, n_title]
+
+    parameterised_notification_text = dict(
+        [
+            (i, n.replace(n, n.format(**notification_parameters)))
+            for i, n in zip(['body', 'title'], raw_notification_text)
+        ]
+    )
+
+    apobj.notify(
+        body=parameterised_notification_text["body"],
+        title=parameterised_notification_text["title"]
+    )
+
+
+# Notification title + body content parameters get created here.
+def create_notification_parameters(n_object):
+
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    # Create URLs to customise the notification with
+    base_url = os.getenv('BASE_URL', '').strip('"')
+    watch_url = n_object['watch_url']
+
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
+
+    return {
+        'base_url': base_url,
+        'watch_url': watch_url,
+        'diff_url': diff_url,
+        'preview_url': preview_url,
+        'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+    }
+

backend/pytest.ini

@@ -1,7 +1,7 @@
 [pytest]
 addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
-#live_server_scope = session
+#live_server_scope = function
 
 filterwarnings =
     ignore::DeprecationWarning:urllib3.*:

backend/run_all_tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://foobar.com"
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done

backend/static/css/styles.css

@@ -1,277 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- */
-body {
-  color: #333;
-  background: #262626;
-}
-
-.pure-table-even {
-  background: #fff;
-}
-/* Some styles from https://css-tricks.com/ */
-
-a {
-  text-decoration: none;
-  color: #1b98f8;
-}
-a.github-link {
-  color: #fff;
-}
-
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-
-  align-items: center;
-}
-
-section.content {
-
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.pure-table.watch-table td {
-  font-size: 80%;
-}
-
-/* table related */
-.watch-table {
-  width: 100%;
-
-}
-
-.watch-table tr.unviewed {
-    font-weight: bold;
-}
-
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap;
-}
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center;
-}
-
-.watch-table .error {
-  color: #a00;
-}
-
-.watch-table td {
-  white-space: nowrap;
-}
-
-.watch-table td.title-col {
-  word-break: break-all;
-  white-space: normal;
-}
-.watch-table th {
-  white-space: nowrap;
-}
-
-.watch-table .title-col a[target="_blank"]::after, .current-diff-url::after {
-  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-  margin: 0 3px 0 5px;
-}
-
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px;
-}
-#post-list-buttons li {
-  display: inline-block;
-}
-
-#post-list-buttons a {
-   border-top-left-radius: initial;
-   border-top-right-radius: initial;
-   border-bottom-left-radius: 5px;
-   border-bottom-right-radius: 5px;
-}
-
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
-}
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1;
-}
-body::after {
-  opacity: 0.91;
-}
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-body:before {
-  background-size: cover
-}
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
-}
-
-
-.button-small {
-  font-size: 85%;
-}
-
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block;
-}
-
-.edit-form {
-   background: #fff;
-   padding: 2em;
-   margin: 1em;
-   border-radius: 5px;
-}
-.button-secondary {
-    color: white;
-    border-radius: 4px;
-    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
-}
-
-.button-success {
-    background: rgb(28, 184, 65);
-    /* this is a green */
-}
-
-.button-tag {
-    background: rgb(99, 99, 99);
-    color: #fff;
-    font-size: 65%;
-    border-bottom-left-radius: initial;
-    border-bottom-right-radius: initial;
-}
-.button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold;
-}
-.button-error {
-    background: rgb(202, 60, 60);
-    /* this is a maroon */
-}
-
-.button-warning {
-    background: rgb(223, 117, 20);
-    /* this is an orange */
-}
-
-.button-secondary {
-    background: rgb(66, 184, 221);
-    /* this is a light blue */
-}
-
-
-.button-cancel {
-    background: rgb(200, 200, 200);
-    /* this is a green */
-}
-
-.messages {
-    padding: 1em;
-    background: rgba(255,255,255,.2);
-    border-radius: 10px;
-    color: #fff;
-    font-weight: bold;
-}
-
-.pure-form label  {
-  font-weight: bold;
-}
-
-#new-watch-form {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-}
-#new-watch-form legend {
-    color: #fff;
-}
-
-#diff-col {
-    padding-left:40px;
-}
-#diff-jump {
-    position: fixed;
-    left: 0px;
-    top: 80px;
-    background: #fff;
-    padding: 10px;
-    border-top-right-radius: 5px;
-    border-bottom-right-radius: 5px;
-    box-shadow: 5px 0 5px -2px #888;
-}
-
-#diff-jump a {
-    color: #1b98f8;
-    cursor: grabbing;
-    -moz-user-select: none;
-    -webkit-user-select: none;
-    -ms-user-select:none;
-    user-select:none;
-    -o-user-select:none;
-}
-
-footer {
-    padding: 10px;
-    background: #fff;
-    color: #444;
-    text-align: center;
-}
-
-#feed-icon {
-    vertical-align: middle;
-}
-
-#version {
-    position: absolute;
-    top: 80px;
-    right: 0px;
-    font-size: 8px;
-    background: #fff;
-    padding: 10px;
-}
-
-#new-version-text a{
-    color: #e07171;
-}
-
-.paused-state.state-False img {
-    opacity: 0.2;
-}
-
-
-.paused-state.state-False:hover img{
-    opacity: 0.8;
-}

backend/static/js/settings.js

@@ -0,0 +1,16 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+
+  document.getElementById("toggle-customise-notifications").onclick = function () {
+    toggleVisible("notification-customisation");
+  };
+});

backend/static/styles/.gitignore

@@ -0,0 +1 @@
+node_modules

backend/static/styles/diff.css

@@ -0,0 +1,56 @@
+#diff-ui {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  font-size: 9px; }
+  #diff-ui table {
+    table-layout: fixed;
+    width: 100%; }
+  #diff-ui td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left; }
+  #diff-ui pre {
+    white-space: pre-wrap; }
+
+h1 {
+  display: inline;
+  font-size: 100%; }
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7; }
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none; }
+
+#result {
+  white-space: pre-wrap; }
+
+#settings {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%; }
+  #settings label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal; }
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em; }
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */ } }

backend/static/styles/diff.scss

@@ -1,15 +1,25 @@
-table {
-	table-layout: fixed;
-	width: 100%;
-}
-td {
-	width: 33%;
-	padding: 3px 4px;
-	border: 1px solid transparent;
-	vertical-align: top;
-	font: 1em monospace;
-	text-align: left;
-	white-space: pre-wrap;
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin: 1em;
+    border-radius: 5px;
+    font-size: 9px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+    }
+    pre {
+            white-space: pre-wrap;
+    }
 }
 h1 {
 	display: inline;
@@ -33,16 +43,16 @@ ins {
 
 #settings {
     background: rgba(0,0,0,.05);
-        padding: 1em;
+    padding: 1em;
     border-radius: 10px;
     margin-bottom: 1em;
     color: #fff;
     font-size: 80%;
-}
-#settings label {
-	margin-left: 1em;
-	display: inline-block;
-	font-weight: normal;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
 }
 
 .source {
@@ -55,12 +65,4 @@ ins {
 	body {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
-}
-
-#diff-ui {
-    background: #fff;
-    padding: 2em;
-    margin: 1em;
-    border-radius: 5px;
-    font-size: 9px;
 }

backend/static/styles/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "changedetection.io-theme",
+  "version": "0.0.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "scss": "node-sass --watch styles.scss diff.scss -o ."
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-sass": "^6.0.1",
+    "trim-newlines": "^3.0.1"
+  }
+}

backend/static/styles/styles.css

@@ -0,0 +1,365 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626; }
+
+.pure-table-even {
+  background: #fff; }
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8; }
+
+a.github-link {
+  color: #fff; }
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center; }
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center; }
+
+/* table related */
+.watch-table {
+  width: 100%; }
+  .watch-table tr.unviewed {
+    font-weight: bold; }
+  .watch-table .error {
+    color: #a00; }
+  .watch-table td {
+    font-size: 80%;
+    white-space: nowrap; }
+  .watch-table td.title-col {
+    word-break: break-all;
+    white-space: normal; }
+  .watch-table th {
+    white-space: nowrap; }
+  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px; }
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap; }
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center; }
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px; }
+  #post-list-buttons li {
+    display: inline-block; }
+  #post-list-buttons a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px; }
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1; }
+
+body::after {
+  opacity: 0.91; }
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png); }
+
+body:before {
+  background-size: cover; }
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
+
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
+
+.button-small {
+  font-size: 85%; }
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block; }
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  min-width: 70%; }
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
+
+.button-success {
+  background: #1cb841;
+  /* this is a green */ }
+
+.button-tag {
+  background: #636363;
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial; }
+  .button-tag.active {
+    background: #9c9c9c;
+    font-weight: bold; }
+
+.button-error {
+  background: #ca3c3c;
+  /* this is a maroon */ }
+
+.button-warning {
+  background: #df7514;
+  /* this is an orange */ }
+
+.button-secondary {
+  background: #42b8dd;
+  /* this is a light blue */ }
+
+.button-cancel {
+  background: #c8c8c8;
+  /* this is a green */ }
+
+.messages li {
+  list-style: none;
+  padding: 1em;
+  border-radius: 10px;
+  color: #fff;
+  font-weight: bold; }
+  .messages li.message {
+    background: rgba(255, 255, 255, 0.2); }
+  .messages li.error {
+    background: rgba(255, 1, 1, 0.5); }
+  .messages li.notice {
+    background: rgba(255, 255, 255, 0.5); }
+
+#notification-customisation {
+  display: block;
+  border: 1px solid #ccc;
+  padding: 1rem;
+  border-radius: 5px; }
+
+#toggle-customise-notifications {
+  cursor: pointer; }
+
+#token-table.pure-table td, #token-table.pure-table th {
+  font-size: 80%; }
+
+#new-watch-form {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em; }
+  #new-watch-form input {
+    width: auto !important;
+    display: inline-block; }
+  #new-watch-form .label {
+    display: none; }
+  #new-watch-form legend {
+    color: #fff; }
+
+#diff-col {
+  padding-left: 40px; }
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888; }
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none; }
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center; }
+
+#feed-icon {
+  vertical-align: middle; }
+
+#top-right-menu {
+  /*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */ }
+
+.sticky-tab {
+  position: absolute;
+  top: 80px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px; }
+  .sticky-tab#left-sticky {
+    left: 0px; }
+  .sticky-tab#right-sticky {
+    right: 0px; }
+
+#new-version-text a {
+  color: #e07171; }
+
+.paused-state.state-False img {
+  opacity: 0.2; }
+
+.paused-state.state-False:hover img {
+  opacity: 0.8; }
+
+.monospaced-textarea textarea {
+  width: 100%;
+  font-family: monospace;
+  white-space: pre;
+  overflow-wrap: normal;
+  overflow-x: scroll; }
+
+.pure-form {
+  /* The input fields with errors */
+  /* The list of errors */ }
+  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
+    padding-bottom: 1em; }
+    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
+      margin: 0px; }
+  .pure-form .error input {
+    background-color: #ffebeb; }
+  .pure-form ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box; }
+    .pure-form ul.errors li {
+      margin-left: 1em;
+      color: #dd0000; }
+  .pure-form label {
+    font-weight: bold; }
+  .pure-form textarea {
+    width: 100%; }
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%; }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0; }
+  #nav-menu {
+    overflow-x: scroll; } }
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  input[type='text'] {
+    width: 100%; }
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    /* Force table to not be like tables anymore */
+    /* Hide table headers (but not display: none;, for accessibility) */ }
+    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
+      display: block; }
+    .watch-table .last-checked::before {
+      color: #555;
+      content: "Last Checked "; }
+    .watch-table .last-changed::before {
+      color: #555;
+      content: "Last Changed "; }
+    .watch-table td.inline {
+      display: inline-block; }
+    .watch-table thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px; }
+    .watch-table .pure-table td, .watch-table .pure-table th {
+      border: none; }
+    .watch-table td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee; }
+      .watch-table td:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap; }
+    .watch-table.pure-table-striped tr {
+      background-color: #fff; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) {
+      background-color: #eee; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
+      background-color: inherit; } }
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+  /* m-d is medium-desktop */
+  .m-d {
+    min-width: 80%; } }

backend/static/styles/styles.scss

@@ -0,0 +1,509 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626;
+}
+
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    font-size: 80%;
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+}
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png);
+}
+
+body:before {
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px;
+    &.right {
+      transform: rotate(-45deg);
+      -webkit-transform: rotate(-45deg);
+    }
+    &.left {
+      transform: rotate(135deg);
+      -webkit-transform: rotate(135deg);
+    }
+    &.up {
+      transform: rotate(-135deg);
+      -webkit-transform: rotate(-135deg);
+    }
+    &.down {
+      transform: rotate(45deg);
+      -webkit-transform: rotate(45deg);
+    }
+}
+
+.button-small {
+  font-size: 85%;
+}
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  min-width: 70%;
+}
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+}
+
+#notification-customisation {
+    display: block;
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+}
+
+#toggle-customise-notifications {
+    cursor: pointer;
+}
+
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  input {
+    width: auto !important;
+    display: inline-block;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+  }
+}
+
+
+
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+}
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none;
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+
+
+.sticky-tab {
+  position: absolute;
+  top: 80px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.paused-state {
+  &.state-False img {
+    opacity: 0.2;
+  }
+
+  &.state-False:hover img {
+    opacity: 0.8;
+  }
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        div {
+            margin: 0px;
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+
+  /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  textarea {
+    width: 100%;
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+
+}
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+
+  input[type='text'] {
+    width: 100%;
+  }
+    
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
+

backend/store.py

@@ -1,9 +1,7 @@
+from os import unlink, path, mkdir
 import json
 import uuid as uuid_builder
-import os.path
-from os import path
 from threading import Lock
-
 from copy import deepcopy
 
 import logging
@@ -39,7 +37,12 @@ class ChangeDetectionStore:
                     'workers': 10  # Number of threads, lower is better for slow connections
                 },
                 'application': {
-                    'password': False
+                    'password': False,
+                    'extract_title_as_title': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': 'ChangeDetection.io Notification - {watch_url}',
+                    'notification_body': '{watch_url} had a change.'
                 }
             }
         }
@@ -54,15 +57,20 @@ class ChangeDetectionStore:
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
             'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
+            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'css_filter': "",
         }
 
-        if path.isfile('/source.txt'):
-            with open('/source.txt') as f:
+        if path.isfile('backend/source.txt'):
+            with open('backend/source.txt') as f:
                 # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
                 # So when someone gives us a backup file to examine, we know exactly what code they were running.
                 self.__data['build_sha'] = f.read()
@@ -109,11 +117,21 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+        self.__data['version_tag'] = "0.38.2"
 
-        self.__data['version_tag'] = "0.28"
+        # Helper to remove password protection
+        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
+        if path.isfile(password_reset_lockfile):
+            self.__data['settings']['application']['password'] = False
+            unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            self.__data['app_guid'] = str(uuid_builder.uuid4())
+            import sys
+            import os
+            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
+                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
+            else:
+                self.__data['app_guid'] = str(uuid_builder.uuid4())
 
         self.needs_write = True
 
@@ -161,7 +179,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
@@ -172,6 +189,10 @@ class ChangeDetectionStore:
                 self.__data['watching'][uuid]['viewed'] = False
                 has_unviewed = True
 
+            # #106 - Be sure this is None on empty string, False, None, etc
+            if not self.__data['watching'][uuid]['title']:
+                self.__data['watching'][uuid]['title'] = None
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data
@@ -191,7 +212,7 @@ class ChangeDetectionStore:
 
     def unlink_history_file(self, path):
         try:
-            os.unlink(path)
+            unlink(path)
         except (FileNotFoundError, IOError):
             pass
 
@@ -233,18 +254,20 @@ class ChangeDetectionStore:
         import hashlib
         del_timestamps = []
 
+        changes_removed = 0
+
         for timestamp, path in self.data['watching'][uuid]['history'].items():
             if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
                 self.unlink_history_file(path)
                 del_timestamps.append(timestamp)
-
-
+                changes_removed += 1
 
                 if not limit_timestamp:
                     self.data['watching'][uuid]['last_checked'] = 0
                     self.data['watching'][uuid]['last_changed'] = 0
                     self.data['watching'][uuid]['previous_md5'] = 0
 
+
         for timestamp in del_timestamps:
             del self.data['watching'][uuid]['history'][str(timestamp)]
 
@@ -264,9 +287,8 @@ class ChangeDetectionStore:
                         self.data['watching'][uuid]['previous_md5'] = False
                         pass
 
-
         self.needs_write = True
-
+        return changes_removed
 
     def add_watch(self, url, tag):
         with self.lock:
@@ -284,7 +306,7 @@ class ChangeDetectionStore:
         # Get the directory ready
         output_path = "{}/{}".format(self.datastore_path, new_uuid)
         try:
-            os.mkdir(output_path)
+            mkdir(output_path)
         except FileExistsError:
             print(output_path, "already exists.")
 
@@ -333,8 +355,24 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
-            
+
             if self.needs_write:
                 self.sync_to_json()
             time.sleep(3)
 
+    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+    # This usually is not used, but can be handy.
+    def remove_unused_snapshots(self):
+        print ("Removing snapshots from datastore that are not in the index..")
+
+        index=[]
+        for uuid in self.data['watching']:
+            for id in self.data['watching'][uuid]['history']:
+                index.append(self.data['watching'][uuid]['history'][str(id)])
+
+        import pathlib
+        # Only in the sub-directories
+        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
+            if not str(item) in index:
+                print ("Removing",item)
+                unlink(item)

backend/templates/_helpers.jinja

@@ -0,0 +1,25 @@
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_simple_field(field) %}
+  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </span>
+{% endmacro %}

backend/templates/base.html

@@ -4,9 +4,9 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
-    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/css/styles.css?ver=1000">
+    <title>Change Detection{{extra_title}}</title>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
         <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -16,40 +16,47 @@
 <body>
 
 <div class="header">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        {% if not current_user.is_authenticated %}
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+        {% if has_password and not current_user.is_authenticated %}
             <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
         {% else %}
-            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
         <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
-        {% if new_version_available %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
         <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
         {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list">
-        {% if current_user.is_authenticated %}
+        <ul class="pure-menu-list"  id="top-right-menu">
+        {% if current_user.is_authenticated or not has_password %}
+            {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="/backup" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/import" class="pure-menu-link">IMPORT</a>
+                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/settings" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
+            {% else %}
+            <li class="pure-menu-item">
+                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
+            </li>
+            {% endif %}
         {% else %}
             <li class="pure-menu-item">
                 <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
         {% endif %}
 
-            {% if current_user.is_authenticated %}
-                <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
-            {% endif %}
+        {% if current_user.is_authenticated %}
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
+        {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
                      version="1.1"
@@ -58,28 +65,26 @@
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
-            <!--
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
-            -->
         </ul>
     </div>
 </div>
-<div id="version">v{{ version }}</div>
+
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
 <section class="content">
     <header>
         {% block header %}{% endblock %}
     </header>
 
-    {% if messages %}
-    <div class="messages">
-        {% for message in messages %}
-        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
+    {% with messages = get_flashed_messages(with_categories=true) %}
+      {% if messages %}
+        <ul class=messages>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
-    </div>
-    {% endif %}
-
-
+        </ul>
+      {% endif %}
+    {% endwith %}
     {% block content %}
 
     {% endblock %}

backend/templates/diff.html

@@ -36,7 +36,6 @@
     <a onclick="next_diff();">Jump</a>
 </div>
 <div id="diff-ui">
-
     <table>
         <tbody>
         <tr>
@@ -130,7 +129,8 @@ if ('oninput' in a) {
 
 function onDiffTypeChange(radio) {
 	window.diffType = radio.value;
-	document.title = "Diff " + radio.value.slice(4);
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
 }
 
 var radio = document.getElementsByName('diff_type');

backend/templates/edit.html

@@ -1,73 +1,80 @@
 {% extends 'base.html' %}
-
 {% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
+{% from '_helpers.jinja' import render_field %}
+<div class="edit-form monospaced-textarea">
+    <form class="pure-form pure-form-stacked" action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="url">URL</label>
-                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
-                       size="50"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
             </div>
             <div class="pure-control-group">
-                <label for="tag">Tag</label>
-                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
+                {{ render_field(form.title, class="m-d") }}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.tag) }}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.minutes_between_check) }}
+                {% if using_default_minutes %}
+                    <span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                {% else %}
+                    <span class="pure-form-message-inline">Set to blank to use the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                {% endif %}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", class="m-d") }}
+                <span class="pure-form-message-inline">
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
+                </span>
+            </div>
+            <!-- @todo: move to tabs --->
+            <fieldset class="pure-group">
+                {{ render_field(form.ignore_text, rows=5,  placeholder="Some text to ignore in a line
+/some.regex\d{2}/   for case-INsensitive regex
+") }}
+                <span class="pure-form-message-inline">
+                    Each line processed separately, any line matching will be ignored.<br/>
+                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                </span>
+
+            </fieldset>
+
+            <fieldset class="pure-group">
+                {{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+            </fieldset>
+
+            <div class="pure-control-group">
+                {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+") }}
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+                <span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
             </div>
 
-            <!-- @todo: move to tabs --->
-            <fieldset class="pure-group">
-                <label for="ignore-text">Ignore text</label>
-
-                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
-{% endfor %}</textarea>
-                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
-
-            </fieldset>
-
-            <!-- @todo: move to tabs --->
-            <fieldset class="pure-group">
-                <label for="headers">Extra request headers</label>
-
-                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0"
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
-{% endfor %}</textarea>
-                <br/>
-
-            </fieldset>
-
-
+            <div class="pure-controls">
+                {{ render_field(form.trigger_check, rows=5) }}
+            </div>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
-
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-                <a href="/api/delete?uuid={{uuid}}"
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('api_delete', uuid=uuid)}}"
                    class="pure-button button-small button-error ">Delete</a>
-
             </div>
-
-
         </fieldset>
     </form>
 
-
 </div>
 
 {% endblock %}

backend/templates/import.html

@@ -4,7 +4,7 @@
 <div class="edit-form">
 
 
-    <form class="pure-form pure-form-aligned" action="/import" method="POST">
+    <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
 
         <fieldset class="pure-group">
             <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>

backend/templates/login.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
@@ -11,7 +11,7 @@
                 <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
             </div>
             <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Submit</button>
+                <button type="submit" class="pure-button pure-button-primary">Login</button>
             </div>
         </fieldset>
     </form>

backend/templates/scrub.html

@@ -2,47 +2,34 @@
 
 {% block content %}
 <div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
                 You may like to use the <strong>BACKUP</strong> link first.<br/>
-
-                Type in the word <strong>scrub</strong> to confirm that you understand!
-                <br/>
-
             </div>
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Confirm text</label><br/>
+                <label for="confirmtext">Confirmation text</label>
                 <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
             </div>
-
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Limit delete history including and after date</label><br/>
-                <input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
-                <br/>
-
-
+                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
+                <input type="datetime-local" id="limit_date" name="limit_date"  />
+                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
             </div>
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>
             <br/>
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
             </div>
-
-
         </fieldset>
     </form>
-
-
 </div>
 
 {% endblock %}

backend/templates/settings.html

@@ -1,38 +1,104 @@
 {% extends 'base.html' %}
 
 {% block content %}
+{% from '_helpers.jinja' import render_field %}
+<script type="text/javascript" src="static/js/settings.js"></script>
 <div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
+    <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                {{ render_field(form.minutes_between_check) }}
+                <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
             </div>
-            <br/>
             <div class="pure-control-group">
-                <label for="minutes">Password protection</label>
-                <input type="password" id="password" name="password" size="15"/>
                 {% if current_user.is_authenticated %}
-                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
+                    <a href="{{url_for('settings_page', removepassword='yes')}}" class="pure-button pure-button-primary">Remove password</a>
+                {% else %}
+                {{ render_field(form.password) }}
+                <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                 {% endif %}
             </div>
+            <div class="pure-control-group">
+                {{ render_field(form.extract_title_as_title) }}
+                <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+            </div>
 
-            <br/>
+            <div class="field-group">
+
+                <div class="pure-control-group">
+                    {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com") }}
+                    <div class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!
+                    <a id="toggle-customise-notifications">Customise notification body: <i
+                            class="arrow down"></i></a>
+                        </div>
+                </div>
+                <div id="notification-customisation" style="display:none;">
+
+                    <div class="pure-control-group">
+                        {{ render_field(form.notification_title, class="m-d") }}
+                        <span class="pure-form-message-inline">Title for all notifications</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.notification_body , rows=5) }}
+                        <span class="pure-form-message-inline">Body for all notifications</span>
+                    </div>
+                    <div class="pure-controls">
+                        <span class="pure-form-message-inline">
+                            These tokens can be used in the notification body and title to
+                            customise the notification text.
+                        </span>
+                        <table class="pure-table" id="token-table">
+                            <thead>
+                            <tr>
+                                <th>Token</th>
+                                <th>Description</th>
+                            </tr>
+                            </thead>
+                            <tbody>
+                            <tr>
+                                <td><code>{base_url}</code></td>
+                                <td>The URL of the changedetection.io instance you are running.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{watch_url}</code></td>
+                                <td>The URL being watched.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{preview_url}</code></td>
+                                <td>The URL of the preview page generated by changedetection.io.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{diff_url}</code></td>
+                                <td>The URL of the diff page generated by changedetection.io.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{current_snapshot}</code></td>
+                                <td>The current snapshot value, useful when combined with JSON or CSS filters</td>
+                            </tr>
+                            </tbody>
+                        </table>
+                        <span class="pure-form-message-inline">
+                            URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                            Your <code>BASE_URL</code> var is currently "{{base_url}}"
+                        </span>
+                    </div>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.trigger_check) }}
+                </div>
+            </div>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
-
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
             </div>
-
-
         </fieldset>
     </form>
 

backend/templates/watch-overview.html

@@ -1,24 +1,24 @@
 {% extends 'base.html' %}
-
 {% block content %}
+{% from '_helpers.jinja' import render_simple_field %}
 
 <div class="box">
 
-    <form class="pure-form" action="/api/add" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <fieldset>
             <legend>Add a new change detection watch</legend>
-            <input type="url" placeholder="https://..." name="url"/>
-            <input type="text" placeholder="tag" size="10" name="tag" value="{{active_tag if active_tag}}"/>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
         <!-- add extra stuff, like do a http POST and send headers -->
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
             {% if tag != "" %}
-                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -44,9 +44,10 @@
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause"/></a></td>
+
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
@@ -55,22 +56,22 @@
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
                 </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{% if watch.history|length >= 2 and watch.last_changed %}
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
                     Not yet
                     {% endif %}
                 </td>
                 <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                     {% else %}
                         {% if watch.history|length == 1 %}
-                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -81,15 +82,15 @@
         <ul id="post-list-buttons">
             {% if has_unviewed %}
             <li>
-                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15px"></a>
             </li>
         </ul>
     </div>

backend/tests/conftest.py

@@ -12,6 +12,21 @@ import os
 
 global app
 
+
+def cleanup(datastore_path):
+    # Unlink test output files
+    files = ['output.txt',
+             'url-watches.json',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt']
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+            x = 1
+        except FileNotFoundError:
+            pass
+
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
@@ -22,10 +37,10 @@ def app(request):
     except FileExistsError:
         pass
 
-    try:
-        os.unlink("{}/url-watches.json".format(datastore_path))
-    except FileNotFoundError:
-        pass
+    # Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
+    os.environ["BASE_URL"] = "http://mysite.com/"
+
+    cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
@@ -35,14 +50,8 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config.exit.set()
-        try:
-            os.unlink("{}/url-watches.json".format(datastore_path))
-        except FileNotFoundError:
-            # This is fine in the case of a failure.
-            pass
-
-        assert 1 == 1
-
+        cleanup(datastore_path)
+        
     request.addfinalizer(teardown)
     yield app
 

backend/tests/test_access_control.py

@@ -0,0 +1,102 @@
+from flask import url_for
+
+
+def test_check_access_control(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "foobar", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+
+        assert b"Login" in res.data
+
+        # Menu should not be available yet
+        #        assert b"SETTINGS" not in res.data
+        #        assert b"BACKUP" not in res.data
+        #        assert b"IMPORT" not in res.data
+
+        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
+        res = c.post(
+            url_for("login"),
+            data={"password": "foobar"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" in res.data
+        res = c.get(url_for("settings_page"))
+
+        # Menu should be available now
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+        assert b"LOG OUT" in res.data
+
+        # Now remove the password so other tests function, @todo this should happen before each test automatically
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
+        assert b"Password protection removed." in res.data
+
+        res = c.get(url_for("index"))
+        assert b"LOG OUT" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_control_no_blank_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." not in res.data
+        assert b"Login" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_no_remote_access_to_remove_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "password", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"Login" in res.data
+
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
+        assert b"Password protection removed." not in res.data
+
+        res = c.get(url_for("index"),
+              follow_redirects=True)
+        assert b"watch-table-wrapper" not in res.data

backend/tests/test_backend.py

@@ -3,55 +3,14 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-import pytest
+from . util import set_original_response, set_modified_response, live_server_setup
 
 sleep_time_for_fetch_thread = 3
 
 
-def test_setup_liveserver(live_server):
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
-    live_server.start()
-
-    assert 1 == 1
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
+    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -75,6 +34,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
+        # Default no password set, this stuff should be always available.
+
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
     #####################
 
     # Make a change
@@ -93,6 +58,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # #75, and it should be in the RSS feed
+    res = client.get(url_for("index", rss="true"))
+    expected_url = url_for('test_endpoint', _external=True)
+    assert b'<rss' in res.data
+    assert expected_url.encode('utf-8') in res.data
+
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
     res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
@@ -109,52 +80,28 @@ def test_check_basic_change_detection_functionality(client, live_server):
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
         assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
         assert b'test-endpoint' in res.data
 
     set_original_response()
 
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"extract_title_as_title": "1", "minutes_between_check": 180},
+        follow_redirects=True
+    )
+
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
 
+    #
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-
-def test_check_access_control(client):
-    return
-    # @note: does not seem to handle the last logout step correctly, we're still logged in.. but yet..
-    #        pytest team keep telling us that we have a new context.. i'm lost :(
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("settings_page"),
-        data={"password": "foobar"},
-        follow_redirects=True
-    )
-    assert b"LOG OUT" not in res.data
-
-    client.get(url_for("import_page"), follow_redirects=True)
-    assert b"Password" in res.data
-
-    #defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
-    res = client.post(
-        url_for("login"),
-        data={"password": "foobar", "email": "defaultuser@changedetection.io"},
-        follow_redirects=True
-    )
-
-    assert b"LOG OUT" in res.data
-
-    client.get(url_for("settings_page"), follow_redirects=True)
-    assert b"LOG OUT" in res.data
-
-    # Now remove the password so other tests function, @todo this should happen before each test automatically
-
-    print(res.data)
-    client.get(url_for("settings_page", removepassword="true"), follow_redirects=True)
-
-    client.get(url_for("import_page", removepassword="true"), follow_redirects=True)
-    assert b"LOG OUT" not in res.data

backend/tests/test_css_selector.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from backend import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
+def test_check_markup_css_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    css_filter = "#sametext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

backend/tests/test_headers.py

@@ -0,0 +1,79 @@
+import json
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_headers_in_request(client, live_server):
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request headers
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Flask will convert the header key to uppercase
+    assert b"Xxx:ooo" in res.data
+    assert b"Cool:yeah" in res.data
+
+    # The test call service will return the headers as the body
+    from html import escape
+    assert escape(cookie_header).encode('utf-8') in res.data
+
+    time.sleep(5)
+
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    watches_with_headers = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
+                watches_with_headers += 1
+
+    # Should be only one with headers set
+    assert watches_with_headers==1
+
+
+
+
+
+
+

backend/tests/test_ignore_regex_text.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_regex_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    but sometimes we want to remove the lines.
+    
+    but 1 lines
+    but including 1234 lines
+    igNORe-cAse text we dont want to keep    
+    but not always."""
+
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"but 1 lines" in stripped_content
+    assert b"igNORe-cAse text" not in stripped_content
+    assert b"but 1234 lines" not in stripped_content
+

backend/tests/test_ignore_text.py

@@ -2,9 +2,10 @@
 
 import time
 from flask import url_for
-from urllib.request import urlopen
-import pytest
+from . util import live_server_setup
 
+def test_setup(live_server):
+    live_server_setup(live_server)
 
 # Unit test of the stripper
 # Always we are dealing in utf-8
@@ -81,7 +82,7 @@ def set_modified_ignore_response():
 def test_check_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
     set_original_ignore_response()
 
     # Give the endpoint time to spin up
@@ -106,7 +107,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        data={"ignore_text": ignore_text, "url": test_url},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

backend/tests/test_jsonpath_selector.py

@@ -0,0 +1,160 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+import pytest
+
+def test_unittest_inline_html_extract():
+    # So lets pretend that the JSON we want is inside some HTML
+    content="""
+    <html>
+    
+    food and stuff and more
+    <script>
+    alert('nothing really good here');
+    </script>
+    
+    <script type="application/ld+json">
+  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<body>
+and it can also be repeated
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<h4>ok</h4>
+</body>
+</html>
+
+    """
+    from .. import html_tools
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(content, "$.offers.price")
+    assert text == "23.5"
+
+    text = html_tools.extract_json_as_string('{"id":5}', "$.id")
+    assert text == "5"
+
+    # When nothing at all is found, it should throw JSONNOTFound
+    # Which is caught and shown to the user in the watch-overview table
+    with pytest.raises(html_tools.JSONNotFound) as e_info:
+        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Fat guy"
+      }
+    }
+    """
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Foobar"
+      }
+    }
+        """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+
+def test_check_json_filter(client, live_server):
+
+    json_filter = 'json:boss.name'
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # It should have 'unviewed' still
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Should not see this, because its not in the JSONPath we entered
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'Foobar' in res.data

backend/tests/test_notification.py

@@ -0,0 +1,130 @@
+import os
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_check_notification(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    url = url_for('test_notification_endpoint', _external=True)
+    notification_url = url.replace('http', 'json')
+
+    print (">>>> Notification URL: "+notification_url)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "trigger_check": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    assert b"Notifications queued" in res.data
+
+    # Hit the edit page, be sure that we saved it
+    res = client.get(
+        url_for("edit_page", uuid="first"))
+    assert bytes(notification_url.encode('utf-8')) in res.data
+
+
+    # Because we hit 'send test notification on save'
+    time.sleep(3)
+
+    notification_submission = None
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+    os.unlink("test-datastore/notification.txt")
+
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    notification_submission=None
+    # Verify what was sent as a notification
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+        # Re #65 - did we see our foobar.com BASE_URL ?
+        #assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
+
+
+    ##  Now configure something clever, we go into custom config (non-default) mode
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "{base_url}\n{watch_url}\n{preview_url}\n{diff_url}\n{current_snapshot}\n:-)",
+              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
+              "minutes_between_check": 180},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+    # Re #143 - should not see this if we didnt hit the test box
+    assert b"Notifications queued" not in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+
+        assert "diff/" in notification_submission
+        assert "preview/" in notification_submission
+        assert ":-)" in notification_submission
+        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
+        # This should insert the {current_snapshot}
+        assert "stuff we will detect" in notification_submission

backend/tests/test_watch_fields_storage.py

@@ -0,0 +1,145 @@
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_check_watch_field_storage(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={ "notification_urls": "http://myapi.com",
+               "minutes_between_check": 126,
+               "css_filter" : ".fooclass",
+               "title" : "My title",
+               "ignore_text" : "ignore this",
+               "url": test_url,
+               "tag": "woohoo",
+               "headers": "curl:foo",
+
+               },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"http://myapi.com" in res.data
+    assert b"126" in res.data
+    assert b".fooclass" in res.data
+    assert b"My title" in res.data
+    assert b"ignore this" in res.data
+    assert b"http://somerandomsitewewatch.com" in res.data
+    assert b"woohoo" in res.data
+    assert b"curl: foo" in res.data
+
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 1566,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 222,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": 55,
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 666,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": "",
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data
+

backend/tests/util.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <head><title>head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def live_server_setup(live_server):
+
+
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    # Just return the headers in the request
+    @live_server.app.route('/test-headers')
+    def test_headers():
+
+        from flask import request
+        output= []
+
+        for header in request.headers:
+             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
+
+        return "\n".join(output)
+
+
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
+    def test_notification_endpoint():
+        from flask import request
+
+        with open("test-datastore/notification.txt", "wb") as f:
+            # Debug method, dump all POST to file also, used to prove #65
+            data = request.stream.read()
+            if data != None:
+                f.write(data)
+
+        print("\n>> Test notification endpoint was hit.\n")
+        return "Text was set"
+
+    live_server.start()

backend/update_worker.py

@@ -0,0 +1,83 @@
+import threading
+import queue
+
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+class update_worker(threading.Thread):
+    current_uuid = None
+
+    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        self.q = q
+        self.app = app
+        self.notification_q = notification_q
+        self.datastore = datastore
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from backend import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
+
+        while not self.app.config.exit.is_set():
+
+            try:
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
+
+            else:
+                self.current_uuid = uuid
+
+                if uuid in list(self.datastore.data['watching'].keys()):
+                    try:
+                        changed_detected, result, contents = update_handler.run(uuid)
+
+                    except PermissionError as e:
+                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except Exception as e:
+                        self.app.logger.error("Exception reached", uuid, str(e))
+                    else:
+                        if result:
+                            try:
+                                self.datastore.update_watch(uuid=uuid, update_obj=result)
+                                if changed_detected:
+
+                                    # A change was detected
+                                    newest_version_file_contents = ""
+                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
+                                    watch = self.datastore.data['watching'][uuid]
+
+                                    print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+
+                                    # Get the newest snapshot data to be possibily used in a notification
+                                    newest_key = self.datastore.get_newest_history_key(uuid)
+                                    if newest_key:
+                                        with open(watch['history'][newest_key], 'r') as f:
+                                            newest_version_file_contents = f.read().strip()
+
+                                    n_object = {
+                                        'watch_url': watch['url'],
+                                        'uuid': uuid,
+                                        'current_snapshot': newest_version_file_contents
+                                    }
+
+                                    # Did it have any notification alerts to hit?
+                                    if len(watch['notification_urls']):
+                                        print(">>> Notifications queued for UUID from watch {}".format(uuid))
+                                        n_object['notification_urls'] = watch['notification_urls']
+                                        self.notification_q.put(n_object)
+
+                                    # No? maybe theres a global setting, queue them all
+                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                        print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+                                        self.notification_q.put(n_object)
+                                    else:
+                                        print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
+                            except Exception as e:
+                                print("!!!! Exception in update_worker !!!\n", e)
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            self.app.config.exit.wait(1)

changedetection.py

@@ -12,35 +12,16 @@ import backend
 
 from backend import store
 
-
-def init_app_secret(datastore_path):
-    secret = ""
-
-    path = "{}/secret.txt".format(datastore_path)
-
-    try:
-        with open(path, "r") as f:
-            secret = f.read()
-
-    except FileNotFoundError:
-
-        import secrets
-        with open(path, "w") as f:
-            secret = secrets.token_hex(32)
-            f.write(secret)
-
-    return secret
-
-
 def main(argv):
     ssl_mode = False
-    port = 5000
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
 
     # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
     datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
-        opts, args = getopt.getopt(argv, "sd:p:", "purge")
+        opts, args = getopt.getopt(argv, "csd:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port] -d [datastore path]')
         sys.exit(2)
@@ -60,21 +41,41 @@ def main(argv):
         if opt == '-d':
             datastore_path = arg
 
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
     app = backend.changedetection_app(app_config, datastore)
 
-    app.secret_key = init_app_secret(app_config['datastore_path'])
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
 
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['version_tag'])
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
 
-    @app.context_processor
-    def inject_new_version_available():
-        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
@@ -84,7 +85,7 @@ def main(argv):
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
+        eventlet.wsgi.server(eventlet.listen(('', int(port))), app)
 
 
 if __name__ == '__main__':

docker-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+services:
+    changedetection.io:
+      image: dgtlmoon/changedetection.io
+      container_name: changedetection.io
+      hostname: changedetection.io
+      volumes:
+        - changedetection-data:/datastore
+  #    environment:
+  #        Default listening port, can also be changed with the -p option
+  #      - PORT=5000
+
+  #      - PUID=1000
+  #      - PGID=1000
+  #        Proxy support example.
+  #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
+  #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
+  #        An exclude list (useful for notification URLs above) can be specified by with
+  #      - NO_PROXY="localhost,192.168.0.0/24"
+  #        Base URL of your changedetection.io install (Added to the notification alert)
+  #      - BASE_URL="https://mysite.com"
+
+  #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
+  #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
+  #      - USE_X_SETTINGS=1
+
+      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
+      ports:
+        - 5000:5000
+
+      restart: always
+
+volumes:
+  changedetection-data:
+

heroku.yml

@@ -0,0 +1,5 @@
+build:
+  docker:
+    changedetection: Dockerfile
+run:
+  changedetection: python ./changedetection.py -d /datastore -p $PORT

requirements.txt

@@ -1,13 +1,23 @@
 chardet==2.3.0
 flask~= 1.0
 pytest ~=6.2
-pytest-flask ~=1.1
-eventlet ~= 0.30
-requests ~= 2.15
+pytest-flask ~=1.2
+eventlet>=0.31.0
+requests[socks] ~= 2.15
 validators
 timeago ~=1.0
-inscriptis ~= 1.1
+inscriptis ~= 1.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
-urllib3
+urllib3
+wtforms ~= 2.3.3
+jsonpath-ng ~= 1.5.3
+
+
+# Notification library
+apprise ~= 0.9
+
+# Used for CSS filtering, replace with soupsieve and lxml for xpath
+bs4
+