Merge branch 'master' of github.com:dgtlmoon/changedetection.io

* Re #9 CSS Selector filtering,  Adding test for #9

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.github/FUNDING.yml

@@ -1,12 +1,3 @@
 # These are supported funding model platforms
 
 github: dgtlmoon
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,62 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  schedule:
+    - cron: '27 9 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/image.yml

@@ -0,0 +1,88 @@
+name: Test, build and push to Docker Hub
+
+on:
+  push:
+    branches: [ master, arm-build ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > backend/source.txt
+          echo ${{ github.ref }} > backend/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+#          platforms: linux/amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+# failed: Cache service responded with 503
+#      - name: Cache Docker layers
+#        uses: actions/cache@v2
+#        with:
+#          path: /tmp/.buildx-cache
+#          key: ${{ runner.os }}-buildx-${{ github.sha }}
+#          restore-keys: |
+#            ${{ runner.os }}-buildx-
+
+

.github/workflows/python-app.yml

@@ -1,37 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: changedetection.io
-
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        cd backend; pytest

.github/workflows/test-only.yml

@@ -0,0 +1,33 @@
+name: ChangeDetection.io Test
+
+# Triggers the workflow on push or pull request events
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+

Dockerfile

@@ -1,7 +1,13 @@
 FROM python:3.8-slim
-COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
 
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN apt-get update && apt-get install -y libssl-dev libffi-dev gcc libc-dev libxslt-dev zlib1g-dev rustc g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+# Update pip, install requirements, remove rust and dev packages that are no longer needed.
+RUN pip3 install --upgrade pip && pip3 install --no-cache-dir -r /tmp/requirements.txt && apt-get remove rustc *-dev --purge -y
 
 RUN [ ! -d "/app" ] && mkdir /app
 RUN [ ! -d "/datastore" ] && mkdir /datastore
@@ -17,11 +23,6 @@ WORKDIR /app
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 
-# Attempt to store the triggered commit
-ARG SOURCE_COMMIT
-ARG SOURCE_BRANCH
-RUN echo "commit: $SOURCE_COMMIT branch: $SOURCE_BRANCH" >/source.txt
-
 CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
 
 

README.md

@@ -1,13 +1,20 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/python-app.yml/badge.svg?branch=master)
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Docker Pulls">
-<img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
 </a>
 
 ## Self-hosted change monitoring of web pages.
 
-_Know when web pages change! Stay ontop of new information!_
+_Know when web pages change! Stay ontop of new information!_ 
 
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
 #### Example use cases
 
@@ -18,6 +25,7 @@ Know when ...
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
+- COVID related news from government websites
 
 
 **Get monitoring now! super simple, one command!**
@@ -41,22 +49,67 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
   
 ### Screenshots
 
-Application running.
-
-![Self-hosted web page change monitoring application screenshot](screenshot.png?raw=true "Self-hosted web page change monitoring screenshot")
-
 Examining differences in content.
 
-![Self-hosted web page change monitoring context difference screenshot](screenshot-diff.png?raw=true "Self-hosted web page change monitoring context difference screenshot")
 
-### Future plans
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
-- Greater configuration of check interval times, page request headers.
-- ~~General options for timeout, default headers~~
-- On change detection, callout to another API (handy for notices/issue trackers)
-- ~~Explore the differences that were detected~~ 
-- Add more options to explore versions of differences
-- Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
-
- 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
+### Notifications
+
+ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
+Simply set one or more notification URL's in the _[edit]_ tab of that watch.
+
+Just some examples
+
+    discord://webhook_id/webhook_token
+    flock://app_token/g:channel_id
+    gitter://token/room
+    gchat://workspace/key/token
+    msteams://TokenA/TokenB/TokenC/
+    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    rocket://user:password@hostname/#Channel
+    mailto://user:pass@example.com?to=receivingAddress@example.com
+    json://someserver.com/custom-api
+    syslog://
+ 
+<a href="https://github.com/caronc/apprise">And everything else in this list!</a>
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
+
+### Proxy
+
+A proxy for ChangeDectection.io can be configured by setting environment the 
+`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
+
+`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
+
+as `docker run` with `-e`
+
+```
+docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+With `docker-compose`, see the `Proxy support example` in <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a>.
+
+For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies
+
+This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
+
+### Notes
+
+- Does not yet support Javascript
+- Wont work with Cloudfare type "Please turn on javascript" protected pages
+- You can use the 'headers' section to monitor password protected web page changes
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

backend/README-pytest.md

@@ -1 +0,0 @@
-Note: run `pytest` from this directory.

backend/__init__.py

@@ -2,10 +2,8 @@
 
 
 # @todo logging
-# @todo sort by last_changed
 # @todo extra options for url like , verify=False etc.
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
 # @todo make tables responsive!
@@ -17,11 +15,20 @@
 import time
 import os
 import timeago
+import flask_login
+from flask_login import login_required
 
 import threading
+from threading import Event
+
 import queue
 
-from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
+from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for
+
+from feedgen.feed import FeedGenerator
+from flask import make_response
+import datetime
+import pytz
 
 datastore = None
 
@@ -34,12 +41,18 @@ extra_stylesheets = []
 
 update_q = queue.Queue()
 
-app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
+notification_q = queue.Queue()
+
+app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
-app.config['STOP_THREADS'] = False
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
+
+app.config['LOGIN_DISABLED'] = False
 
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
@@ -73,26 +86,129 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
 
-def changedetection_app(config=None, datastore_o=None):
+class User(flask_login.UserMixin):
+    id=None
+
+    def set_password(self, password):
+        return True
+    def get_user(self, email="defaultuser@changedetection.io"):
+        return self
+    def is_authenticated(self):
+
+        return True
+    def is_active(self):
+        return True
+    def is_anonymous(self):
+        return False
+    def get_id(self):
+        return str(self.id)
+
+    def check_password(self, password):
+
+        import hashlib
+        import base64
+
+        # Getting the values back out
+        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
+        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
+
+        # Use the exact same setup you used to generate the key, but this time put in the password to check
+        new_key = hashlib.pbkdf2_hmac(
+            'sha256',
+            password.encode('utf-8'),  # Convert the password to bytes
+            salt_from_storage,
+            100000
+        )
+        new_key =  salt_from_storage + new_key
+
+        return new_key == raw_salt_pass
+
+    pass
+
+def changedetection_app(conig=None, datastore_o=None):
     global datastore
     datastore = datastore_o
-    # Hmm
+
     app.config.update(dict(DEBUG=True))
-    app.config.update(config or {})
+    #app.config.update(config or {})
+
+    login_manager = flask_login.LoginManager(app)
+    login_manager.login_view = 'login'
+
 
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
 
+    @login_manager.user_loader
+    def user_loader(email):
+        user = User()
+        user.get_user(email)
+        return user
+
+    @login_manager.unauthorized_handler
+    def unauthorized_handler():
+        # @todo validate its a URL of this host and use that
+        return redirect(url_for('login', next=url_for('index')))
+
+    @app.route('/logout')
+    def logout():
+        flask_login.logout_user()
+        return redirect(url_for('index'))
+
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
+    @app.route('/login', methods=['GET', 'POST'])
+    def login():
 
-    @app.route("/", methods=['GET'])
-    def index():
         global messages
 
+        if request.method == 'GET':
+            output = render_template("login.html", messages=messages)
+            # Show messages but once.
+            messages = []
+            return output
+
+        user = User()
+        user.id = "defaultuser@changedetection.io"
+
+        password = request.form.get('password')
+
+        if (user.check_password(password)):
+            flask_login.login_user(user, remember=True)
+            next = request.args.get('next')
+            #            if not is_safe_url(next):
+            #                return flask.abort(400)
+            return redirect(next or url_for('index'))
+
+        else:
+            messages.append({'class': 'error', 'message': 'Incorrect password'})
+
+        return redirect(url_for('login'))
+
+    @app.before_request
+    def do_something_whenever_a_request_comes_in():
+        # Disable password  loginif there is not one set
+        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
+
+    @app.route("/", methods=['GET'])
+    @login_required
+    def index():
+        global messages
         limit_tag = request.args.get('tag')
 
+        pause_uuid = request.args.get('pause')
+
+        if pause_uuid:
+            try:
+                datastore.data['watching'][pause_uuid]['paused'] ^= True
+                datastore.needs_write = True
+
+                return redirect(url_for('index', tag = limit_tag))
+            except KeyError:
+                pass
+
+
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
         for uuid, watch in datastore.data['watching'].items():
@@ -112,17 +228,44 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
 
         existing_tags = datastore.get_all_tags()
-        output = render_template("watch-overview.html",
-                                 watches=sorted_watches,
-                                 messages=messages,
-                                 tags=existing_tags,
-                                 active_tag=limit_tag)
+        rss = request.args.get('rss')
+
+        if rss:
+            fg = FeedGenerator()
+            fg.title('changedetection.io')
+            fg.description('Feed description')
+            fg.link(href='https://changedetection.io')
+
+            for watch in sorted_watches:
+                if not watch['viewed']:
+                    fe = fg.add_entry()
+                    fe.title(watch['url'])
+                    fe.link(href=watch['url'])
+                    fe.description(watch['url'])
+                    fe.guid(watch['uuid'], permalink=False)
+                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
+                    dt = dt.replace(tzinfo=pytz.UTC)
+                    fe.pubDate(dt)
+
+            response = make_response(fg.rss_str())
+            response.headers.set('Content-Type', 'application/rss+xml')
+            return response
+
+        else:
+            output = render_template("watch-overview.html",
+                                     watches=sorted_watches,
+                                     messages=messages,
+                                     tags=existing_tags,
+                                     active_tag=limit_tag,
+                                     has_unviewed=datastore.data['has_unviewed'])
+
+            # Show messages but once.
+            messages = []
 
-        # Show messages but once.
-        messages = []
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
+    @login_required
     def scrub_page():
         from pathlib import Path
 
@@ -130,19 +273,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
+            limit_timestamp = int(request.form.get('limit_date'))
 
             if confirmtext == 'scrub':
 
-                for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
-                    os.unlink(txt_file_path)
-
                 for uuid, watch in datastore.data['watching'].items():
-                    watch['last_checked'] = 0
-                    watch['last_changed'] = 0
-                    watch['previous_md5'] = None
-                    watch['history'] = {}
+                    if len(str(limit_timestamp)) == 10:
+                        datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
+                    else:
+                        datastore.scrub_watch(uuid)
 
-                datastore.needs_write = True
                 messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
             else:
                 messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
@@ -151,68 +291,229 @@ def changedetection_app(config=None, datastore_o=None):
 
         return render_template("scrub.html")
 
-    @app.route("/edit", methods=['GET', 'POST'])
-    def edit_page():
+    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+    # the excluded text.
+    def get_current_checksum_include_ignore_text(uuid):
+
+        import hashlib
+        from backend import fetch_site_status
+
+        # Get the most recent one
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+
+        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
+        if newest_history_key == 0:
+            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
+
+        if newest_history_key:
+            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
+                      encoding='utf-8') as file:
+                raw_content = file.read()
+
+                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                stripped_content = handler.strip_ignore_text(raw_content,
+                                                             datastore.data['watching'][uuid]['ignore_text'])
+
+                checksum = hashlib.md5(stripped_content).hexdigest()
+                return checksum
+
+        return datastore.data['watching'][uuid]['previous_md5']
+
+    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    @login_required
+    def edit_page(uuid):
         global messages
         import validators
 
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
         if request.method == 'POST':
-            uuid = request.args.get('uuid')
 
             url = request.form.get('url').strip()
             tag = request.form.get('tag').strip()
 
+            minutes_recheck = request.form.get('minutes')
+            if minutes_recheck:
+                minutes = int(minutes_recheck.strip())
+                if minutes >= 1:
+                    datastore.data['watching'][uuid]['minutes_between_check'] = minutes
+                else:
+                    messages.append(
+                        {'class': 'error', 'message': "Must be atleast 1 minute."})
+
+
+
+            # Extra headers
             form_headers = request.form.get('headers').strip().split("\n")
             extra_headers = {}
             if form_headers:
                 for header in form_headers:
                     if len(header):
                         parts = header.split(':', 1)
-                        extra_headers.update({parts[0].strip(): parts[1].strip()})
+                        if len(parts) == 2:
+                            extra_headers.update({parts[0].strip(): parts[1].strip()})
+
+            update_obj = {'url': url,
+                          'tag': tag,
+                          'headers': extra_headers
+                          }
+
+            # Notification URLs
+            form_notification_text = request.form.get('notification_urls')
+            notification_urls = []
+            if form_notification_text:
+                for text in form_notification_text.strip().split("\n"):
+                    text = text.strip()
+                    if len(text):
+                        notification_urls.append(text)
+
+            datastore.data['watching'][uuid]['notification_urls'] = notification_urls
+
+            # Ignore text
+            form_ignore_text = request.form.get('ignore-text')
+            ignore_text = []
+            if form_ignore_text:
+                for text in form_ignore_text.strip().split("\n"):
+                    text = text.strip()
+                    if len(text):
+                        ignore_text.append(text)
+
+                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
+
+                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
+
+            # CSS Filter
+            css_filter = request.form.get('css_filter')
+            if css_filter:
+                datastore.data['watching'][uuid]['css_filter'] = css_filter.strip()
+
+                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
 
             validators.url(url)  # @todo switch to prop/attr/observer
-            datastore.data['watching'][uuid].update({'url': url,
-                                                     'tag': tag,
-                                                     'headers': extra_headers})
+            datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
 
             messages.append({'class': 'ok', 'message': 'Updated watch.'})
 
+            trigger_n = request.form.get('trigger-test-notification')
+            if trigger_n:
+                n_object = {'watch_url': url,
+                            'notification_urls': notification_urls}
+                notification_q.put(n_object)
+
+                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
+
             return redirect(url_for('index'))
 
         else:
-
-            uuid = request.args.get('uuid')
             output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
 
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
+    @login_required
     def settings_page():
         global messages
+
+        if request.method == 'GET':
+            if request.values.get('notification-test'):
+                url_count = len(datastore.data['settings']['application']['notification_urls'])
+                if url_count:
+                    import apprise
+                    apobj = apprise.Apprise()
+                    apobj.debug = True
+
+                    # Add each notification
+                    for n in datastore.data['settings']['application']['notification_urls']:
+                        apobj.add(n)
+                    outcome = apobj.notify(
+                        body='Hello from the worlds best and simplest web page change detection and monitoring service!',
+                        title='Changedetection.io Notification Test',
+                    )
+
+                    if outcome:
+                        messages.append(
+                            {'class': 'notice', 'message': "{} Notification URLs reached.".format(url_count)})
+                    else:
+                        messages.append(
+                            {'class': 'error', 'message': "One or more Notification URLs failed"})
+
+                return redirect(url_for('settings_page'))
+
+            if request.values.get('removepassword'):
+                from pathlib import Path
+
+                datastore.data['settings']['application']['password'] = False
+                messages.append({'class': 'notice', 'message': "Password protection removed."})
+                flask_login.logout_user()
+
+                return redirect(url_for('settings_page'))
+
         if request.method == 'POST':
+
+            password = request.values.get('password')
+            if password:
+                import hashlib
+                import base64
+                import secrets
+
+                # Make a new salt on every new password and store it with the password
+                salt = secrets.token_bytes(32)
+
+                key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+                store = base64.b64encode(salt + key).decode('ascii')
+                datastore.data['settings']['application']['password'] = store
+                messages.append({'class': 'notice', 'message': "Password protection enabled."})
+                flask_login.logout_user()
+                return redirect(url_for('index'))
+
             try:
                 minutes = int(request.values.get('minutes').strip())
             except ValueError:
                 messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
 
             else:
-                if minutes >= 5:
+                if minutes >= 1:
                     datastore.data['settings']['requests']['minutes_between_check'] = minutes
                     datastore.needs_write = True
-
-                    messages.append({'class': 'ok', 'message': "Updated"})
                 else:
                     messages.append(
-                        {'class': 'error', 'message': "Must be atleast 5 minutes."})
+                        {'class': 'error', 'message': "Must be atleast 1 minute."})
+
+            # 'validators' package doesnt work because its often a non-stanadard protocol. :(
+            datastore.data['settings']['application']['notification_urls'] = []
+            trigger_n = request.form.get('trigger-test-notification')
+
+            for n in request.values.get('notification_urls').strip().split("\n"):
+                url = n.strip()
+                datastore.data['settings']['application']['notification_urls'].append(url)
+                datastore.needs_write = True
+
+            if trigger_n:
+                n_object = {'watch_url': "Test from changedetection.io!",
+                            'notification_urls': datastore.data['settings']['application']['notification_urls']}
+                notification_q.put(n_object)
+
+                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
 
         output = render_template("settings.html", messages=messages,
-                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
+                                 minutes=datastore.data['settings']['requests']['minutes_between_check'],
+                                 notification_urls="\r\n".join(
+                                     datastore.data['settings']['application']['notification_urls']))
         messages = []
 
         return output
 
     @app.route("/import", methods=['GET', "POST"])
+    @login_required
     def import_page():
         import validators
         global messages
@@ -235,28 +536,46 @@ def changedetection_app(config=None, datastore_o=None):
 
             messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
 
-        if len(remaining_urls) == 0:
-            return redirect(url_for('index'))
-        else:
-            output = render_template("import.html",
-                                     messages=messages,
-                                     remaining="\n".join(remaining_urls)
-                                     )
-            messages = []
+            if len(remaining_urls) == 0:
+                # Looking good, redirect to index.
+                return redirect(url_for('index'))
+
+        # Could be some remaining, or we could be on GET
+        output = render_template("import.html",
+                                 messages=messages,
+                                 remaining="\n".join(remaining_urls)
+                                 )
+        messages = []
+
         return output
 
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @app.route("/api/mark-all-viewed", methods=['GET'])
+    @login_required
+    def mark_all_viewed():
+
+        # Save the current newest history as the most recently viewed
+        for watch_uuid, watch in datastore.data['watching'].items():
+            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
+
+        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
+        return redirect(url_for('index'))
+
     @app.route("/diff/<string:uuid>", methods=['GET'])
+    @login_required
     def diff_history_page(uuid):
         global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
-            uuid= list(datastore.data['watching'].keys()).pop()
-
+            uuid = list(datastore.data['watching'].keys()).pop()
 
         extra_stylesheets = ['/static/css/diff.css']
-
-        watch = datastore.data['watching'][uuid]
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            return redirect(url_for('index'))
 
         dates = list(watch['history'].keys())
         # Convert to int, sort and back to str again
@@ -264,9 +583,9 @@ def changedetection_app(config=None, datastore_o=None):
         dates.sort(reverse=True)
         dates = [str(i) for i in dates]
 
-
         if len(dates) < 2:
-            messages.append({'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
+            messages.append(
+                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
             return redirect(url_for('index'))
 
         # Save the current newest history as the most recently viewed
@@ -299,41 +618,90 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    @app.route("/preview/<string:uuid>", methods=['GET'])
+    @login_required
+    def preview_page(uuid):
+        global messages
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = ['/static/css/diff.css']
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
+            return redirect(url_for('index'))
+
+        print(watch)
+        with open(list(watch['history'].values())[-1], 'r') as f:
+            content = f.readlines()
+
+        output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets)
+        return output
+
+
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("/app/static/images", filename="favicon.ico")
 
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
+    @login_required
     def get_backup():
+
         import zipfile
         from pathlib import Path
 
+        # Remove any existing backup file, for now we just keep one file
+        for previous_backup_filename in Path(app.config['datastore_path']).rglob('changedetection-backup-*.zip'):
+            os.unlink(previous_backup_filename)
+
         # create a ZipFile object
         backupname = "changedetection-backup-{}.zip".format(int(time.time()))
 
         # We only care about UUIDS from the current index file
         uuids = list(datastore.data['watching'].keys())
+        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
 
-        with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
+        with zipfile.ZipFile(backup_filepath, "w",
                              compression=zipfile.ZIP_DEFLATED,
-                             compresslevel=6) as zipObj:
+                             compresslevel=8) as zipObj:
 
             # Be sure we're written fresh
             datastore.sync_to_json()
 
             # Add the index
-            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"))
-            # Add any snapshot data we find
+            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
+
+            # Add the flask app secret
+            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
+
+            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
             for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                 parent_p = txt_file_path.parent
                 if parent_p.name in uuids:
-                    zipObj.write(txt_file_path)
+                    zipObj.write(txt_file_path,
+                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
+                                 compress_type=zipfile.ZIP_DEFLATED,
+                                 compresslevel=8)
 
-        return send_file(os.path.join(app.config['datastore_path'], backupname),
-                         as_attachment=True,
-                         mimetype="application/zip",
-                         attachment_filename=backupname)
+            # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+            list_file = os.path.join(app.config['datastore_path'], "url-list.txt")
+            with open(list_file, "w") as f:
+                for uuid in datastore.data['watching']:
+                    url = datastore.data['watching'][uuid]['url']
+                    f.write("{}\r\n".format(url))
+
+            # Add it to the Zip
+            zipObj.write(list_file,
+                         arcname="url-list.txt",
+                         compress_type=zipfile.ZIP_DEFLATED,
+                         compresslevel=8)
+
+        return send_from_directory(app.config['datastore_path'], backupname, as_attachment=True)
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
@@ -347,11 +715,17 @@ def changedetection_app(config=None, datastore_o=None):
             abort(404)
 
     @app.route("/api/add", methods=['POST'])
+    @login_required
     def api_watch_add():
         global messages
 
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            messages.append({'class': 'error', 'message': 'The URL {} already exists'.format(url)})
+            return redirect(url_for('index'))
+
         # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
         # Straight into the queue.
         update_q.put(new_uuid)
 
@@ -359,6 +733,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
+    @login_required
     def api_delete():
         global messages
         uuid = request.args.get('uuid')
@@ -368,6 +743,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/checknow", methods=['GET'])
+    @login_required
     def api_watch_checknow():
 
         global messages
@@ -391,15 +767,17 @@ def changedetection_app(config=None, datastore_o=None):
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
                 if (tag != None and tag in watch['tag']):
-                    i += 1
-                    if watch_uuid not in running_uuids:
+                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                         update_q.put(watch_uuid)
+                        i += 1
+
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
-                i += 1
-                if watch_uuid not in running_uuids:
+
+                if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(watch_uuid)
+                    i += 1
 
         messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
         return redirect(url_for('index', tag=tag))
@@ -407,81 +785,109 @@ def changedetection_app(config=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
+    threading.Thread(target=notification_runner).start()
+
+    # Check for new release version
+    threading.Thread(target=check_for_new_version).start()
     return app
 
 
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-class Worker(threading.Thread):
-    current_uuid = None
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
 
-    def __init__(self, q, *args, **kwargs):
-        self.q = q
-        super().__init__(*args, **kwargs)
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-    def run(self):
-        from backend import fetch_site_status
+    while not app.config.exit.is_set():
+        try:
+            r = requests.post("https://changedetection.io/check-ver.php",
+                              data={'version': datastore.data['version_tag'],
+                                    'app_guid': datastore.data['app_guid']},
 
-        update_handler = fetch_site_status.perform_site_check(datastore=datastore)
+                              verify=False)
+        except:
+            pass
 
-        while True:
+        try:
+            if "new_version" in r.text:
+                app.config['NEW_VERSION_AVAILABLE'] = True
+        except:
+            pass
 
+        # Check daily
+        app.config.exit.wait(86400)
+
+def notification_runner():
+
+    while not app.config.exit.is_set():
+        try:
+            # At the moment only one thread runs (single runner)
+            n_object = notification_q.get(block=False)
+        except queue.Empty:
+            time.sleep(1)
+            pass
+
+        else:
+            import apprise
+
+            # Create an Apprise instance
             try:
-                uuid = self.q.get(block=True, timeout=1)
-            except queue.Empty:
-                # We have a chance to kill this thread that needs to monitor for new jobs..
-                # Delays here would be caused by a current response object pending
-                # @todo switch to threaded response handler
-                if app.config['STOP_THREADS']:
-                    return
-            else:
-                self.current_uuid = uuid
+                apobj = apprise.Apprise()
+                for url in n_object['notification_urls']:
+                    apobj.add(url.strip())
 
-                if uuid in list(datastore.data['watching'].keys()):
+                n_body = n_object['watch_url']
 
-                    try:
-                        changed_detected, result, contents = update_handler.run(uuid)
+                # 65 - Append URL of instance to the notification if it is set.
+                base_url = os.getenv('BASE_URL')
+                if base_url != None:
+                    n_body += "\n" + base_url
 
-                    except PermissionError as s:
-                        app.logger.error("File permission error updating", uuid, str(s))
-                    else:
-                        if result:
+                apobj.notify(
+                    body=n_body,
+                    # @todo This should be configurable.
+                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
+                )
 
-                            datastore.update_watch(uuid=uuid, update_obj=result)
-                            if changed_detected:
-                                # A change was detected
-                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
-
-
-                self.current_uuid = None  # Done
-                self.q.task_done()
+            except Exception as e:
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
 
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+    from backend import update_worker
+
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
-        new_worker = Worker(update_q)
+        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
         running_update_threads.append(new_worker)
         new_worker.start()
 
-    # Every minute check for new UUIDs to follow up on
-    while True:
-
-        if app.config['STOP_THREADS']:
-            return
+    while not app.config.exit.is_set():
 
         running_uuids = []
         for t in running_update_threads:
-            running_uuids.append(t.current_uuid)
+            if t.current_uuid:
+                running_uuids.append(t.current_uuid)
 
         # Look at the dataset, find a stale watch to process
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
-        for uuid, watch in datastore.data['watching'].items():
-            if watch['last_checked'] <= time.time() - (minutes * 60):
 
-                # @todo maybe update_q.queue is enough?
+        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
+        minutes = datastore.data['settings']['requests']['minutes_between_check']
+
+        threshold = time.time() - (minutes * 60)
+        for uuid, watch in datastore.data['watching'].items():
+
+            # If they supplied an individual entry minutes to recheck and its not the same as the default
+            if 'minutes_between_check' in watch and minutes != watch['minutes_between_check']:
+                threshold = time.time() - (watch['minutes_between_check'] * 60)
+
+            if not watch['paused'] and watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
+            time.sleep(0.1)
+
         # Should be low so we can break this out in testing
-        time.sleep(1)
+        app.config.exit.wait(1)

backend/dev-docker/Dockerfile

@@ -3,9 +3,6 @@ FROM python:3.8-slim
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 
-# Should be mounted from docker-compose-development.yml
-RUN pip3 install -r /requirements.txt
-
 WORKDIR /app
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore

backend/fetch_site_status.py

@@ -2,7 +2,8 @@ import time
 import requests
 import hashlib
 from inscriptis import get_text
-
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some common stuff here that can be moved to a base class
 class perform_site_check():
@@ -11,6 +12,24 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    def strip_ignore_text(self, content, list_ignore_text):
+        ignore = []
+        for k in list_ignore_text:
+            ignore.append(k.encode('utf8'))
+
+        output = []
+        for line in content.splitlines():
+            line = line.encode('utf8')
+
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                if not any(skip_text in line for skip_text in ignore):
+                    output.append(line)
+
+        return "\n".encode('utf8').join(output)
+
+
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
         stripped_text_from_html = False
@@ -47,25 +66,36 @@ class perform_site_check():
                              timeout=timeout,
                              verify=False)
 
-            stripped_text_from_html = get_text(r.text)
+            # CSS Filter
+            css_filter = self.datastore.data['watching'][uuid]['css_filter']
+            if css_filter and len(css_filter.strip()):
+                from bs4 import BeautifulSoup
+                soup = BeautifulSoup(r.content, "html.parser")
+                stripped_text_from_html = ""
+                for item in soup.select(css_filter):
+                    text = str(item.get_text())+"\n"
+                    stripped_text_from_html += text
+
+            else:
+                stripped_text_from_html = get_text(r.text)
 
         # Usually from networkIO/requests level
         except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
             update_obj["last_error"] = str(e)
-
             print(str(e))
 
         except requests.exceptions.MissingSchema:
             print("Skipping {} due to missing schema/bad url".format(uuid))
 
         # Usually from html2text level
-        except UnicodeDecodeError as e:
-
+        except Exception as e:
+            #        except UnicodeDecodeError as e:
             update_obj["last_error"] = str(e)
             print(str(e))
             # figure out how to deal with this cleaner..
             # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
 
+
         else:
             # We rely on the actual text in the html output.. many sites have random script vars etc,
             # in the future we'll implement other mechanisms.
@@ -76,7 +106,15 @@ class perform_site_check():
             if not len(r.text):
                 update_obj["last_error"] = "Empty reply"
 
-            fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            if len(self.datastore.data['watching'][uuid]['ignore_text']):
+                content = self.strip_ignore_text(stripped_text_from_html,
+                                                 self.datastore.data['watching'][uuid]['ignore_text'])
+            else:
+                content = stripped_text_from_html.encode('utf8')
+
+            fetched_md5 = hashlib.md5(content).hexdigest()
 
             # could be None or False depending on JSON type
             if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:

backend/pytest.ini

@@ -1,2 +1,12 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=5005
+addopts = --no-start-live-server --live-server-port=5005
+#testpaths = tests pytest_invenio
+#live_server_scope = function
+
+filterwarnings =
+    ignore::DeprecationWarning:urllib3.*:
+
+; logging options
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s

backend/run_all_tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://foobar.com"
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done

backend/static/css/styles.css

@@ -88,11 +88,16 @@ section.content {
   margin: 0 3px 0 5px;
 }
 
-#check-all-button {
-  text-align:right;
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+}
+#post-list-buttons li {
+  display: inline-block;
 }
 
-#check-all-button a {
+#post-list-buttons a {
    border-top-left-radius: initial;
    border-top-right-radius: initial;
    border-bottom-left-radius: 5px;
@@ -214,3 +219,59 @@ body:after, body:before {
     color: #fff;
 }
 
+#diff-col {
+    padding-left:40px;
+}
+#diff-jump {
+    position: fixed;
+    left: 0px;
+    top: 80px;
+    background: #fff;
+    padding: 10px;
+    border-top-right-radius: 5px;
+    border-bottom-right-radius: 5px;
+    box-shadow: 5px 0 5px -2px #888;
+}
+
+#diff-jump a {
+    color: #1b98f8;
+    cursor: grabbing;
+    -moz-user-select: none;
+    -webkit-user-select: none;
+    -ms-user-select:none;
+    user-select:none;
+    -o-user-select:none;
+}
+
+footer {
+    padding: 10px;
+    background: #fff;
+    color: #444;
+    text-align: center;
+}
+
+#feed-icon {
+    vertical-align: middle;
+}
+
+#version {
+    position: absolute;
+    top: 80px;
+    right: 0px;
+    font-size: 8px;
+    background: #fff;
+    padding: 10px;
+}
+
+#new-version-text a{
+    color: #e07171;
+}
+
+.paused-state.state-False img {
+    opacity: 0.2;
+}
+
+
+.paused-state.state-False:hover img{
+    opacity: 0.8;
+}

backend/static/images/Generic_Feed-icon.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg"
+     id="RSSicon"
+     viewBox="0 0 8 8" width="256" height="256">
+
+  <title>RSS feed icon</title>
+
+  <style type="text/css">
+    .button {stroke: none; fill: orange;}
+    .symbol {stroke: none; fill: white;}
+  </style>
+
+  <rect   class="button" width="8" height="8" rx="1.5" />
+  <circle class="symbol" cx="2" cy="6" r="1" />
+  <path   class="symbol" d="m 1,4 a 3,3 0 0 1 3,3 h 1 a 4,4 0 0 0 -4,-4 z" />
+  <path   class="symbol" d="m 1,2 a 5,5 0 0 1 5,5 h 1 a 6,6 0 0 0 -6,-6 z" />
+
+</svg>

backend/static/images/pause.svg

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   version="1.1"
+   id="Capa_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 15 14.998326"
+   xml:space="preserve"
+   width="15"
+   height="14.998326"><metadata
+   id="metadata39"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs37" />
+<path
+   id="path2"
+   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
+   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
+<g
+   id="g4"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g6"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g8"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g10"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g12"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g14"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g16"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g18"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g20"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g22"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g24"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g26"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g28"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g30"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g32"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+</svg>

backend/store.py

@@ -22,10 +22,10 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
+
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
-            'tag': "0.25",
             'settings': {
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
@@ -37,6 +37,10 @@ class ChangeDetectionStore:
                     'timeout': 15,  # Default 15 seconds
                     'minutes_between_check': 3 * 60,  # Default 3 hours
                     'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'notification_urls': [] # Apprise URL list
                 }
             }
         }
@@ -47,22 +51,28 @@ class ChangeDetectionStore:
             'tag': None,
             'last_checked': 0,
             'last_changed': 0,
+            'paused': False,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
+            'minutes_between_check': 3 * 60,  # Default 3 hours
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
-            'history': {}  # Dict of timestamp and output stripped filename
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'css_filter': "",
         }
 
-        if path.isfile('/source.txt'):
-            with open('/source.txt') as f:
+        if path.isfile('backend/source.txt'):
+            with open('backend/source.txt') as f:
                 # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
                 # So when someone gives us a backup file to examine, we know exactly what code they were running.
                 self.__data['build_sha'] = f.read()
 
         try:
+            # @todo retest with ", encoding='utf-8'"
             with open(self.json_store_path) as json_file:
                 from_disk = json.load(json_file)
 
@@ -71,6 +81,9 @@ class ChangeDetectionStore:
                 if 'watching' in from_disk:
                     self.__data['watching'].update(from_disk['watching'])
 
+                if 'app_guid' in from_disk:
+                    self.__data['app_guid'] = from_disk['app_guid']
+
                 if 'settings' in from_disk:
                     if 'headers' in from_disk['settings']:
                         self.__data['settings']['headers'].update(from_disk['settings']['headers'])
@@ -78,10 +91,12 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
+                    if 'application' in from_disk['settings']:
+                        self.__data['settings']['application'].update(from_disk['settings']['application'])
+
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
-
-                for uuid, watch in self.data['watching'].items():
+                for uuid, watch in self.__data['watching'].items():
                     _blank = deepcopy(self.generic_definition)
                     _blank.update(watch)
                     self.__data['watching'].update({uuid: _blank})
@@ -98,6 +113,18 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+        self.__data['version_tag'] = "0.30"
+
+        if not 'app_guid' in self.__data:
+            import sys
+            import os
+            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
+                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
+            else:
+                self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        self.needs_write = True
+
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
@@ -117,11 +144,15 @@ class ChangeDetectionStore:
         return 0
 
     def set_last_viewed(self, uuid, timestamp):
-        self.data['watching'][uuid].update({'last_viewed': str(timestamp)})
+        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
     def update_watch(self, uuid, update_obj):
 
+        # Skip if 'paused' state
+        if self.__data['watching'][uuid]['paused']:
+            return
+
         with self.lock:
 
             # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -139,6 +170,18 @@ class ChangeDetectionStore:
     @property
     def data(self):
 
+        has_unviewed = False
+        for uuid, v in self.__data['watching'].items():
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
+
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
+
+        self.__data['has_unviewed'] = has_unviewed
+
         return self.__data
 
     def get_all_tags(self):
@@ -154,15 +197,35 @@ class ChangeDetectionStore:
         tags.sort()
         return tags
 
+    def unlink_history_file(self, path):
+        try:
+            os.unlink(path)
+        except (FileNotFoundError, IOError):
+            pass
+
+    # Delete a single watch by UUID
     def delete(self, uuid):
         with self.lock:
-            del (self.__data['watching'][uuid])
+            if uuid == 'all':
+                self.__data['watching'] = {}
+
+                # GitHub #30 also delete history records
+                for uuid in self.data['watching']:
+                    for path in self.data['watching'][uuid]['history'].values():
+                        self.unlink_history_file(path)
+
+            else:
+                for path in self.data['watching'][uuid]['history'].values():
+                    self.unlink_history_file(path)
+
+                del self.data['watching'][uuid]
+
             self.needs_write = True
 
     def url_exists(self, url):
 
         # Probably their should be dict...
-        for watch in self.data['watching']:
+        for watch in self.data['watching'].values():
             if watch['url'] == url:
                 return True
 
@@ -172,6 +235,47 @@ class ChangeDetectionStore:
         # Probably their should be dict...
         return self.data['watching'][uuid].get(val)
 
+    # Remove a watchs data but keep the entry (URL etc)
+    def scrub_watch(self, uuid, limit_timestamp = False):
+
+        import hashlib
+        del_timestamps = []
+
+        for timestamp, path in self.data['watching'][uuid]['history'].items():
+            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
+                self.unlink_history_file(path)
+                del_timestamps.append(timestamp)
+
+
+
+                if not limit_timestamp:
+                    self.data['watching'][uuid]['last_checked'] = 0
+                    self.data['watching'][uuid]['last_changed'] = 0
+                    self.data['watching'][uuid]['previous_md5'] = 0
+
+        for timestamp in del_timestamps:
+            del self.data['watching'][uuid]['history'][str(timestamp)]
+
+            # If there was a limitstamp, we need to reset some meta data about the entry
+            # This has to happen after we remove the others from the list
+            if limit_timestamp:
+                newest_key = self.get_newest_history_key(uuid)
+                if newest_key:
+                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
+                    # @todo should be the original value if it was less than newest key
+                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
+                    try:
+                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
+                            content = fp.read()
+                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
+                    except (FileNotFoundError, IOError):
+                        self.data['watching'][uuid]['previous_md5'] = False
+                        pass
+
+
+        self.needs_write = True
+
+
     def add_watch(self, url, tag):
         with self.lock:
             # @todo use a common generic version of this
@@ -213,11 +317,21 @@ class ChangeDetectionStore:
 
     def sync_to_json(self):
         print("Saving..")
-        with open(self.json_store_path, 'w') as json_file:
-            json.dump(self.__data, json_file, indent=4)
-            logging.info("Re-saved index")
+        data ={}
 
-        self.needs_write = False
+        try:
+            data = deepcopy(self.__data)
+        except RuntimeError:
+            time.sleep(0.5)
+            print ("! Data changed when writing to JSON, trying again..")
+            self.sync_to_json()
+            return
+        else:
+            with open(self.json_store_path, 'w') as json_file:
+                json.dump(data, json_file, indent=4)
+                logging.info("Re-saved index")
+
+            self.needs_write = False
 
     # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
     # by just running periodically in one thread, according to python, dict updates are threadsafe.
@@ -227,8 +341,8 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
+            
             if self.needs_write:
                 self.sync_to_json()
-            time.sleep(1)
+            time.sleep(3)
 
-# body of the constructor

backend/templates/base.html

@@ -17,13 +17,21 @@
 
 <div class="header">
     <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+        {% if has_password and not current_user.is_authenticated %}
+            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
+        {% else %}
+            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+        {% endif %}
         {% if current_diff_url %}
-            <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
+        {% else %}
+        {% if new_version_available %}
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
+        {% endif %}
         {% endif %}
 
         <ul class="pure-menu-list">
-
+        {% if current_user.is_authenticated or not has_password %}
             <li class="pure-menu-item">
                 <a href="/backup" class="pure-menu-link">BACKUP</a>
             </li>
@@ -33,21 +41,27 @@
             <li class="pure-menu-item">
                 <a href="/settings" class="pure-menu-link">SETTINGS</a>
             </li>
+        {% else %}
+            <li class="pure-menu-item">
+                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+            </li>
+        {% endif %}
+
+        {% if current_user.is_authenticated %}
+            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
+        {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
+                     version="1.1"
                      width="32" aria-hidden="true">
                     <path fill-rule="evenodd"
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
-            <!--
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
-            -->
         </ul>
     </div>
 </div>
-
+<div id="version">v{{ version }}</div>
 <section class="content">
     <header>
         {% block header %}{% endblock %}
@@ -66,5 +80,6 @@
 
     {% endblock %}
 </section>
+
 </body>
 </html>

backend/templates/diff.html

@@ -8,7 +8,7 @@
         <fieldset>
 
             <label for="diffWords" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffWords" value="diffWords" /> Words</label>
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
             <label for="diffLines" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
 
@@ -19,9 +19,9 @@
             <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
             <select id="diff-version" name="previous_version">
                 {% for version in versions %}
-                    <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
-                        {{version}}
-                    </option>
+                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                    {{version}}
+                </option>
                 {% endfor %}
             </select>
             <button type="submit" class="pure-button pure-button-primary">Go</button>
@@ -32,6 +32,9 @@
     <ins>Inserted Text</ins>
 </div>
 
+<div id="diff-jump">
+    <a onclick="next_diff();">Jump</a>
+</div>
 <div id="diff-ui">
 
     <table>
@@ -40,7 +43,7 @@
             <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
             <td id="a" style="display: none;">{{previous}}</td>
             <td id="b" style="display: none;">{{newest}}</td>
-            <td>
+            <td id="diff-col">
                 <span id="result"></span>
             </td>
         </tr>
@@ -72,9 +75,12 @@ function changed() {
 		var node;
 		if (diff[i].removed) {
 			node = document.createElement('del');
+			node.classList.add("change");
 			node.appendChild(document.createTextNode(diff[i].value));
+
 		} else if (diff[i].added) {
 			node = document.createElement('ins');
+			node.classList.add("change");
 			node.appendChild(document.createTextNode(diff[i].value));
 		} else {
 			node = document.createTextNode(diff[i].value);
@@ -84,6 +90,10 @@ function changed() {
 
 	result.textContent = '';
 	result.appendChild(fragment);
+
+	// Jump at start
+	inputs.current=0;
+    next_diff();
 }
 
 window.onload = function() {
@@ -106,6 +116,7 @@ window.onload = function() {
 
 	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
 	changed();
+
 };
 
 a.onpaste = a.onchange =
@@ -131,6 +142,27 @@ for (var i = 0; i < radio.length; i++) {
 }
 
 
+var inputs = document.getElementsByClassName('change');
+inputs.current=0;
+
+
+function next_diff() {
+
+    var element = inputs[inputs.current];
+    var headerOffset = 80;
+    var elementPosition = element.getBoundingClientRect().top;
+    var offsetPosition = elementPosition - headerOffset +  window.scrollY;
+
+    window.scrollTo({
+         top: offsetPosition,
+         behavior: "smooth"
+    });
+
+    inputs.current++;
+    if(inputs.current >= inputs.length) {
+      inputs.current=0;
+    }
+}
 
 
 

backend/templates/edit.html

@@ -4,7 +4,7 @@
 <div class="edit-form">
 
 
-    <form class="pure-form pure-form-stacked" action="/edit?uuid={{uuid}}" method="POST">
+    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="url">URL</label>
@@ -17,11 +17,40 @@
                 <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
                 <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
             </div>
+            </br>
+            <div class="pure-control-group">
+                <label for="minutes">Maximum time in minutes until recheck.</label>
+                <input type="text" id="minutes" name="minutes" value="{{watch.minutes_between_check}}"
+                       size="5"/>
+                <span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
+            </div>
+            </br>
+            <div class="pure-control-group">
+                <label for="minutes">CSS Filter</label>
+                <input type="text" id="css_filter" name="css_filter" value="{{watch.css_filter}}"
+                       size="25"/>
+                <span class="pure-form-message-inline">Limit text to this CSS rule, all matching CSS is included.</span>
+            </div>
+            <!-- @todo: move to tabs --->
+            <fieldset class="pure-group">
+                <label for="ignore-text">Ignore text</label>
 
+                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
+                          style="width: 100%;
+                            font-family:monospace;
+                            white-space: pre;
+                            overflow-wrap: normal;
+                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
+{% endfor %}</textarea>
+                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
+
+            </fieldset>
+
+            <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
                 <label for="headers">Extra request headers</label>
 
-                <textarea id=headers name="headers" class="pure-input-1-2" placeholder="Example
+                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0"
                           style="width: 100%;
@@ -33,6 +62,24 @@ User-Agent: wonderbra 1.0"
                 <br/>
 
             </fieldset>
+            <div class="pure-control-group">
+                <label for="tag">Notification URLs</label>
+                <textarea id="notification_urls" name="notification_urls" class="pure-input-1-2" placeholder=""
+                          style="width: 100%;
+                            font-family:monospace;
+                            white-space: pre;
+                            overflow-wrap: normal;
+                            overflow-x: scroll;" rows="5">{% for value in watch.notification_urls %}{{ value }}
+{% endfor %}</textarea>
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</a> </span>
+                <br/>
+                <div class="pure-controls">
+                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
+                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
+
+                </div>
+            </div>
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>

backend/templates/login.html

@@ -0,0 +1,20 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+        <fieldset>
+            <div class="pure-control-group">
+                <label for="password">Password</label>
+                <input type="password" id="password" required="" name="password" value=""
+                       size="15"/>
+                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
+            </div>
+            <div class="pure-control-group">
+                <button type="submit" class="pure-button pure-button-primary">Login</button>
+            </div>
+        </fieldset>
+    </form>
+</div>
+
+{% endblock %}

backend/templates/preview.html

@@ -0,0 +1,26 @@
+{% extends 'base.html' %}
+
+{% block content %}
+
+<div id="settings">
+    <h1>Current</h1>
+</div>
+
+
+<div id="diff-ui">
+
+    <table>
+        <tbody>
+        <tr>
+            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+
+            <td id="diff-col">
+                <span id="result">{% for row in content %}<pre>{{row}}</pre>{% endfor %}</span>
+            </td>
+        </tr>
+        </tbody>
+    </table>
+</div>
+
+
+{% endblock %}

backend/templates/scrub.html

@@ -17,14 +17,19 @@
 
             <div class="pure-control-group">
                 <br/>
-                <label for="confirmtext">Confirm</label><br/>
+                <label for="confirmtext">Confirm text</label><br/>
                 <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+            </div>
+
+
+            <div class="pure-control-group">
+                <br/>
+                <label for="confirmtext">Limit delete history including and after date</label><br/>
+                <input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
                 <br/>
 
 
             </div>
-
-
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>

backend/templates/settings.html

@@ -10,9 +10,39 @@
                 <label for="minutes">Maximum time in minutes until recheck.</label>
                 <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
                        size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                <span class="pure-form-message-inline">This is a required field.</span><br/>
+                <span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
+            </div>
+            <br/>
+            <hr>
+            <div class="pure-control-group">
+                <label for="minutes">Password protection</label>
+                <input type="password" id="password" name="password" size="15"/>
+                {% if current_user.is_authenticated %}
+                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
+                {% endif %}
             </div>
 
+            <br/>
+            <hr>
+
+            <div class="pure-control-group">
+                <label for="minutes">Global notification settings</label><br/>
+                Notification URLs <a href="https://github.com/caronc/apprise"> see Apprise examples</a>.
+                <textarea style="overflow-wrap: normal;  overflow-x: scroll;" id="notification_urls" name="notification_urls" cols="80"
+                          rows="6" wrap=off placeholder="Example:
+
+Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+">{{notification_urls}}</textarea>
+            </div>
+                <div class="pure-controls">
+                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
+                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
+
+                </div>
 
             <br/>
             <div class="pure-control-group">
@@ -22,7 +52,7 @@
 
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Reset all version data</a>
+                <a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
             </div>
 
 

backend/templates/watch-overview.html

@@ -15,13 +15,11 @@
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-
+        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
-        {% if tag == "" %}
-        <a href="/" class="pure-button button-tag {{'active' if active_tag == tag }}">All</a>
-        {% else %}
-        <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
-        {% endif %}
+            {% if tag != "" %}
+                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% endif %}
         {% endfor %}
     </div>
 
@@ -31,6 +29,7 @@
             <tr>
                 <th>#</th>
                 <th></th>
+                <th></th>
                 <th>Last Checked</th>
                 <th>Last Changed</th>
                 <th></th>
@@ -43,10 +42,12 @@
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                 <td>{{ loop.index }}</td>
+                <td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
                 <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
-                    <a class="external" target=_blank href="{{ watch.url }}"></a>
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
@@ -64,22 +65,33 @@
                 <td>
                     <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    {% else %}
+                        {% if watch.history|length == 1 %}
+                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                        {% endif %}
                     {% endif %}
                 </td>
             </tr>
             {% endfor %}
-
-
             </tbody>
         </table>
-        <div id="check-all-button">
-
-            <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
-        </div>
+            </li>
+            <li>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+            </li>
+        </ul>
     </div>
 </div>
 {% endblock %}

backend/tests/conftest.py

@@ -7,17 +7,14 @@ import os
 
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
-
 # Much better boilerplate than the docs
 # https://www.python-boilerplate.com/py3+flask+pytest/
 
 global app
 
-
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
-
     datastore_path = "./test-datastore"
 
     try:
@@ -33,11 +30,18 @@ def app(request):
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+    app.config['STOP_THREADS'] = True
 
     def teardown():
         datastore.stop_thread = True
-        app.config['STOP_THREADS'] = True
+        app.config.exit.set()
+        for fname in ["url-watches.json", "count.txt", "output.txt"]:
+            try:
+                os.unlink("{}/{}".format(datastore_path, fname))
+            except FileNotFoundError:
+                # This is fine in the case of a failure.
+                pass
 
     request.addfinalizer(teardown)
+    yield app
 
-    return app

backend/tests/test_access_control.py

@@ -0,0 +1,58 @@
+from flask import url_for
+
+def test_check_access_control(app, client):
+    # Still doesnt work, but this is closer.
+    return
+    with app.test_client() as c:
+
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "foobar"},
+            follow_redirects=True
+        )
+        assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
+        print ("SESSION:", res.session)
+        # Check we hit the login
+
+        res = c.get(url_for("settings_page"), follow_redirects=True)
+        res = c.get(url_for("login"), follow_redirects=True)
+
+        assert b"Login" in res.data
+
+        print ("DEBUG >>>>>",res.data)
+        # Menu should not be available yet
+        assert b"SETTINGS" not in res.data
+        assert b"BACKUP" not in res.data
+        assert b"IMPORT" not in res.data
+
+
+
+        #defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
+        res = c.post(
+            url_for("login"),
+            data={"password": "foobar", "email": "defaultuser@changedetection.io"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" in res.data
+        res = c.get(url_for("settings_page"))
+
+        # Menu should be available now
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
+        assert b"LOG OUT" in res.data
+
+        # Now remove the password so other tests function, @todo this should happen before each test automatically
+
+        c.get(url_for("settings_page", removepassword="true"))
+        c.get(url_for("import_page"))
+        assert b"LOG OUT" not in res.data
+

backend/tests/test_backend.py

@@ -3,52 +3,16 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
 
 
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
 
 
 def test_check_basic_change_detection_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 5
-
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
     set_original_response()
-
-    live_server.start()
+    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -72,6 +36,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
+        # Default no password set, this stuff should be always available.
+
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
     #####################
 
     # Make a change
@@ -90,14 +60,20 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # #75, and it should be in the RSS feed
+    res = client.get(url_for("index", rss="true"))
+    expected_url = url_for('test_endpoint', _external=True)
+    assert b'<rss' in res.data
+    assert expected_url.encode('utf-8') in res.data
+
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
-    res = client.get(url_for("diff_history_page", uuid="first") )
+    res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
 
     time.sleep(2)
 
     # Do this a few times.. ensures we dont accidently set the status
-    for n in range(3):
+    for n in range(2):
         client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
@@ -108,10 +84,14 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
-
     set_original_response()
 
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
+    assert b'unviewed' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+

backend/tests/test_css_selector.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def test_check_markup_css_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    css_filter = "#sametext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

backend/tests/test_ignore_text.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+
+    but sometimes we want to remove the lines.
+
+    but not always."""
+
+    ignore_lines = ["sometimes"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZZ</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

backend/tests/test_notification.py

@@ -0,0 +1,72 @@
+
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_check_notification(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    url = url_for('test_notification_endpoint', _external=True)
+    notification_url = url.replace('http', 'json')
+
+    print (">>>> Notification URL: "+notification_url)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Hit the edit page, be sure that we saved it
+    res = client.get(
+        url_for("edit_page", uuid="first"))
+    assert bytes(notification_url.encode('utf-8')) in res.data
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+
+    # Check it triggered
+    res = client.get(
+        url_for("test_notification_counter"),
+    )
+
+    assert bytes("we hit it".encode('utf-8')) in res.data
+
+    # Did we see the URL that had a change, in the notification?
+    assert bytes("test-endpoint".encode('utf-8')) in res.data
+
+    # Re #65 - did we see our foobar.com BASE_URL ?
+    assert bytes("https://foobar.com".encode('utf-8')) in res.data

backend/tests/util.py

@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def live_server_setup(live_server):
+
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
+    def test_notification_endpoint():
+        from flask import request
+
+        with open("test-datastore/count.txt", "w") as f:
+            f.write("we hit it\n")
+            # Debug method, dump all POST to file also, used to prove #65
+            data = request.stream.read()
+            if data != None:
+                f.write(str(data))
+
+        print("\n>> Test notification endpoint was hit.\n")
+        return "Text was set"
+
+    # And this should return not zero.
+    @live_server.app.route('/test_notification_counter')
+    def test_notification_counter():
+        try:
+            with open("test-datastore/count.txt", "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            return "nope :("
+
+    live_server.start()

backend/update_worker.py

@@ -0,0 +1,67 @@
+import threading
+import queue
+
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+class update_worker(threading.Thread):
+    current_uuid = None
+
+    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        self.q = q
+        self.app = app
+        self.notification_q = notification_q
+        self.datastore = datastore
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from backend import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
+
+        while not self.app.config.exit.is_set():
+
+            try:
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
+
+            else:
+                self.current_uuid = uuid
+
+                if uuid in list(self.datastore.data['watching'].keys()):
+                    try:
+                        changed_detected, result, contents = update_handler.run(uuid)
+
+                    except PermissionError as s:
+                        self.app.logger.error("File permission error updating", uuid, str(s))
+                    else:
+                        if result:
+                            try:
+                                self.datastore.update_watch(uuid=uuid, update_obj=result)
+                                if changed_detected:
+                                    # A change was detected
+                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
+
+                                    watch = self.datastore.data['watching'][uuid]
+
+                                    # Did it have any notification alerts to hit?
+                                    if len(watch['notification_urls']):
+                                        print("Processing notifications for UUID: {}".format(uuid))
+                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
+                                                    'notification_urls': watch['notification_urls']}
+                                        self.notification_q.put(n_object)
+
+
+                                    # No? maybe theres a global setting, queue them all
+                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
+                                                    'notification_urls': self.datastore.data['settings']['application'][
+                                                        'notification_urls']}
+                                        self.notification_q.put(n_object)
+                            except Exception as e:
+                                print("!!!! Exception in update_worker !!!\n", e)
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            self.app.config.exit.wait(1)

changedetection.py

@@ -3,6 +3,7 @@
 # Launch as a eventlet.wsgi server instance.
 
 import getopt
+import os
 import sys
 
 import eventlet
@@ -12,10 +13,31 @@ import backend
 from backend import store
 
 
+def init_app_secret(datastore_path):
+    secret = ""
+
+    path = "{}/secret.txt".format(datastore_path)
+
+    try:
+        with open(path, "r") as f:
+            secret = f.read()
+
+    except FileNotFoundError:
+
+        import secrets
+        with open(path, "w") as f:
+            secret = secrets.token_hex(32)
+            f.write(secret)
+
+    return secret
+
+
 def main(argv):
     ssl_mode = False
     port = 5000
-    datastore_path = "./datastore"
+
+    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+    datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
         opts, args = getopt.getopt(argv, "sd:p:", "purge")
@@ -38,18 +60,22 @@ def main(argv):
         if opt == '-d':
             datastore_path = arg
 
-
-
-    # threads can read from disk every x seconds right?
-    # front end can just save
-    # We just need to know which threads are looking at which UUIDs
-
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
     app = backend.changedetection_app(app_config, datastore)
 
+    app.config['datastore_path'] = datastore_path
+    app.secret_key = init_app_secret(app_config['datastore_path'])
+
+    @app.context_processor
+    def inject_version():
+        return dict(version=datastore.data['version_tag'],
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
         eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),

docker-compose-development.yml

@@ -1,23 +0,0 @@
-version: "2"
-services:
-
-# I have a feeling we can get rid of this, and just use one docker-compose.yml, and just set a ENV var if
-# we want dev mode (just gives a docker shell) or not.
-
-  backend:
-    build: ./backend/dev-docker
-    image: dgtlmoon/changedetection.io:dev
-    container_name: changedetection.io-dev
-    volumes:
-      - ./backend:/app
-      - ./requirements.txt:/requirements.txt # Normally COPY'ed in the Dockerfile
-      - ./datastore:/datastore
-
-    ports:
-      - "127.0.0.1:5001:5000"
-
-    networks:
-      - changenet
-
-networks:
-  changenet:

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '2'
+services:
+    changedetection.io:
+      image: dgtlmoon/changedetection.io
+      container_name: changedetection.io
+      hostname: changedetection.io
+      volumes:
+        - changedetection-data:/datastore
+
+  #    environment:
+  #        Proxy support example.
+  #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
+  #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
+  #        An exclude list (useful for notification URLs above) can be specified by with
+  #      - NO_PROXY="localhost,192.168.0.0/24"
+  #        Base URL of your changedetection.io install (Added to notification alert
+  #      - BASE_URL="https://mysite.com"
+
+      ports:
+        - 5000:5000
+      restart: always
+
+volumes:
+  changedetection-data:
+

requirements.txt

@@ -1,24 +1,20 @@
-aiohttp
-async-timeout
 chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-pytest
-pytest-flask # for live_server
-
-eventlet
-requests
+flask~= 1.0
+pytest ~=6.2
+pytest-flask ~=1.2
+eventlet>=0.31.0
+requests[socks] ~= 2.15
 validators
+timeago ~=1.0
+inscriptis ~= 1.1
+feedgen ~= 0.9
+flask-login ~= 0.5
+pytz
+urllib3
 
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-inscriptis
+# Notification library
+apprise ~= 0.9
+
+# Used for CSS filtering
+bs4
 
-# @notes
-# - Dont install socketio, it interferes with flask_socketio