Adding a GC memory cleanup (releases cached libxml memory and others)

.github/workflows/test-only.yml

@@ -28,7 +28,6 @@ jobs:
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.11'
-      skip-pypuppeteer: true
 
   test-application-3-12:
     needs: lint-code

.github/workflows/test-stack-reusable-workflow.yml

@@ -7,7 +7,7 @@ on:
         description: 'Python version to use'
         required: true
         type: string
-        default: '3.10'
+        default: '3.11'
       skip-pypuppeteer:
         description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
         required: false

Dockerfile

@@ -1,8 +1,5 @@
 # pip dependencies install stage
 
-# @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
-#        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-
 ARG PYTHON_VERSION=3.11
 
 FROM python:${PYTHON_VERSION}-slim-bookworm AS builder

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.4'
+__version__ = '0.49.9'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -19,7 +19,6 @@ from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
 from loguru import logger
 
-
 # Only global so we can access it in the signal handler
 app = None
 datastore = None
@@ -29,8 +28,6 @@ def get_version():
 
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
-    global app
-    global datastore
     name = signal.Signals(_signo).name
     logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
     datastore.sync_to_json()
@@ -147,6 +144,15 @@ def main():
 
     signal.signal(signal.SIGTERM, sigshutdown_handler)
     signal.signal(signal.SIGINT, sigshutdown_handler)
+    
+    # Custom signal handler for memory cleanup
+    def sigusr_clean_handler(_signo, _stack_frame):
+        from changedetectionio.gc_cleanup import memory_cleanup
+        logger.info('SIGUSR1 received: Running memory cleanup')
+        return memory_cleanup(app)
+    
+    # Register the SIGUSR1 signal handler
+    signal.signal(signal.SIGUSR1, sigusr_clean_handler)
 
     # Go into cleanup mode
     if do_cleanup:

changedetectionio/api/Import.py

@@ -0,0 +1,62 @@
+import os
+from changedetectionio.strtobool import strtobool
+from flask_restful import abort, Resource
+from flask import request
+import validators
+from . import auth
+
+
+class Import(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def post(self):
+        """
+        @api {post} /api/v1/import Import a list of watched URLs
+        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
+        @apiName Import
+        @apiGroup Watch
+        @apiSuccess (200) {List} OK List of watch UUIDs added
+        @apiSuccess (500) {String} ERR Some other error
+        """
+
+        extras = {}
+
+        if request.args.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.args.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+            else:
+                extras['proxy'] = request.args.get('proxy')
+
+        dedupe = strtobool(request.args.get('dedupe', 'true'))
+
+        tags = request.args.get('tag')
+        tag_uuids = request.args.get('tag_uuids')
+
+        if tag_uuids:
+            tag_uuids = tag_uuids.split(',')
+
+        urls = request.get_data().decode('utf8').splitlines()
+        added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+            if not validators.url(url, simple_host=allow_simplehost):
+                return f"Invalid or unsupported URL - {url}", 400
+
+            if dedupe and self.datastore.url_exists(url):
+                continue
+
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
+
+        return added

changedetectionio/api/Search.py

@@ -0,0 +1,51 @@
+from flask_restful import Resource, abort
+from flask import request
+from . import auth
+
+class Search(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def get(self):
+        """
+        @api {get} /api/v1/search Search for watches
+        @apiDescription Search watches by URL or title text
+        @apiExample {curl} Example usage:
+            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1?tag=Favourites" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/search?q=https://example.com?partial=true" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiName Search
+        @apiGroup Watch Management
+        @apiQuery {String} q Search query to match against watch URLs and titles
+        @apiQuery {String} [tag] Optional name of tag to limit results (name not UUID)
+        @apiQuery {String} [partial] Allow partial matching of URL query
+        @apiSuccess (200) {Object} JSON Object containing matched watches
+        """
+        query = request.args.get('q', '').strip()
+        tag_limit = request.args.get('tag', '').strip()
+        from changedetectionio.strtobool import strtobool
+        partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False
+
+        # Require a search query
+        if not query:
+            abort(400, message="Search query 'q' parameter is required")
+
+        # Use the search function from the datastore
+        matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)
+
+        # Build the response with watch details
+        results = {}
+        for uuid in matching_uuids:
+            watch = self.datastore.data['watching'].get(uuid)
+            results[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        return results, 200

changedetectionio/api/SystemInfo.py

@@ -0,0 +1,54 @@
+from flask_restful import Resource
+from . import auth
+
+
+class SystemInfo(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    def get(self):
+        """
+        @api {get} /api/v1/systeminfo Return system info
+        @apiDescription Return some info about the current system state
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            HTTP/1.0 200
+            {
+                'queue_size': 10 ,
+                'overdue_watches': ["watch-uuid-list"],
+                'uptime': 38344.55,
+                'watch_count': 800,
+                'version': "0.40.1"
+            }
+        @apiName Get Info
+        @apiGroup System Information
+        """
+        import time
+        overdue_watches = []
+
+        # Check all watches and report which have not been checked but should have been
+
+        for uuid, watch in self.datastore.data.get('watching', {}).items():
+            # see if now - last_checked is greater than the time that should have been
+            # this is not super accurate (maybe they just edited it) but better than nothing
+            t = watch.threshold_seconds()
+            if not t:
+                # Use the system wide default
+                t = self.datastore.threshold_seconds
+
+            time_since_check = time.time() - watch.get('last_checked')
+
+            # Allow 5 minutes of grace time before we decide it's overdue
+            if time_since_check - (5 * 60) > t:
+                overdue_watches.append(uuid)
+        from changedetectionio import __version__ as main_version
+        return {
+                   'queue_size': self.update_q.qsize(),
+                   'overdue_watches': overdue_watches,
+                   'uptime': round(time.time() - self.datastore.start_time, 2),
+                   'watch_count': len(self.datastore.data.get('watching', {})),
+                   'version': main_version
+               }, 200

changedetectionio/api/Tags.py

@@ -0,0 +1,156 @@
+from flask_expects_json import expects_json
+from flask_restful import abort, Resource
+from flask import request
+from . import auth
+
+# Import schemas from __init__.py
+from . import schema_tag, schema_create_tag, schema_update_tag
+
+
+class Tag(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get information about a single tag
+    # curl http://localhost:5000/api/v1/tag/<string:uuid>
+    @auth.check_token
+    def get(self, uuid):
+        """
+        @api {get} /api/v1/tag/:uuid Single tag - get data or toggle notification muting.
+        @apiDescription Retrieve tag information and set notification_muted status
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=muted" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiName Tag
+        @apiGroup Tag
+        @apiParam {uuid} uuid Tag unique ID.
+        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
+        @apiSuccess (200) {String} OK When muted operation OR full JSON object of the tag
+        @apiSuccess (200) {JSON} TagJSON JSON Full JSON object of the tag
+        """
+        from copy import deepcopy
+        tag = deepcopy(self.datastore.data['settings']['application']['tags'].get(uuid))
+        if not tag:
+            abort(404, message=f'No tag exists with the UUID of {uuid}')
+
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = True
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = False
+            return "OK", 200
+
+        return tag
+
+    @auth.check_token
+    def delete(self, uuid):
+        """
+        @api {delete} /api/v1/tag/:uuid Delete a tag and remove it from all watches
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiParam {uuid} uuid Tag unique ID.
+        @apiName DeleteTag
+        @apiGroup Tag
+        @apiSuccess (200) {String} OK Was deleted
+        """
+        if not self.datastore.data['settings']['application']['tags'].get(uuid):
+            abort(400, message='No tag exists with the UUID of {}'.format(uuid))
+
+        # Delete the tag, and any tag reference
+        del self.datastore.data['settings']['application']['tags'][uuid]
+        
+        # Remove tag from all watches
+        for watch_uuid, watch in self.datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                watch['tags'].remove(uuid)
+
+        return 'OK', 204
+
+    @auth.check_token
+    @expects_json(schema_update_tag)
+    def put(self, uuid):
+        """
+        @api {put} /api/v1/tag/:uuid Update tag information
+        @apiExample {curl} Example usage:
+            Update (PUT)
+            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"title": "New Tag Title"}'
+
+        @apiDescription Updates an existing tag using JSON
+        @apiParam {uuid} uuid Tag unique ID.
+        @apiName UpdateTag
+        @apiGroup Tag
+        @apiSuccess (200) {String} OK Was updated
+        @apiSuccess (500) {String} ERR Some other error
+        """
+        tag = self.datastore.data['settings']['application']['tags'].get(uuid)
+        if not tag:
+            abort(404, message='No tag exists with the UUID of {}'.format(uuid))
+
+        tag.update(request.json)
+        self.datastore.needs_write_urgent = True
+
+        return "OK", 200
+
+
+    @auth.check_token
+    # Only cares for {'title': 'xxxx'}
+    def post(self):
+        """
+        @api {post} /api/v1/watch Create a single tag
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"name": "Work related"}'
+        @apiName Create
+        @apiGroup Tag
+        @apiSuccess (200) {String} OK Was created
+        @apiSuccess (500) {String} ERR Some other error
+        """
+
+        json_data = request.get_json()
+        title = json_data.get("title",'').strip()
+
+
+        new_uuid = self.datastore.add_tag(title=title)
+        if new_uuid:
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported tag", 400
+
+class Tags(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def get(self):
+        """
+        @api {get} /api/v1/tags List tags
+        @apiDescription Return list of available tags
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/tags -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            {
+                "cc0cfffa-f449-477b-83ea-0caafd1dc091": {
+                    "title": "Tech News",
+                    "notification_muted": false,
+                    "date_created": 1677103794
+                },
+                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
+                    "title": "Shopping",
+                    "notification_muted": true,
+                    "date_created": 1676662819
+                }
+            }
+        @apiName ListTags
+        @apiGroup Tag Management
+        @apiSuccess (200) {String} OK JSON dict
+        """
+        result = {}
+        for uuid, tag in self.datastore.data['settings']['application']['tags'].items():
+            result[uuid] = {
+                'date_created': tag.get('date_created', 0),
+                'notification_muted': tag.get('notification_muted', False),
+                'title': tag.get('title', ''),
+                'uuid': tag.get('uuid')
+            }
+
+        return result, 200

changedetectionio/api/Watch.py

@@ -9,20 +9,9 @@ import validators
 from . import auth
 import copy
 
-# See docs/README.md for rebuilding the docs/apidoc information
+# Import schemas from __init__.py
+from . import schema, schema_create_watch, schema_update_watch
 
-from . import api_schema
-from ..model import watch_base
-
-# Build a JSON Schema atleast partially based on our Watch model
-watch_base_config = watch_base()
-schema = api_schema.build_watch_json_schema(watch_base_config)
-
-schema_create_watch = copy.deepcopy(schema)
-schema_create_watch['required'] = ['url']
-
-schema_update_watch = copy.deepcopy(schema)
-schema_update_watch['additionalProperties'] = False
 
 class Watch(Resource):
     def __init__(self, **kwargs):
@@ -285,8 +274,6 @@ class CreateWatch(Resource):
         list = {}
 
         tag_limit = request.args.get('tag', '').lower()
-
-
         for uuid, watch in self.datastore.data['watching'].items():
             # Watch tags by name (replace the other calls?)
             tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
@@ -307,110 +294,4 @@ class CreateWatch(Resource):
                 self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
             return {'status': "OK"}, 200
 
-        return list, 200
-
-class Import(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    def post(self):
-        """
-        @api {post} /api/v1/import Import a list of watched URLs
-        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
-        @apiName Import
-        @apiGroup Watch
-        @apiSuccess (200) {List} OK List of watch UUIDs added
-        @apiSuccess (500) {String} ERR Some other error
-        """
-
-        extras = {}
-
-        if request.args.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not request.args.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-            else:
-                extras['proxy'] = request.args.get('proxy')
-
-        dedupe = strtobool(request.args.get('dedupe', 'true'))
-
-        tags = request.args.get('tag')
-        tag_uuids = request.args.get('tag_uuids')
-
-        if tag_uuids:
-            tag_uuids = tag_uuids.split(',')
-
-        urls = request.get_data().decode('utf8').splitlines()
-        added = []
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        for url in urls:
-            url = url.strip()
-            if not len(url):
-                continue
-
-            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not validators.url(url, simple_host=allow_simplehost):
-                return f"Invalid or unsupported URL - {url}", 400
-
-            if dedupe and self.datastore.url_exists(url):
-                continue
-
-            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
-            added.append(new_uuid)
-
-        return added
-
-class SystemInfo(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def get(self):
-        """
-        @api {get} /api/v1/systeminfo Return system info
-        @apiDescription Return some info about the current system state
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            HTTP/1.0 200
-            {
-                'queue_size': 10 ,
-                'overdue_watches': ["watch-uuid-list"],
-                'uptime': 38344.55,
-                'watch_count': 800,
-                'version': "0.40.1"
-            }
-        @apiName Get Info
-        @apiGroup System Information
-        """
-        import time
-        overdue_watches = []
-
-        # Check all watches and report which have not been checked but should have been
-
-        for uuid, watch in self.datastore.data.get('watching', {}).items():
-            # see if now - last_checked is greater than the time that should have been
-            # this is not super accurate (maybe they just edited it) but better than nothing
-            t = watch.threshold_seconds()
-            if not t:
-                # Use the system wide default
-                t = self.datastore.threshold_seconds
-
-            time_since_check = time.time() - watch.get('last_checked')
-
-            # Allow 5 minutes of grace time before we decide it's overdue
-            if time_since_check - (5 * 60) > t:
-                overdue_watches.append(uuid)
-        from changedetectionio import __version__ as main_version
-        return {
-                   'queue_size': self.update_q.qsize(),
-                   'overdue_watches': overdue_watches,
-                   'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {})),
-                   'version': main_version
-               }, 200
+        return list, 200

changedetectionio/api/__init__.py

@@ -0,0 +1,26 @@
+import copy
+from . import api_schema
+from ..model import watch_base
+
+# Build a JSON Schema atleast partially based on our Watch model
+watch_base_config = watch_base()
+schema = api_schema.build_watch_json_schema(watch_base_config)
+
+schema_create_watch = copy.deepcopy(schema)
+schema_create_watch['required'] = ['url']
+
+schema_update_watch = copy.deepcopy(schema)
+schema_update_watch['additionalProperties'] = False
+
+# Tag schema is also based on watch_base since Tag inherits from it
+schema_tag = copy.deepcopy(schema)
+schema_create_tag = copy.deepcopy(schema_tag)
+schema_create_tag['required'] = ['title']
+schema_update_tag = copy.deepcopy(schema_tag)
+schema_update_tag['additionalProperties'] = False
+
+# Import all API resources
+from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch
+from .Tags import Tags, Tag
+from .Import import Import
+from .SystemInfo import SystemInfo

changedetectionio/api/auth.py

@@ -11,22 +11,14 @@ def check_token(f):
         datastore = args[0].datastore
 
         config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
-        if not config_api_token_enabled:
-            return
-
-        try:
-            api_key_header = request.headers['x-api-key']
-        except KeyError:
-            return make_response(
-                jsonify("No authorization x-api-key header."), 403
-            )
-
         config_api_token = datastore.data['settings']['application'].get('api_access_token')
 
-        if api_key_header != config_api_token:
-            return make_response(
-                jsonify("Invalid access - API key invalid."), 403
-            )
+        # config_api_token_enabled - a UI option in settings if access should obey the key or not
+        if config_api_token_enabled:
+            if request.headers.get('x-api-key') != config_api_token:
+                return make_response(
+                    jsonify("Invalid access - API key invalid."), 403
+                )
 
         return f(*args, **kwargs)
 

changedetectionio/apprise_asset.py

@@ -1,12 +0,0 @@
-from changedetectionio import apprise_plugin
-import apprise
-
-# Create our AppriseAsset and populate it with some of our new values:
-# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
-asset = apprise.AppriseAsset(
-   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
-)
-
-asset.app_id = "changedetection.io"
-asset.app_desc = "ChangeDetection.io best and simplest website monitoring and change detection"
-asset.app_url = "https://changedetection.io"

changedetectionio/apprise_plugin/__init__.py

@@ -1,98 +0,0 @@
-# include the decorator
-from apprise.decorators import notify
-from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
-
-@notify(on="delete")
-@notify(on="deletes")
-@notify(on="get")
-@notify(on="gets")
-@notify(on="post")
-@notify(on="posts")
-@notify(on="put")
-@notify(on="puts")
-def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
-    import requests
-    import json
-    import re
-
-    from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
-
-    url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
-
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
-
-    params = CaseInsensitiveDict({}) # Added to requests
-    auth = None
-    has_error = False
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
-
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
-
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
-    try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
-
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True

changedetectionio/apprise_plugin/assets.py

@@ -0,0 +1,16 @@
+from apprise import AppriseAsset
+
+# Refer to:
+# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
+
+APPRISE_APP_ID = "changedetection.io"
+APPRISE_APP_DESC = "ChangeDetection.io best and simplest website monitoring and change detection"
+APPRISE_APP_URL = "https://changedetection.io"
+APPRISE_AVATAR_URL = "https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png"
+
+apprise_asset = AppriseAsset(
+    app_id=APPRISE_APP_ID,
+    app_desc=APPRISE_APP_DESC,
+    app_url=APPRISE_APP_URL,
+    image_url_logo=APPRISE_AVATAR_URL,
+)

changedetectionio/apprise_plugin/custom_handlers.py

@@ -0,0 +1,112 @@
+import json
+import re
+from urllib.parse import unquote_plus
+
+import requests
+from apprise.decorators import notify
+from apprise.utils.parse import parse_url as apprise_parse_url
+from loguru import logger
+from requests.structures import CaseInsensitiveDict
+
+SUPPORTED_HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head"}
+
+
+def notify_supported_methods(func):
+    for method in SUPPORTED_HTTP_METHODS:
+        func = notify(on=method)(func)
+        # Add support for https, for each supported http method
+        func = notify(on=f"{method}s")(func)
+    return func
+
+
+def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
+    user: str | None = parsed_url.get("user")
+    password: str | None = parsed_url.get("password")
+
+    if user is not None and password is not None:
+        return (unquote_plus(user), unquote_plus(password))
+
+    if user is not None:
+        return unquote_plus(user)
+
+    return ""
+
+
+def _get_headers(parsed_url: dict, body: str) -> CaseInsensitiveDict:
+    headers = CaseInsensitiveDict(
+        {unquote_plus(k).title(): unquote_plus(v) for k, v in parsed_url["qsd+"].items()}
+    )
+
+    # If Content-Type is not specified, guess if the body is a valid JSON
+    if headers.get("Content-Type") is None:
+        try:
+            json.loads(body)
+            headers["Content-Type"] = "application/json; charset=utf-8"
+        except Exception:
+            pass
+
+    return headers
+
+
+def _get_params(parsed_url: dict) -> CaseInsensitiveDict:
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+    # but here we are making straight requests, so we need todo convert this against apprise's logic
+    params = CaseInsensitiveDict(
+        {
+            unquote_plus(k): unquote_plus(v)
+            for k, v in parsed_url["qsd"].items()
+            if k.strip("-") not in parsed_url["qsd-"]
+            and k.strip("+") not in parsed_url["qsd+"]
+        }
+    )
+
+    return params
+
+
+@notify_supported_methods
+def apprise_http_custom_handler(
+    body: str,
+    title: str,
+    notify_type: str,
+    meta: dict,
+    *args,
+    **kwargs,
+) -> bool:
+    url: str = meta.get("url")
+    schema: str = meta.get("schema")
+    method: str = re.sub(r"s$", "", schema).upper()
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    parsed_url: dict[str, str | dict | None] | None = apprise_parse_url(url)
+    if parsed_url is None:
+        return False
+
+    auth = _get_auth(parsed_url=parsed_url)
+    headers = _get_headers(parsed_url=parsed_url, body=body)
+    params = _get_params(parsed_url=parsed_url)
+
+    url = re.sub(rf"^{schema}", "https" if schema.endswith("s") else "http", parsed_url.get("url"))
+
+    try:
+        response = requests.request(
+            method=method,
+            url=url,
+            auth=auth,
+            headers=headers,
+            params=params,
+            data=body.encode("utf-8") if isinstance(body, str) else body,
+        )
+
+        response.raise_for_status()
+
+        logger.info(f"Successfully sent custom notification to {url}")
+        return True
+
+    except requests.RequestException as e:
+        logger.error(f"Remote host error while sending custom notification to {url}: {e}")
+        return False
+
+    except Exception as e:
+        logger.error(f"Unexpected error occurred while sending custom notification to {url}: {e}")
+        return False

changedetectionio/auth_decorator.py

@@ -0,0 +1,33 @@
+import os
+from functools import wraps
+from flask import current_app, redirect, request
+from loguru import logger
+
+def login_optionally_required(func):
+    """
+    If password authentication is enabled, verify the user is logged in.
+    To be used as a decorator for routes that should optionally require login.
+    This version is blueprint-friendly as it uses current_app instead of directly accessing app.
+    """
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        from flask import current_app
+        import flask_login
+        from flask_login import current_user
+
+        # Access datastore through the app config
+        datastore = current_app.config['DATASTORE']
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        # Permitted
+        if request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
+            return func(*args, **kwargs)
+        elif request.method in flask_login.config.EXEMPT_METHODS:
+            return func(*args, **kwargs)
+        elif current_app.config.get('LOGIN_DISABLED'):
+            return func(*args, **kwargs)
+        elif has_password_enabled and not current_user.is_authenticated:
+            return current_app.login_manager.unauthorized()
+
+        return func(*args, **kwargs)
+    return decorated_view

changedetectionio/blueprint/backups/__init__.py

@@ -138,7 +138,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
 
     @login_optionally_required
-    @backups_blueprint.route("/", methods=['GET'])
+    @backups_blueprint.route("", methods=['GET'])
     def index():
         backups = find_backups()
         output = render_template("overview.html",

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,7 +23,6 @@ from loguru import logger
 browsersteps_sessions = {}
 io_interface_context = None
 import json
-import base64
 import hashlib
 from flask import Response
 
@@ -34,10 +33,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         from . import nonContext
         from . import browser_steps
         import time
-        global browsersteps_sessions
         global io_interface_context
 
-
         # We keep the playwright session open for many minutes
         keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
@@ -104,8 +101,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # A new session was requested, return sessionID
 
         import uuid
-        global browsersteps_sessions
-
         browsersteps_session_id = str(uuid.uuid4())
         watch_uuid = request.args.get('uuid')
 
@@ -149,7 +144,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def browsersteps_ui_update():
         import base64
         import playwright._impl._errors
-        global browsersteps_sessions
         from changedetectionio.blueprint.browser_steps import browser_steps
 
         remaining =0

changedetectionio/blueprint/imports/__init__.py

@@ -0,0 +1,74 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.blueprint.imports.importer import (
+    import_url_list, 
+    import_distill_io_json, 
+    import_xlsx_wachete, 
+    import_xlsx_custom
+)
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    import_blueprint = Blueprint('imports', __name__, template_folder="templates")
+    
+    @import_blueprint.route("/import", methods=['GET', 'POST'])
+    @login_optionally_required
+    def import_page():
+        remaining_urls = []
+        from changedetectionio import forms
+
+        if request.method == 'POST':
+            # URL List import
+            if request.values.get('urls') and len(request.values.get('urls').strip()):
+                # Import and push into the queue for immediate update check
+                importer_handler = import_url_list()
+                importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
+                for uuid in importer_handler.new_uuids:
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+                if len(importer_handler.remaining_data) == 0:
+                    return redirect(url_for('watchlist.index'))
+                else:
+                    remaining_urls = importer_handler.remaining_data
+
+            # Distill.io import
+            if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):
+                # Import and push into the queue for immediate update check
+                d_importer = import_distill_io_json()
+                d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
+                for uuid in d_importer.new_uuids:
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # XLSX importer
+            if request.files and request.files.get('xlsx_file'):
+                file = request.files['xlsx_file']
+
+                if request.values.get('file_mapping') == 'wachete':
+                    w_importer = import_xlsx_wachete()
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+                else:
+                    w_importer = import_xlsx_custom()
+                    # Building mapping of col # to col # type
+                    map = {}
+                    for i in range(10):
+                        c = request.values.get(f"custom_xlsx[col_{i}]")
+                        v = request.values.get(f"custom_xlsx[col_type_{i}]")
+                        if c and v:
+                            map[int(c)] = v
+
+                    w_importer.import_profile = map
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+
+                for uuid in w_importer.new_uuids:
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+        # Could be some remaining, or we could be on GET
+        form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
+        output = render_template("import.html",
+                                form=form,
+                                import_url_list_remaining="\n".join(remaining_urls),
+                                original_distill_json=''
+                                )
+        return output
+
+    return import_blueprint

changedetectionio/blueprint/imports/importer.py

@@ -1,6 +1,5 @@
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 import time
-import validators
 from wtforms import ValidationError
 from loguru import logger
 
@@ -241,7 +240,7 @@ class import_xlsx_custom(Importer):
             return
 
         # @todo cehck atleast 2 rows, same in other method
-        from .forms import validate_url
+        from changedetectionio.forms import validate_url
         row_i = 1
 
         try:
@@ -300,4 +299,4 @@ class import_xlsx_custom(Importer):
             row_i += 1
 
         flash(
-            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))
+            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))

changedetectionio/blueprint/imports/templates/import.html

@@ -13,29 +13,27 @@
     </div>
 
     <div class="box-wrap inner">
-        <form class="pure-form" action="{{url_for('import_page')}}" method="POST" enctype="multipart/form-data">
+        <form class="pure-form" action="{{url_for('imports.import_page')}}" method="POST" enctype="multipart/form-data">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
             <div class="tab-pane-inner" id="url-list">
-                    <legend>
+                <div class="pure-control-group">
                         Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma
                         (,):
                         <br>
-                        <code>https://example.com tag1, tag2, last tag</code>
-                        <br>
+                        <p><strong>Example:  </strong><code>https://example.com tag1, tag2, last tag</code></p>
                         URLs which do not pass validation will stay in the textarea.
-                    </legend>
+                </div>
                 {{ render_field(form.processor, class="processor") }}
-
+                
+                <div class="pure-control-group">
                     <textarea name="urls" class="pure-input-1-2" placeholder="https://"
                               style="width: 100%;
                                 font-family:monospace;
                                 white-space: pre;
                                 overflow-wrap: normal;
                                 overflow-x: scroll;" rows="25">{{ import_url_list_remaining }}</textarea>
-
-<div id="quick-watch-processor-type">
-
-                    </div>
+                 </div>
+                 <div id="quick-watch-processor-type"></div>
 
             </div>
 
@@ -43,7 +41,7 @@
 
 
 
-                    <legend>
+                    <div class="pure-control-group">
                         Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.<br>
                         This is <i>experimental</i>, supported fields are <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, the rest (including <code>schedule</code>) are ignored.
                         <br>
@@ -51,7 +49,7 @@
                         How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br>
                         Be sure to set your default fetcher to Chrome if required.<br>
                         </p>
-                    </legend>
+                    </div>
 
 
                     <textarea name="distill-io" class="pure-input-1-2" style="width: 100%;
@@ -122,4 +120,4 @@
     </div>
 </div>
 
-{% endblock %}
+{% endblock %}

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -20,13 +20,13 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
         datastore.data['watching'][uuid]['processor'] = 'restock_diff'
         datastore.data['watching'][uuid].clear_watch()
         update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-        return redirect(url_for("index"))
+        return redirect(url_for("watchlist.index"))
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
     def reject(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
-        return redirect(url_for("index"))
+        return redirect(url_for("watchlist.index"))
 
 
     return price_data_follower_blueprint

changedetectionio/blueprint/rss/__init__.py

@@ -0,0 +1 @@
+RSS_FORMAT_TYPES = [('plaintext', 'Plain text'), ('html', 'HTML Color')]

changedetectionio/blueprint/rss/blueprint.py

@@ -0,0 +1,147 @@
+
+from changedetectionio.safe_jinja import render as jinja_render
+from changedetectionio.store import ChangeDetectionStore
+from feedgen.feed import FeedGenerator
+from flask import Blueprint, make_response, request, url_for, redirect
+from loguru import logger
+import datetime
+import pytz
+import re
+import time
+
+
+BAD_CHARS_REGEX=r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
+
+# Anything that is not text/UTF-8 should be stripped before it breaks feedgen (such as binary data etc)
+def scan_invalid_chars_in_rss(content):
+    for match in re.finditer(BAD_CHARS_REGEX, content):
+        i = match.start()
+        bad_char = content[i]
+        hex_value = f"0x{ord(bad_char):02x}"
+        # Grab context
+        start = max(0, i - 20)
+        end = min(len(content), i + 21)
+        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
+        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
+        # First match is enough
+        return True
+
+    return False
+
+
+def clean_entry_content(content):
+    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
+    return cleaned
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    rss_blueprint = Blueprint('rss', __name__)
+
+    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
+    # to some earlier blueprint rerouting work, it should goto feed.
+    @rss_blueprint.route("/", methods=['GET'])
+    def extraslash():
+        return redirect(url_for('rss.feed'))
+
+    # Import the login decorator if needed
+    # from changedetectionio.auth_decorator import login_optionally_required
+    @rss_blueprint.route("", methods=['GET'])
+    def feed():
+        now = time.time()
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
+        from changedetectionio import diff
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
+        for uuid, watch in datastore.data['watching'].items():
+            # @todo tag notification_muted skip also (improve Watch model)
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+            if limit_tag and not limit_tag in watch['tags']:
+                continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+
+        html_colour_enable = False
+        if datastore.data['settings']['application'].get('rss_content_format') == 'html':
+            html_colour_enable = True
+
+        for watch in sorted_watches:
+
+            dates = list(watch.history.keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            if not watch.viewed:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
+                fe = fg.add_entry()
+
+                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
+                # Description is the page you watch, link takes you to the diff JS UI page
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+                # @todo fix
+
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                fe.link(link=diff_link)
+
+                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
+
+                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
+                fe.title(title=watch_title)
+                try:
+
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                                                 include_equal=False,
+                                                 line_feed_sep="<br>",
+                                                 html_colour=html_colour_enable
+                                                 )
+                except FileNotFoundError as e:
+                    html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
+
+                # @todo Make this configurable and also consider html-colored markup
+                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
+
+                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+
+                # Out of range chars could also break feedgen
+                if scan_invalid_chars_in_rss(content):
+                    content = clean_entry_content(content)
+
+                fe.content(content=content, type='CDATA')
+                fe.guid(guid, permalink=False)
+                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
+                dt = dt.replace(tzinfo=pytz.UTC)
+                fe.pubDate(dt)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
+        return response
+
+    return rss_blueprint

changedetectionio/blueprint/settings/__init__.py

@@ -0,0 +1,120 @@
+import os
+from copy import deepcopy
+from datetime import datetime
+from zoneinfo import ZoneInfo, available_timezones
+import secrets
+import flask_login
+from flask import Blueprint, render_template, request, redirect, url_for, flash
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
+
+    @settings_blueprint.route("", methods=['GET', "POST"])
+    @login_optionally_required
+    def settings_page():
+        from changedetectionio import forms
+
+        default = deepcopy(datastore.data['settings'])
+        if datastore.proxy_list is not None:
+            available_proxies = list(datastore.proxy_list.keys())
+            # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            # In the case it doesnt exist anymore
+            if not system_proxy in available_proxies:
+                system_proxy = None
+
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0]
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = available_proxies[0]
+
+        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=default,
+                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                        )
+
+        # Remove the last option 'System default'
+        form.application.form.notification_format.choices.pop()
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.requests.form.proxy
+        else:
+            form.requests.form.proxy.choices = []
+            for p in datastore.proxy_list:
+                form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+        if request.method == 'POST':
+            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
+            if form.application.form.data.get('removepassword_button', False):
+                # SALTED_PASS means the password is "locked" to what we set in the Env var
+                if not os.getenv("SALTED_PASS", False):
+                    datastore.remove_password()
+                    flash("Password protection removed.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('settings.settings_page'))
+
+            if form.validate():
+                # Don't set password to False when a password is set - should be only removed with the `removepassword` button
+                app_update = dict(deepcopy(form.data['application']))
+
+                # Never update password with '' or False (Added by wtforms when not in submission)
+                if 'password' in app_update and not app_update['password']:
+                    del (app_update['password'])
+
+                datastore.data['settings']['application'].update(app_update)
+                datastore.data['settings']['requests'].update(form.data['requests'])
+
+                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
+                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+                    datastore.needs_write_urgent = True
+                    flash("Password protection enabled.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('watchlist.index'))
+
+                datastore.needs_write_urgent = True
+                flash("Settings updated.")
+
+            else:
+                flash("An error occurred, please see below.", "error")
+
+        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
+        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
+        output = render_template("settings.html",
+                                api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                available_timezones=sorted(available_timezones()),
+                                emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
+                                form=form,
+                                hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
+                                settings_application=datastore.data['settings']['application'],
+                                timezone_default_config=datastore.data['settings']['application'].get('timezone'),
+                                utc_time=utc_time,
+                                )
+
+        return output
+
+    @settings_blueprint.route("/reset-api-key", methods=['GET'])
+    @login_optionally_required
+    def settings_reset_api_key():
+        secret = secrets.token_hex(16)
+        datastore.data['settings']['application']['api_access_token'] = secret
+        datastore.needs_write_urgent = True
+        flash("API Key was regenerated.")
+        return redirect(url_for('settings.settings_page')+'#api')
+        
+    @settings_blueprint.route("/notification-logs", methods=['GET'])
+    @login_optionally_required
+    def notification_logs():
+        from changedetectionio.flask_app import notification_debug_log
+        output = render_template("notification-log.html",
+                               logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
+        return output
+
+    return settings_blueprint

changedetectionio/blueprint/settings/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -28,7 +28,7 @@
         </ul>
     </div>
     <div class="box-wrap inner">
-        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings.settings_page')}}" method="POST">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
             <div class="tab-pane-inner" id="general">
                 <fieldset>
@@ -78,7 +78,10 @@
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
                     </div>
-
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
@@ -203,7 +206,7 @@ nav
                     </div>
                 </div>
                 <div class="pure-control-group">
-                    <a href="{{url_for('settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
                 </div>
                 <div class="pure-control-group">
                     <h4>Chrome Extension</h4>
@@ -214,7 +217,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
@@ -299,8 +302,8 @@ nav
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
+                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -124,7 +124,7 @@ nav
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
                             <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -47,7 +47,7 @@
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}">{{ tag.title }}</a></td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/blueprint/ui/__init__.py

@@ -0,0 +1,302 @@
+import time
+from flask import Blueprint, request, redirect, url_for, flash, render_template, session
+from loguru import logger
+from functools import wraps
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint
+from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint
+from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData):
+    ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
+    
+    # Register the edit blueprint
+    edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)
+    ui_blueprint.register_blueprint(edit_blueprint)
+    
+    # Register the notification blueprint
+    notification_blueprint = construct_notification_blueprint(datastore)
+    ui_blueprint.register_blueprint(notification_blueprint)
+    
+    # Register the views blueprint
+    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData)
+    ui_blueprint.register_blueprint(views_blueprint)
+    
+    # Import the login decorator
+    from changedetectionio.auth_decorator import login_optionally_required
+
+    @ui_blueprint.route("/clear_history/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def clear_watch_history(uuid):
+        try:
+            datastore.clear_watch_history(uuid)
+        except KeyError:
+            flash('Watch not found', 'error')
+        else:
+            flash("Cleared snapshot history for watch {}".format(uuid))
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
+    @login_optionally_required
+    def clear_all_history():
+        if request.method == 'POST':
+            confirmtext = request.form.get('confirmtext')
+
+            if confirmtext == 'clear':
+                for uuid in datastore.data['watching'].keys():
+                    datastore.clear_watch_history(uuid)
+
+                flash("Cleared snapshot history for all watches")
+            else:
+                flash('Incorrect confirmation text.', 'error')
+
+            return redirect(url_for('watchlist.index'))
+
+        output = render_template("clear_all_history.html")
+        return output
+
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @ui_blueprint.route("/form/mark-all-viewed", methods=['GET'])
+    @login_optionally_required
+    def mark_all_viewed():
+        # Save the current newest history as the most recently viewed
+        with_errors = request.args.get('with_errors') == "1"
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+            datastore.set_last_viewed(watch_uuid, int(time.time()))
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/delete", methods=['GET'])
+    @login_optionally_required
+    def form_delete():
+        uuid = request.args.get('uuid')
+
+        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
+            flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+        datastore.delete(uuid)
+        flash('Deleted.')
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clone", methods=['GET'])
+    @login_optionally_required
+    def form_clone():
+        uuid = request.args.get('uuid')
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        new_uuid = datastore.clone(uuid)
+
+        if not datastore.data['watching'].get(uuid).get('paused'):
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+
+        flash('Cloned, you are editing the new watch.')
+
+        return redirect(url_for("ui.ui_edit.edit_page", uuid=new_uuid))
+
+    @ui_blueprint.route("/checknow", methods=['GET'])
+    @login_optionally_required
+    def form_watch_checknow():
+        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
+        tag = request.args.get('tag')
+        uuid = request.args.get('uuid')
+        with_errors = request.args.get('with_errors') == "1"
+
+        i = 0
+
+        running_uuids = []
+        for t in running_update_threads:
+            running_uuids.append(t.current_uuid)
+
+        if uuid:
+            if uuid not in running_uuids:
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                i += 1
+
+        else:
+            # Recheck all, including muted
+            for watch_uuid, watch in datastore.data['watching'].items():
+                if not watch['paused']:
+                    if watch_uuid not in running_uuids:
+                        if with_errors and not watch.get('last_error'):
+                            continue
+
+                        if tag != None and tag not in watch['tags']:
+                            continue
+
+                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                        i += 1
+
+        if i == 1:
+            flash("Queued 1 watch for rechecking.")
+        if i > 1:
+            flash("Queued {} watches for rechecking.".format(i))
+        if i == 0:
+            flash("No watches available to recheck.")
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/form/checkbox-operations", methods=['POST'])
+    @login_optionally_required
+    def form_watch_list_checkbox_operations():
+        op = request.form['op']
+        uuids = request.form.getlist('uuids')
+
+        if (op == 'delete'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.delete(uuid.strip())
+            flash("{} watches deleted".format(len(uuids)))
+
+        elif (op == 'pause'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['paused'] = True
+            flash("{} watches paused".format(len(uuids)))
+
+        elif (op == 'unpause'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['paused'] = False
+            flash("{} watches unpaused".format(len(uuids)))
+
+        elif (op == 'mark-viewed'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.set_last_viewed(uuid, int(time.time()))
+            flash("{} watches updated".format(len(uuids)))
+
+        elif (op == 'mute'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['notification_muted'] = True
+            flash("{} watches muted".format(len(uuids)))
+
+        elif (op == 'unmute'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['notification_muted'] = False
+            flash("{} watches un-muted".format(len(uuids)))
+
+        elif (op == 'recheck'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    # Recheck and require a full reprocessing
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            flash("{} watches queued for rechecking".format(len(uuids)))
+
+        elif (op == 'clear-errors'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid]["last_error"] = False
+            flash(f"{len(uuids)} watches errors cleared")
+
+        elif (op == 'clear-history'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.clear_watch_history(uuid)
+            flash("{} watches cleared/reset.".format(len(uuids)))
+
+        elif (op == 'notification-default'):
+            from changedetectionio.notification import (
+                default_notification_format_for_watch
+            )
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['notification_title'] = None
+                    datastore.data['watching'][uuid.strip()]['notification_body'] = None
+                    datastore.data['watching'][uuid.strip()]['notification_urls'] = []
+                    datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
+            flash("{} watches set to use default notification settings".format(len(uuids)))
+
+        elif (op == 'assign-tag'):
+            op_extradata = request.form.get('op_extradata', '').strip()
+            if op_extradata:
+                tag_uuid = datastore.add_tag(title=op_extradata)
+                if op_extradata and tag_uuid:
+                    for uuid in uuids:
+                        uuid = uuid.strip()
+                        if datastore.data['watching'].get(uuid):
+                            # Bug in old versions caused by bad edit page/tag handler
+                            if isinstance(datastore.data['watching'][uuid]['tags'], str):
+                                datastore.data['watching'][uuid]['tags'] = []
+
+                            datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+
+            flash(f"{len(uuids)} watches were tagged")
+
+        return redirect(url_for('watchlist.index'))
+
+
+    @ui_blueprint.route("/share-url/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_share_put_watch(uuid):
+        """Given a watch UUID, upload the info and return a share-link
+           the share-link can be imported/added"""
+        import requests
+        import json
+        from copy import deepcopy
+
+        # more for testing
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        # copy it to memory as trim off what we dont need (history)
+        watch = deepcopy(datastore.data['watching'].get(uuid))
+        # For older versions that are not a @property
+        if (watch.get('history')):
+            del (watch['history'])
+
+        # for safety/privacy
+        for k in list(watch.keys()):
+            if k.startswith('notification_'):
+                del watch[k]
+
+        for r in['uuid', 'last_checked', 'last_changed']:
+            if watch.get(r):
+                del (watch[r])
+
+        # Add the global stuff which may have an impact
+        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
+        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
+
+        watch_json = json.dumps(watch)
+
+        try:
+            r = requests.request(method="POST",
+                                 data={'watch': watch_json},
+                                 url="https://changedetection.io/share/share",
+                                 headers={'App-Guid': datastore.data['app_guid']})
+            res = r.json()
+
+            # Add to the flask session
+            session['share-link'] = f"https://changedetection.io/share/{res['share_key']}"
+
+
+        except Exception as e:
+            logger.error(f"Error sharing -{str(e)}")
+            flash(f"Could not share, something went wrong while communicating with the share server - {str(e)}", 'error')
+
+        return redirect(url_for('watchlist.index'))
+
+    return ui_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -0,0 +1,333 @@
+import time
+from copy import deepcopy
+import os
+import importlib.resources
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+from loguru import logger
+from jinja2 import Environment, FileSystemLoader
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.time_handler import is_within_schedule
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates")
+    
+    def _watch_has_tag_options_set(watch):
+        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
+        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
+                return True
+
+    @edit_blueprint.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    @login_optionally_required
+    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
+    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
+    def edit_page(uuid):
+        from changedetectionio import forms
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
+        from changedetectionio import processors
+        import importlib
+
+        # More for testing, possible to return the first/only
+        if not datastore.data['watching'].keys():
+            flash("No watches to edit", "error")
+            return redirect(url_for('watchlist.index'))
+
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        if not uuid in datastore.data['watching']:
+            flash("No watch with the UUID %s found." % (uuid), "error")
+            return redirect(url_for('watchlist.index'))
+
+        switch_processor = request.args.get('switch_processor')
+        if switch_processor:
+            for p in processors.available_processors():
+                if p[0] == switch_processor:
+                    datastore.data['watching'][uuid]['processor'] = switch_processor
+                    flash(f"Switched to mode - {p[1]}.")
+                    datastore.clear_watch_history(uuid)
+                    redirect(url_for('ui_edit.edit_page', uuid=uuid))
+
+        # be sure we update with a copy instead of accidently editing the live object by reference
+        default = deepcopy(datastore.data['watching'][uuid])
+
+        # Defaults for proxy choice
+        if datastore.proxy_list is not None:  # When enabled
+            # @todo
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''
+        # proxy_override set to the json/text list of the items
+
+        # Does it use some custom form? does one exist?
+        processor_name = datastore.data['watching'][uuid].get('processor', '')
+        processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)
+        if not processor_classes:
+            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
+            return redirect(url_for('watchlist.index'))
+
+        parent_module = processors.get_parent_module(processor_classes[0])
+
+        try:
+            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+            # Access the 'processor_settings_form' class from the 'forms' module
+            form_class = getattr(forms_module, 'processor_settings_form')
+        except ModuleNotFoundError as e:
+            # .forms didnt exist
+            form_class = forms.processor_text_json_diff_form
+        except AttributeError as e:
+            # .forms exists but no useful form
+            form_class = forms.processor_text_json_diff_form
+
+        form = form_class(formdata=request.form if request.method == 'POST' else None,
+                          data=default,
+                          extra_notification_tokens=default.extra_notification_token_values(),
+                          default_system_settings=datastore.data['settings']
+                          )
+
+        # For the form widget tag UUID back to "string name" for the field
+        form.tags.datastore = datastore
+
+        # Used by some forms that need to dig deeper
+        form.datastore = datastore
+        form.watch = default
+
+        for p in datastore.extra_browsers:
+            form.fetch_backend.choices.append(p)
+
+        form.fetch_backend.choices.append(("system", 'System settings default'))
+
+        # form.browser_steps[0] can be assumed that we 'goto url' first
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.proxy
+        else:
+            form.proxy.choices = [('', 'Default')]
+            for p in datastore.proxy_list:
+                form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+
+        if request.method == 'POST' and form.validate():
+
+            # If they changed processor, it makes sense to reset it.
+            if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
+                datastore.data['watching'][uuid].clear_watch()
+                flash("Reset watch history due to change of processor")
+
+            extra_update_obj = {
+                'consecutive_filter_failures': 0,
+                'last_error' : False
+            }
+
+            if request.args.get('unpause_on_save'):
+                extra_update_obj['paused'] = False
+
+            extra_update_obj['time_between_check'] = form.time_between_check.data
+
+             # Ignore text
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+
+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
+            # Unsetting all filter_text methods should make it go back to default
+            # This particularly affects tests running
+            if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \
+                    and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \
+                    and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'):
+                extra_update_obj['filter_text_added'] = True
+                extra_update_obj['filter_text_replaced'] = True
+                extra_update_obj['filter_text_removed'] = True
+
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(title=t))
+                    extra_update_obj['tags'] = tag_uuids
+
+            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(extra_update_obj)
+
+            if not datastore.data['watching'][uuid].get('tags'):
+                # Force it to be a list, because form.data['tags'] will be string if nothing found
+                # And del(form.data['tags'] ) wont work either for some reason
+                datastore.data['watching'][uuid]['tags'] = []
+
+            # Recast it if need be to right data Watch handler
+            watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor'))
+            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid])
+            flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
+
+            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
+            # But in the case something is added we should save straight away
+            datastore.needs_write_urgent = True
+
+            # Do not queue on edit if its not within the time range
+
+            # @todo maybe it should never queue anyway on edit...
+            is_in_schedule = True
+            watch = datastore.data['watching'].get(uuid)
+
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+
+            tz_name = time_schedule_limit.get('timezone')
+            if not tz_name:
+                tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                      default_tz=tz_name
+                                                      )
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+
+            #############################
+            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
+                # Queue the watch for immediate recheck, with a higher priority
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid))
+
+            return redirect(url_for('watchlist.index', tag=request.args.get("tag",'')))
+
+        else:
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
+
+
+            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
+            jq_support = True
+            try:
+                import jq
+            except ModuleNotFoundError:
+                jq_support = False
+
+            watch = datastore.data['watching'].get(uuid)
+
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+            watch_uses_webdriver = False
+            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+                watch_uses_webdriver = True
+
+            from zoneinfo import available_timezones
+
+            # Only works reliably with Playwright
+
+            template_args = {
+                'available_processors': processors.available_processors(),
+                'available_timezones': sorted(available_timezones()),
+                'browser_steps_config': browser_step_ui_config,
+                'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_title': f" - Edit - {watch.label}",
+                'form': form,
+                'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+                'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
+                'watch_uses_webdriver': watch_uses_webdriver,
+                'jq_support': jq_support,
+                'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
+                'settings_application': datastore.data['settings']['application'],
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
+                'using_global_webdriver_wait': not default['webdriver_delay'],
+                'uuid': uuid,
+                'watch': watch
+            }
+
+            included_content = None
+            if form.extra_form_content():
+                # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+                # And then render the code from the module
+                templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+                env = Environment(loader=FileSystemLoader(templates_dir))
+                template = env.from_string(form.extra_form_content())
+                included_content = template.render(**template_args)
+
+            output = render_template("edit.html",
+                                     extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                     extra_form_content=included_content,
+                                     **template_args
+                                     )
+
+        return output
+
+    @edit_blueprint.route("/edit/<string:uuid>/get-html", methods=['GET'])
+    @login_optionally_required
+    def watch_get_latest_html(uuid):
+        from io import BytesIO
+        from flask import send_file
+        import brotli
+
+        watch = datastore.data['watching'].get(uuid)
+        if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
+            latest_filename = list(watch.history.keys())[-1]
+            html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                if html_fname.endswith('.br'):
+                    # Read and decompress the Brotli file
+                    decompressed_data = brotli.decompress(f.read())
+                else:
+                    decompressed_data = f.read()
+
+            buffer = BytesIO(decompressed_data)
+
+            return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html')
+
+        # Return a 500 error
+        abort(500)
+
+    # Ajax callback
+    @edit_blueprint.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
+    @login_optionally_required
+    def watch_get_preview_rendered(uuid):
+        '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        from flask import jsonify
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
+        return jsonify(result)
+
+    @edit_blueprint.route("/highlight_submit_ignore_url", methods=['POST'])
+    @login_optionally_required
+    def highlight_submit_ignore_url():
+        import re
+        mode = request.form.get('mode')
+        selection = request.form.get('selection')
+
+        uuid = request.args.get('uuid','')
+        if datastore.data["watching"].get(uuid):
+            if mode == 'exact':
+                for l in selection.splitlines():
+                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+            elif mode == 'digit-regex':
+                for l in selection.splitlines():
+                    # Replace any series of numbers with a regex
+                    s = re.escape(l.strip())
+                    s = re.sub(r'[0-9]+', r'\\d+', s)
+                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+
+        return f"<a href={url_for('ui.ui_views.preview_page', uuid=uuid)}>Click to preview</a>"
+    
+    return edit_blueprint

changedetectionio/blueprint/ui/notification.py

@@ -0,0 +1,106 @@
+from flask import Blueprint, request, make_response
+import random
+from loguru import logger
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.notification import process_notification
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates")
+    
+    # AJAX endpoint for sending a test
+    @notification_blueprint.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test/", methods=['POST'])
+    @login_optionally_required
+    def ajax_callback_send_notification_test(watch_uuid=None):
+
+        # Watch_uuid could be unset in the case it`s used in tag editor, global settings
+        import apprise
+        from ...apprise_plugin.assets import apprise_asset
+        from ...apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
+        apobj = apprise.Apprise(asset=apprise_asset)
+
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
+
+        # Use an existing random one on the global/main settings form
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
+                and datastore.data.get('watching'):
+            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+            watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+
+        if not watch_uuid:
+            return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
+
+        watch = datastore.data['watching'].get(watch_uuid)
+
+        notification_urls = None
+
+        if request.form.get('notification_urls'):
+            notification_urls = request.form['notification_urls'].strip().splitlines()
+
+        if not notification_urls:
+            logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # On an edit page, we should also fire off to the tags if they have notifications
+            if request.form.get('tags') and request.form['tags'].strip():
+                for k in request.form['tags'].split(','):
+                    tag = datastore.tag_exists_by_name(k.strip())
+                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
+
+        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
+            # In the global settings, use only what is typed currently in the text box
+            logger.debug("Test notification - Trying by global system settings notifications")
+            if datastore.data['settings']['application'].get('notification_urls'):
+                notification_urls = datastore.data['settings']['application']['notification_urls']
+
+        if not notification_urls:
+            return 'Error: No Notification URLs set/found'
+
+        for n_url in notification_urls:
+            if len(n_url.strip()):
+                if not apobj.add(n_url):
+                    return f'Error:  {n_url} is not a valid AppRise URL.'
+
+        try:
+            # use the same as when it is triggered, but then override it with the form test values
+            n_object = {
+                'watch_url': request.form.get('window_url', "https://changedetection.io"),
+                'notification_urls': notification_urls
+            }
+
+            # Only use if present, if not set in n_object it should use the default system value
+            if 'notification_format' in request.form and request.form['notification_format'].strip():
+                n_object['notification_format'] = request.form.get('notification_format', '').strip()
+
+            if 'notification_title' in request.form and request.form['notification_title'].strip():
+                n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"
+
+            if 'notification_body' in request.form and request.form['notification_body'].strip():
+                n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"
+
+            n_object['as_async'] = False
+            n_object.update(watch.extra_notification_token_values())
+            sent_obj = process_notification(n_object, datastore)
+
+        except Exception as e:
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)
+
+        return 'OK - Sent test notifications'
+
+    return notification_blueprint

changedetectionio/blueprint/ui/templates/clear_all_history.html

@@ -3,7 +3,7 @@
   <div class="box-wrap inner">
     <form
       class="pure-form pure-form-stacked"
-      action="{{url_for('clear_all_history')}}"
+      action="{{url_for('ui.clear_all_history')}}"
       method="POST"
     >
       <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
@@ -37,7 +37,7 @@
         </div>
         <br />
         <div class="pure-control-group">
-          <a href="{{url_for('index')}}" class="pure-button button-cancel"
+          <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel"
             >Cancel</a
           >
         </div>

changedetectionio/blueprint/ui/views.py

@@ -0,0 +1,220 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+from flask_login import current_user
+import os
+import time
+from copy import deepcopy
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio import html_tools
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
+    
+    @views_blueprint.route("/preview/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def preview_page(uuid):
+        content = []
+        versions = []
+        timestamp = None
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+        triggered_line_numbers = []
+        if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
+            flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
+        else:
+            # So prepare the latest preview or not
+            preferred_version = request.args.get('version')
+            versions = list(watch.history.keys())
+            timestamp = versions[-1]
+            if preferred_version and preferred_version in versions:
+                timestamp = preferred_version
+
+            try:
+                versions = list(watch.history.keys())
+                content = watch.get_history_snapshot(timestamp)
+
+                triggered_line_numbers = html_tools.strip_ignore_text(content=content,
+                                                                      wordlist=watch['trigger_text'],
+                                                                      mode='line numbers'
+                                                                      )
+
+            except Exception as e:
+                content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
+
+        output = render_template("preview.html",
+                                 content=content,
+                                 current_version=timestamp,
+                                 history_n=watch.history_n,
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label} @ {timestamp}",
+                                 triggered_line_numbers=triggered_line_numbers,
+                                 current_diff_url=watch['url'],
+                                 screenshot=watch.get_screenshot(),
+                                 watch=watch,
+                                 uuid=uuid,
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_text=watch.get_error_text(),
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 versions=versions
+                                )
+
+        return output
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET', 'POST'])
+    @login_optionally_required
+    def diff_history_page(uuid):
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(request.form)
+        if request.method == 'POST':
+            if not extract_form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            else:
+                extract_regex = request.form.get('extract_regex').strip()
+                output = watch.extract_regex_from_all_history(extract_regex)
+                if output:
+                    watch_dir = os.path.join(datastore.datastore_path, uuid)
+                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                    response.headers['Content-type'] = 'text/csv'
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = 0
+                    return response
+
+                flash('Nothing matches that RegEx', 'error')
+                redirect(url_for('ui_views.diff_history_page', uuid=uuid)+'#extract')
+
+        history = watch.history
+        dates = list(history.keys())
+
+        if len(dates) < 2:
+            flash("Not enough saved change detection snapshots to produce a report.", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # Save the current newest history as the most recently viewed
+        datastore.set_last_viewed(uuid, time.time())
+
+        # Read as binary and force decode as UTF-8
+        # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
+        from_version = request.args.get('from_version')
+        from_version_index = -2  # second newest
+        if from_version and from_version in dates:
+            from_version_index = dates.index(from_version)
+        else:
+            from_version = dates[from_version_index]
+
+        try:
+            from_version_file_contents = watch.get_history_snapshot(dates[from_version_index])
+        except Exception as e:
+            from_version_file_contents = f"Unable to read to-version at index {dates[from_version_index]}.\n"
+
+        to_version = request.args.get('to_version')
+        to_version_index = -1
+        if to_version and to_version in dates:
+            to_version_index = dates.index(to_version)
+        else:
+            to_version = dates[to_version_index]
+
+        try:
+            to_version_file_contents = watch.get_history_snapshot(dates[to_version_index])
+        except Exception as e:
+            to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index])
+
+        screenshot_url = watch.get_screenshot()
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+
+        password_enabled_and_share_is_off = False
+        if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+            password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
+
+        output = render_template("diff.html",
+                                 current_diff_url=watch['url'],
+                                 from_version=str(from_version),
+                                 to_version=str(to_version),
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label}",
+                                 extract_form=extract_form,
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 last_error_text=watch.get_error_text(),
+                                 left_sticky=True,
+                                 newest=to_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 password_enabled_and_share_is_off=password_enabled_and_share_is_off,
+                                 from_version_file_contents=from_version_file_contents,
+                                 to_version_file_contents=to_version_file_contents,
+                                 screenshot=screenshot_url,
+                                 uuid=uuid,
+                                 versions=dates, # All except current/last
+                                 watch_a=watch
+                                 )
+
+        return output
+
+    @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
+    @login_optionally_required
+    def form_quick_watch_add():
+        from changedetectionio import forms
+        form = forms.quickWatchForm(request.form)
+
+        if not form.validate():
+            for widget, l in form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            flash(f'Warning, URL {url} already exists', "notice")
+
+        add_paused = request.form.get('edit_and_watch_submit_button') != None
+        processor = request.form.get('processor', 'text_json_diff')
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+
+        if new_uuid:
+            if add_paused:
+                flash('Watch added in Paused state, saving will unpause.')
+                return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
+            else:
+                # Straight into the queue.
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+                flash("Watch added.")
+
+        return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
+
+    return views_blueprint

changedetectionio/blueprint/watchlist/__init__.py

@@ -0,0 +1,111 @@
+import os
+import time
+
+from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
+from flask_login import current_user
+from flask_paginate import Pagination, get_page_parameter
+
+from changedetectionio import forms
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    watchlist_blueprint = Blueprint('watchlist', __name__, template_folder="templates")
+    
+    @watchlist_blueprint.route("/", methods=['GET'])
+    @login_optionally_required
+    def index():
+        active_tag_req = request.args.get('tag', '').lower().strip()
+        active_tag_uuid = active_tag = None
+
+        # Be sure limit_tag is a uuid
+        if active_tag_req:
+            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
+                    active_tag = tag
+                    active_tag_uuid = uuid
+                    break
+
+        # Redirect for the old rss path which used the /?rss=true
+        if request.args.get('rss'):
+            return redirect(url_for('rss.feed', tag=active_tag_uuid))
+
+        op = request.args.get('op')
+        if op:
+            uuid = request.args.get('uuid')
+            if op == 'pause':
+                datastore.data['watching'][uuid].toggle_pause()
+            elif op == 'mute':
+                datastore.data['watching'][uuid].toggle_mute()
+
+            datastore.needs_write = True
+            return redirect(url_for('watchlist.index', tag = active_tag_uuid))
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+        with_errors = request.args.get('with_errors') == "1"
+        errored_count = 0
+        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
+        for uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+                    continue
+            if watch.get('last_error'):
+                errored_count += 1
+
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                    sorted_watches.append(watch)
+                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
+                    sorted_watches.append(watch)
+            else:
+                sorted_watches.append(watch)
+
+        form = forms.quickWatchForm(request.form)
+        page = request.args.get(get_page_parameter(), type=int, default=1)
+        total_count = len(sorted_watches)
+
+        pagination = Pagination(page=page,
+                                total=total_count,
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+        output = render_template(
+            "watch-overview.html",
+                                 active_tag=active_tag,
+                                 active_tag_uuid=active_tag_uuid,
+                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+                                 datastore=datastore,
+                                 errored_count=errored_count,
+                                 form=form,
+                                 guid=datastore.data['app_guid'],
+                                 has_proxies=datastore.proxy_list,
+                                 has_unviewed=datastore.has_unviewed,
+                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 now_time_server=time.time(),
+                                 pagination=pagination,
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 search_q=request.args.get('q', '').strip(),
+                                 sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
+                                 sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
+                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+                                 tags=sorted_tags,
+                                 watches=sorted_watches
+                                 )
+
+        if session.get('share-link'):
+            del(session['share-link'])
+
+        resp = make_response(output)
+
+        # The template can run on cookie or url query info
+        if request.args.get('sort'):
+            resp.set_cookie('sort', request.args.get('sort'))
+        if request.args.get('order'):
+            resp.set_cookie('order', request.args.get('order'))
+
+        return resp
+        
+    return watchlist_blueprint

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -3,10 +3,19 @@
 {% from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+<script>let nowtimeserver={{ now_time_server }};</script>
 
+<style>
+.checking-now .last-checked {
+    background-image: linear-gradient(to bottom, transparent 0%, rgba(0,0,0,0.05) 40%, rgba(0,0,0,0.1) 100%);
+    background-size: 0 100%;
+    background-repeat: no-repeat;
+    transition: background-size 0.9s ease
+}
+</style>
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -25,7 +34,7 @@
         <span style="color:#eee; font-size: 80%;"><img alt="Create a shareable link" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></span>
     </form>
 
-    <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
+    <form class="pure-form" action="{{ url_for('ui.form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
     <input type="hidden" id="op_extradata" name="op_extradata" value="" >
     <div id="checkbox-operations">
@@ -46,12 +55,12 @@
     {% endif %}
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
-        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
+        <a href="{{url_for('watchlist.index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
 
     <!-- tag list -->
     {% for uuid, tag in tags %}
         {% if tag != "" %}
-            <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
+            <a href="{{url_for('watchlist.index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
         {% endif %}
     {% endfor %}
     </div>
@@ -72,27 +81,27 @@
             <tr>
                 {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
                 {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
                 <th class="empty-cell"></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
              {% if any_has_restock_price_processor %}
                 <th>Restock &amp; Price</th>
              {% endif %}
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th class="empty-cell"></th>
             </tr>
             </thead>
             <tbody>
             {% if not watches|length %}
             <tr>
-                <td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+                <td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('imports.import_page')}}" >import a list</a>.</td>
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
 
-                {% set is_unviewed =  watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
-
+                {% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
+                {% set checking_now = is_checking_now(watch) %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
@@ -100,26 +109,28 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if is_unviewed %}unviewed{% endif %}
                 {% if watch.has_restock_info %} has-restock-info {% if watch['restock']['in_stock'] %}in-stock{% else %}not-in-stock{% endif %} {% else %}no-restock-info{% endif %}
-                {% if watch.uuid in queued_uuids %}queued{% endif %}">
+                {% if watch.uuid in queued_uuids %}queued{% endif %}
+                {% if checking_now %}checking-now{% endif %}
+                ">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="state-off" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="state-on" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
                     {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
-                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
+                    <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
 
                     {% if watch.get_fetch_backend == "html_webdriver"
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -129,9 +140,9 @@
 
                         {% if '403' in watch.last_error %}
                             {% if has_proxies %}
-                                <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
+                                <a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
                             {% endif %}
-                            <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
+                            <a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
                         
                         {% endif %}
                         {% if 'empty result or contain only an image' in watch.last_error %}
@@ -140,7 +151,7 @@
                     </div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
-                    <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
+                    <div class="fetch-error notification-error"><a href="{{url_for('settings.notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
 
                     {% if watch['processor'] == 'text_json_diff'  %}
@@ -178,7 +189,14 @@
                     {% endif %}
                 </td>
 {% endif %}
-                <td class="last-checked" data-timestamp="{{ watch.last_checked }}">{{watch|format_last_checked_time|safe}}</td>
+            {#last_checked becomes fetch-start-time#}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" {% if checking_now %} data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" {% endif %} >
+                    {% if checking_now %}
+                        <span class="spinner"></span><span> Checking now</span>
+                    {% else %}
+                        {{watch|format_last_checked_time|safe}}</td>
+                    {% endif %}
+
                 <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
@@ -186,20 +204,20 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% endif %}
 
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
-                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>
+                            <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -210,20 +228,20 @@
         <ul id="post-list-buttons">
             {% if errored_count %}
             <li>
-                <a href="{{url_for('index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
+                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
             </li>
             {% endif %}
             {% if has_unviewed %}
             <li>
-                <a href="{{url_for('mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('rss', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+                <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
         {{ pagination.links }}

changedetectionio/conditions/__init__.py

@@ -116,8 +116,7 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat
             if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
                 result = False
 
-    return result
-
+    return {'executed_data': EXECUTE_DATA, 'result': result}
 
 # Load plugins dynamically
 for plugin in plugin_manager.get_plugins():

changedetectionio/conditions/blueprint.py

@@ -46,6 +46,8 @@ def construct_blueprint(datastore):
             # Override the conditions in the temporary watch
             rule_json = request.args.get("rule")
             rule = json.loads(rule_json) if rule_json else None
+
+            # Should be key/value of field, operator, value
             tmp_watch_data['conditions'] = [rule]
             tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
 
@@ -65,7 +67,8 @@ def construct_blueprint(datastore):
 
             return jsonify({
                 'status': 'success',
-                'result': result,
+                'result': result.get('result'),
+                'data': result.get('executed_data'),
                 'message': 'Condition passes' if result else 'Condition does not pass'
             })
 

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -75,13 +75,19 @@ function isItemInStock() {
         'rupture',
         'sold out',
         'sold-out',
+        'stok habis',
+        'stok kosong',
+        'stok varian ini habis',
         'stokta yok',
         'temporarily out of stock',
         'temporarily unavailable',
         'there were no search results for',
         'this item is currently unavailable',
         'tickets unavailable',
+        'tidak dijual',
+        'tidak tersedia',
         'tijdelijk uitverkocht',
+        'tiket tidak tersedia',
         'tükendi',
         'unavailable nearby',
         'unavailable tickets',

changedetectionio/forms.py

@@ -3,6 +3,7 @@ import re
 from loguru import logger
 from wtforms.widgets.core import TimeInput
 
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.strtobool import strtobool
 
@@ -305,10 +306,10 @@ class ValidateAppRiseServers(object):
 
     def __call__(self, form, field):
         import apprise
-        apobj = apprise.Apprise()
+        from .apprise_plugin.assets import apprise_asset
+        from .apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
 
-        # so that the custom endpoints are registered
-        from .apprise_asset import asset
+        apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
             url = server_url.strip()
@@ -739,6 +740,9 @@ class globalSettingsApplicationForm(commonSettingsForm):
                               render_kw={"style": "width: 5em;"},
                               validators=[validators.NumberRange(min=0,
                                                                  message="Should be atleast zero (disabled)")])
+
+    rss_content_format = SelectField('RSS Content format', choices=RSS_FORMAT_TYPES)
+
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])

changedetectionio/gc_cleanup.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+import ctypes
+import gc
+import re
+import psutil
+import sys
+import threading
+import importlib
+from loguru import logger
+
+def memory_cleanup(app=None):
+    """
+    Perform comprehensive memory cleanup operations and log memory usage
+    at each step with nicely formatted numbers.
+    
+    Args:
+        app: Optional Flask app instance for clearing Flask-specific caches
+        
+    Returns:
+        str: Status message
+    """
+    # Get current process
+    process = psutil.Process()
+    
+    # Log initial memory usage with nicely formatted numbers
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"Memory cleanup started - Current memory usage: {current_memory:,.2f} MB")
+
+    # 1. Standard garbage collection - force full collection on all generations
+    gc.collect(0)  # Collect youngest generation
+    gc.collect(1)  # Collect middle generation
+    gc.collect(2)  # Collect oldest generation
+
+    # Run full collection again to ensure maximum cleanup
+    gc.collect()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After full gc.collect() - Memory usage: {current_memory:,.2f} MB")
+    
+
+    # 3. Call libc's malloc_trim to release memory back to the OS
+    libc = ctypes.CDLL("libc.so.6")
+    libc.malloc_trim(0)
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After malloc_trim(0) - Memory usage: {current_memory:,.2f} MB")
+    
+    # 4. Clear Python's regex cache
+    re.purge()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After re.purge() - Memory usage: {current_memory:,.2f} MB")
+
+    # 5. Reset thread-local storage
+    # Create a new thread local object to encourage cleanup of old ones
+    threading.local()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After threading.local() - Memory usage: {current_memory:,.2f} MB")
+
+    # 6. Clear sys.intern cache if Python version supports it
+    try:
+        sys.intern.clear()
+        current_memory = process.memory_info().rss / 1024 / 1024
+        logger.debug(f"After sys.intern.clear() - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("sys.intern.clear() not supported in this Python version")
+    
+    # 7. Clear XML/lxml caches if available
+    try:
+        # Check if lxml.etree is in use
+        lxml_etree = sys.modules.get('lxml.etree')
+        if lxml_etree:
+            # Clear module-level caches
+            if hasattr(lxml_etree, 'clear_error_log'):
+                lxml_etree.clear_error_log()
+            
+            # Check for _ErrorLog and _RotatingErrorLog objects and clear them
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('_ErrorLog', '_RotatingErrorLog', '_DomainErrorLog') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+                    
+                    # Clear Element objects which can hold references to documents
+                    elif class_name in ('_Element', 'ElementBase') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.etree cleanup - Memory usage: {current_memory:,.2f} MB")
+
+        # Check if lxml.html is in use
+        lxml_html = sys.modules.get('lxml.html')
+        if lxml_html:
+            # Clear HTML-specific element types
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('HtmlElement', 'FormElement', 'InputElement',
+                                    'SelectElement', 'TextareaElement', 'CheckboxGroup',
+                                    'RadioGroup', 'MultipleSelectOptions', 'FieldsDict') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.html cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (ImportError, AttributeError):
+        logger.debug("lxml cleanup not applicable")
+    
+    # 8. Clear JSON parser caches if applicable
+    try:
+        # Check if json module is being used and try to clear its cache
+        json_module = sys.modules.get('json')
+        if json_module and hasattr(json_module, '_default_encoder'):
+            json_module._default_encoder.markers.clear()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After JSON parser cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, KeyError):
+        logger.debug("JSON cleanup not applicable")
+    
+    # 9. Force Python's memory allocator to release unused memory
+    try:
+        if hasattr(sys, 'pypy_version_info'):
+            # PyPy has different memory management
+            gc.collect()
+        else:
+            # CPython - try to release unused memory
+            ctypes.pythonapi.PyGC_Collect()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After PyGC_Collect - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("PyGC_Collect not supported")
+    
+    # 10. Clear Flask-specific caches if applicable
+    if app:
+        try:
+            # Clear Flask caches if they exist
+            for key in list(app.config.get('_cache', {}).keys()):
+                app.config['_cache'].pop(key, None)
+            
+            # Clear Jinja2 template cache if available
+            if hasattr(app, 'jinja_env') and hasattr(app.jinja_env, 'cache'):
+                app.jinja_env.cache.clear()
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After Flask cache clear - Memory usage: {current_memory:,.2f} MB")
+        except (AttributeError, KeyError):
+            logger.debug("No Flask cache to clear")
+    
+    # Final garbage collection pass
+    gc.collect()
+    libc.malloc_trim(0)
+    
+    # Log final memory usage
+    final_memory = process.memory_info().rss / 1024 / 1024
+    logger.info(f"Memory cleanup completed - Final memory usage: {final_memory:,.2f} MB")
+    return "cleaned"

changedetectionio/model/App.py

@@ -1,4 +1,7 @@
 from os import getenv
+
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
+
 from changedetectionio.notification import (
     default_notification_body,
     default_notification_format,
@@ -9,6 +12,8 @@ from changedetectionio.notification import (
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
+
+
 class model(dict):
     base_config = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
@@ -48,6 +53,7 @@ class model(dict):
                     'password': False,
                     'render_anchor_tag_content': False,
                     'rss_access_token': None,
+                    'rss_content_format': RSS_FORMAT_TYPES[0][0],
                     'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,

changedetectionio/model/Watch.py

@@ -83,7 +83,7 @@ class model(watch_base):
                     flash, Markup, url_for
                 )
                 message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
-                    url_for('edit_page', uuid=self.get('uuid')), self.get('url', '')))
+                    url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', '')))
                 flash(message, 'error')
                 return ''
 
@@ -296,11 +296,11 @@ class model(watch_base):
         with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
             return f.read()
 
-    # Save some text file to the appropriate path and bump the history
+   # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
     def save_history_text(self, contents, timestamp, snapshot_id):
         import brotli
-
+        import tempfile
         logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
 
         self.ensure_data_dir_exists()
@@ -308,26 +308,37 @@ class model(watch_base):
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
 
+        # Decide on snapshot filename and destination path
         if not skip_brotli and len(contents) > threshold:
             snapshot_fname = f"{snapshot_id}.txt.br"
-            dest = os.path.join(self.watch_data_dir, snapshot_fname)
-            if not os.path.exists(dest):
-                with open(dest, 'wb') as f:
-                    f.write(brotli.compress(contents.encode('utf-8'), mode=brotli.MODE_TEXT))
+            encoded_data = brotli.compress(contents.encode('utf-8'), mode=brotli.MODE_TEXT)
         else:
             snapshot_fname = f"{snapshot_id}.txt"
-            dest = os.path.join(self.watch_data_dir, snapshot_fname)
-            if not os.path.exists(dest):
-                with open(dest, 'wb') as f:
-                    f.write(contents.encode('utf-8'))
+            encoded_data = contents.encode('utf-8')
 
-        # Append to index
-        # @todo check last char was \n
+        dest = os.path.join(self.watch_data_dir, snapshot_fname)
+
+        # Write snapshot file atomically if it doesn't exist
+        if not os.path.exists(dest):
+            with tempfile.NamedTemporaryFile('wb', delete=False, dir=self.watch_data_dir) as tmp:
+                tmp.write(encoded_data)
+                tmp.flush()
+                os.fsync(tmp.fileno())
+                tmp_path = tmp.name
+            os.rename(tmp_path, dest)
+
+        # Append to history.txt atomically
         index_fname = os.path.join(self.watch_data_dir, "history.txt")
-        with open(index_fname, 'a') as f:
-            f.write("{},{}\n".format(timestamp, snapshot_fname))
-            f.close()
+        index_line = f"{timestamp},{snapshot_fname}\n"
 
+        # Lets try force flush here since it's usually a very small file
+        # If this still fails in the future then try reading all to memory first, re-writing etc
+        with open(index_fname, 'a', encoding='utf-8') as f:
+            f.write(index_line)
+            f.flush()
+            os.fsync(f.fileno())
+
+        # Update internal state
         self.__newest_history_key = timestamp
         self.__history_n += 1
 
@@ -564,7 +575,7 @@ class model(watch_base):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
-        if not os.path.isfile(filepath):
+        if not os.path.isfile(filepath) or os.path.getsize(filepath) == 0:
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):

changedetectionio/notification.py

@@ -4,6 +4,9 @@ from apprise import NotifyFormat
 import apprise
 from loguru import logger
 
+from .apprise_plugin.assets import APPRISE_AVATAR_URL
+from .apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
+from .safe_jinja import render as jinja_render
 
 valid_tokens = {
     'base_url': '',
@@ -39,10 +42,6 @@ valid_notification_formats = {
 
 
 def process_notification(n_object, datastore):
-    # so that the custom endpoints are registered
-    from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
-
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
@@ -66,12 +65,12 @@ def process_notification(n_object, datastore):
     # raise it as an exception
 
     sent_objs = []
-    from .apprise_asset import asset
+    from .apprise_plugin.assets import apprise_asset
 
     if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
+        apprise_asset.async_mode = n_object.get('as_async')
 
-    apobj = apprise.Apprise(debug=True, asset=asset)
+    apobj = apprise.Apprise(debug=True, asset=apprise_asset)
 
     if not n_object.get('notification_urls'):
         return None
@@ -112,7 +111,7 @@ def process_notification(n_object, datastore):
                     and not url.startswith('get') \
                     and not url.startswith('delete') \
                     and not url.startswith('put'):
-                url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
             if url.startswith('tgram://'):
                 # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.

changedetectionio/processors/text_json_diff/processor.py

@@ -334,12 +334,14 @@ class perform_site_check(difference_detection_processor):
 
         # And check if 'conditions' will let this pass through
         if watch.get('conditions') and watch.get('conditions_match_logic'):
-            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
-                                                application_datastruct=self.datastore.data,
-                                                ephemeral_data={
-                                                    'text': stripped_text_from_html
-                                                }
-                                                ):
+            conditions_result = execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
+                                                                    application_datastruct=self.datastore.data,
+                                                                    ephemeral_data={
+                                                                        'text': stripped_text_from_html
+                                                                    }
+                                                                    )
+
+            if not conditions_result.get('result'):
                 # Conditions say "Condition not met" so we block it.
                 blocked = True
 

changedetectionio/static/js/conditions.js

@@ -52,7 +52,7 @@ $(document).ready(function () {
 
             
             // Create a rule object
-            const rule = {
+            let rule = {
                 field: field,
                 operator: operator,
                 value: value
@@ -96,6 +96,10 @@ $(document).ready(function () {
                 contentType: false, // Let the browser set the correct content type
                 success: function (response) {
                     if (response.status === "success") {
+                        if(rule['field'] !== "page_filtered_text") {
+                            // A little debug helper for the user
+                            $('#verify-state-text').text(`${rule['field']} was value "${response.data[rule['field']]}"`)
+                        }
                         if (response.result) {
                             alert("✅ Condition PASSES verification against current snapshot!");
                         } else {

changedetectionio/static/js/watch-overview.js

@@ -48,6 +48,8 @@ $(function () {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
 
+    const time_check_step_size_seconds=1;
+
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {
@@ -57,5 +59,30 @@ $(function () {
         }
     });
 
+    setInterval(function () {
+        // Background ETA completion for 'checking now'
+        $(".watch-table .checking-now .last-checked").each(function () {
+            const eta_complete = parseFloat($(this).data('eta_complete'));
+            const fetch_duration = parseInt($(this).data('fetchduration'));
+
+            if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {
+                const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;
+
+                let r = (1.0 - (remaining_seconds / fetch_duration)) * 100;
+                if (r < 10) {
+                    r = 10;
+                }
+                if (r >= 90) {
+                    r = 100;
+                }
+                $(this).css('background-size', `${r}% 100%`);
+                //$(this).text(`${r}% remain ${remaining_seconds}`);
+            } else {
+                $(this).css('background-size', `100% 100%`);
+            }
+        });
+
+        nowtimeserver = nowtimeserver + time_check_step_size_seconds;
+    }, time_check_step_size_seconds * 1000);
 });
 

changedetectionio/store.py

@@ -251,8 +251,14 @@ class ChangeDetectionStore:
     # Clone a watch by UUID
     def clone(self, uuid):
         url = self.data['watching'][uuid].get('url')
-        extras = self.data['watching'][uuid]
+        extras = deepcopy(self.data['watching'][uuid])
         new_uuid = self.add_watch(url=url, extras=extras)
+        watch = self.data['watching'][new_uuid]
+
+        if self.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+            # Because it will be recalculated on the next fetch
+            self.data['watching'][new_uuid]['title'] = None
+
         return new_uuid
 
     def url_exists(self, url):
@@ -363,7 +369,6 @@ class ChangeDetectionStore:
         new_watch.ensure_data_dir_exists()
         self.__data['watching'][new_uuid] = new_watch
 
-
         if write_to_disk_now:
             self.sync_to_json()
 
@@ -571,16 +576,16 @@ class ChangeDetectionStore:
 
         return ret
 
-    def add_tag(self, name):
+    def add_tag(self, title):
         # If name exists, return that
-        n = name.strip().lower()
+        n = title.strip().lower()
         logger.debug(f">>> Adding new tag - '{n}'")
         if not n:
             return False
 
         for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
             if n == tag.get('title', '').lower().strip():
-                logger.warning(f"Tag '{name}' already exists, skipping creation.")
+                logger.warning(f"Tag '{title}' already exists, skipping creation.")
                 return uuid
 
         # Eventually almost everything todo with a watch will apply as a Tag
@@ -588,7 +593,7 @@ class ChangeDetectionStore:
         with self.lock:
             from .model import Tag
             new_tag = Tag.model(datastore_path=self.datastore_path, default={
-                'title': name.strip(),
+                'title': title.strip(),
                 'date_created': int(time.time())
             })
 
@@ -631,6 +636,41 @@ class ChangeDetectionStore:
             if watch.get('processor') == processor_name:
                 return True
         return False
+        
+    def search_watches_for_url(self, query, tag_limit=None, partial=False):
+        """Search watches by URL, title, or error messages
+        
+        Args:
+            query (str): Search term to match against watch URLs, titles, and error messages
+            tag_limit (str, optional): Optional tag name to limit search results
+            partial: (bool, optional): sub-string matching
+
+        Returns:
+            list: List of UUIDs of watches that match the search criteria
+        """
+        matching_uuids = []
+        query = query.lower().strip()
+        tag = self.tag_exists_by_name(tag_limit) if tag_limit else False
+
+        for uuid, watch in self.data['watching'].items():
+            # Filter by tag if requested
+            if tag_limit:
+                if not tag.get('uuid') in watch.get('tags', []):
+                    continue
+
+            # Search in URL, title, or error messages
+            if partial:
+                if ((watch.get('title') and query in watch.get('title').lower()) or
+                    query in watch.get('url', '').lower() or
+                    (watch.get('last_error') and query in watch.get('last_error').lower())):
+                    matching_uuids.append(uuid)
+            else:
+                if ((watch.get('title') and query == watch.get('title').lower()) or
+                        query == watch.get('url', '').lower() or
+                        (watch.get('last_error') and query == watch.get('last_error').lower())):
+                    matching_uuids.append(uuid)
+
+        return matching_uuids
 
     def get_unique_notification_tokens_available(self):
         # Ask each type of watch if they have any extra notification token to add to the validation
@@ -847,7 +887,7 @@ class ChangeDetectionStore:
             if tag:
                 tag_uuids = []
                 for t in tag.split(','):
-                    tag_uuids.append(self.add_tag(name=t))
+                    tag_uuids.append(self.add_tag(title=t))
 
                 self.data['watching'][uuid]['tags'] = tag_uuids
 

changedetectionio/templates/_common_fields.html

@@ -28,7 +28,7 @@
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
-                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
                               <br>
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>

changedetectionio/templates/_helpers.html

@@ -199,7 +199,7 @@
     </div>
     {% else %}
         <span class="pure-form-message-inline">
-            Want to use a time schedule? <a href="{{url_for('settings_page')}}#timedate">First confirm/save your Time Zone Settings</a>
+            Want to use a time schedule? <a href="{{url_for('settings.settings_page')}}#timedate">First confirm/save your Time Zone Settings</a>
         </span>
         <br>
     {% endif %}

changedetectionio/templates/base.html

@@ -7,7 +7,7 @@
     <meta name="description" content="Self hosted website change detection." >
     <title>Change Detection{{extra_title}}</title>
     {% if app_rss_token %}
-      <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss', tag=active_tag_uuid , token=app_rss_token)}}" >
+      <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss.feed', tag=active_tag_uuid , token=app_rss_token)}}" >
     {% endif %}
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}?v={{ get_css_version() }}" >
@@ -42,7 +42,7 @@
           <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
             <strong>Change</strong>Detection.io</a>
         {% else %}
-          <a class="pure-menu-heading" href="{{url_for('index')}}">
+          <a class="pure-menu-heading" href="{{url_for('watchlist.index')}}">
             <strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
@@ -64,17 +64,17 @@
                 <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
               </li>
               <li class="pure-menu-item">
-                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('settings.settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
               <li class="pure-menu-item">
-                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
+                <a href="{{ url_for('imports.import_page')}}" class="pure-menu-link">IMPORT</a>
               </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('backups.index')}}" class="pure-menu-link">BACKUPS</a>
               </li>
             {% else %}
               <li class="pure-menu-item">
-                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
+                <a href="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
               </li>
             {% endif %}
           {% else %}
@@ -144,7 +144,7 @@
     {% endif %}
     {% if left_sticky %}
       <div class="sticky-tab" id="left-sticky">
-        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a><br>
+        <a href="{{url_for('ui.ui_views.preview_page', uuid=uuid)}}">Show current snapshot</a><br>
           Visualise <strong>triggers</strong> and <strong>ignored text</strong>
       </div>
     {% endif %}
@@ -157,15 +157,13 @@
                 <h4>Try our Chrome extension</h4>
                 <p>
                     <a id="chrome-extension-link"
-                       title="Try our new Chrome Extension!"
+                       title="Chrome Extension - Web Page Change Detection with changedetection.io!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>
-
                 Easily add the current web-page from your browser directly into your changedetection.io tool, more great features coming soon!
-
                 <h4>Changedetection.io needs your support!</h4>
                 <p>
                     You can help us by supporting changedetection.io on these platforms;
@@ -173,17 +171,20 @@
                 <p>
                 <ul>
                     <li>
-                        <a href="https://alternativeto.net/software/changedetection-io/about/">Rate us at
+                        <a href="https://alternativeto.net/software/changedetection-io/about/" title="Web page change detection at alternativeto.net">Rate us at
                         AlternativeTo.net</a>
                     </li>
                 <li>
-                    <a href="https://github.com/dgtlmoon/changedetection.io">Star us on GitHub</a>
+                    <a href="https://github.com/dgtlmoon/changedetection.io" title="Web page change detection on GitHub">Star us on GitHub</a>
                 </li>
                 <li>
-                    <a href="https://twitter.com/change_det_io">Follow us at Twitter/X</a>
+                    <a rel="nofollow" href="https://twitter.com/change_det_io" title="Web page change detection on Twitter">Follow us at Twitter/X</a>
                 </li>
                 <li>
-                    <a href="https://www.linkedin.com/company/changedetection-io">Check us out on LinkedIn</a>
+                    <a rel="nofollow" href="https://www.g2.com/products/changedetection-io/reviews" title="Web page change detection reviews at G2">G2 Software reviews</a>
+                </li>
+                <li>
+                    <a rel="nofollow" href="https://www.linkedin.com/company/changedetection-io" title="Visit web page change detection at LinkedIn">Check us out on LinkedIn</a>
                 </li>
                 <li>
                     And tell your friends and colleagues :)

changedetectionio/templates/diff.html

@@ -7,7 +7,7 @@
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
 
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+    const highlight_submit_ignore_url="{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}";
 
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
@@ -125,7 +125,7 @@
      </div>
     <div class="tab-pane-inner" id="extract">
         <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
-              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
+              action="{{ url_for('ui.ui_views.diff_history_page', uuid=uuid) }}#extract"
               method="POST">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 

changedetectionio/templates/edit.html

@@ -20,7 +20,7 @@
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}";
     const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
@@ -62,7 +62,7 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
-              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
+              action="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
             <div class="tab-pane-inner" id="general">
@@ -274,7 +274,7 @@ Math: {{ 1 + 1 }}") }}
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
                             <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
@@ -305,9 +305,9 @@ Math: {{ 1 + 1 }}") }}
                     {{ render_field(form.conditions_match_logic) }}
                     {{ render_fieldlist_of_formfields_as_table(form.conditions) }}
                     <div class="pure-form-message-inline">
-                        <br>
-                        Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br>
-                        Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br>
+
+                        <p id="verify-state-text">Use the verify (✓) button to test if a condition passes against the current snapshot.</p>
+                       Read a quick tutorial about <a href="https://changedetection.io/tutorial/conditional-actions-web-page-changes">using conditional web page changes here</a>.<br>
                     </div>
                 </div>
             </div>
@@ -337,7 +337,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                         {% if '/text()' in  field %}
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
-                        <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
+                        <span class="pure-form-message-inline">One CSS, xPath 1 &amp; 2, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
                         <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
@@ -480,7 +480,7 @@ keyword") }}
             </div>
               <div id="text-preview" style="display: none;" >
                     <script>
-                        const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}";
+                        const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
                     </script>
                     <br>
                     {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
@@ -577,7 +577,7 @@ keyword") }}
                     </table>
                     {% if watch.history_n %}
                         <p>
-                             <a href="{{url_for('watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>
+                             <a href="{{url_for('ui.ui_edit.watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>
                         </p>
                     {% endif %}
 
@@ -586,12 +586,12 @@ keyword") }}
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('form_delete', uuid=uuid)}}"
+                    <a href="{{url_for('ui.form_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('clear_watch_history', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Clear History</a>
-                    <a href="{{url_for('form_clone', uuid=uuid)}}"
-                       class="pure-button button-small ">Create Copy</a>
+                    {% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Clear History</a>{% endif %}
+                    <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
+                       class="pure-button button-small ">Clone &amp; Edit</a>
                 </div>
             </div>
         </form>

changedetectionio/templates/preview.html

@@ -7,7 +7,7 @@
         {% if last_error_screenshot %}
             const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
         {% endif %}
-        const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+        const highlight_submit_ignore_url = "{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}";
     </script>
     <script src="{{url_for('static_content', group='js', filename='plugins.js')}}"></script>
     <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script>

changedetectionio/tests/apprise/test_apprise_asset.py

@@ -0,0 +1,24 @@
+import pytest
+from apprise import AppriseAsset
+
+from changedetectionio.apprise_asset import (
+    APPRISE_APP_DESC,
+    APPRISE_APP_ID,
+    APPRISE_APP_URL,
+    APPRISE_AVATAR_URL,
+)
+
+
+@pytest.fixture(scope="function")
+def apprise_asset() -> AppriseAsset:
+    from changedetectionio.apprise_asset import apprise_asset
+
+    return apprise_asset
+
+
+def test_apprise_asset_init(apprise_asset: AppriseAsset):
+    assert isinstance(apprise_asset, AppriseAsset)
+    assert apprise_asset.app_id == APPRISE_APP_ID
+    assert apprise_asset.app_desc == APPRISE_APP_DESC
+    assert apprise_asset.app_url == APPRISE_APP_URL
+    assert apprise_asset.image_url_logo == APPRISE_AVATAR_URL

changedetectionio/tests/apprise/test_apprise_custom_api_call.py

@@ -0,0 +1,211 @@
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+from apprise.utils.parse import parse_url as apprise_parse_url
+
+from ...apprise_plugin.custom_handlers import (
+    _get_auth,
+    _get_headers,
+    _get_params,
+    apprise_http_custom_handler,
+    SUPPORTED_HTTP_METHODS,
+)
+
+
+@pytest.mark.parametrize(
+    "url,expected_auth",
+    [
+        ("get://user:pass@localhost:9999", ("user", "pass")),
+        ("get://user@localhost:9999", "user"),
+        ("get://localhost:9999", ""),
+        ("get://user%20name:pass%20word@localhost:9999", ("user name", "pass word")),
+    ],
+)
+def test_get_auth(url, expected_auth):
+    """Test authentication extraction with various URL formats."""
+    parsed_url = apprise_parse_url(url)
+    assert _get_auth(parsed_url) == expected_auth
+
+
+@pytest.mark.parametrize(
+    "url,body,expected_content_type",
+    [
+        (
+            "get://localhost:9999?+content-type=application/xml",
+            "test",
+            "application/xml",
+        ),
+        ("get://localhost:9999", '{"key": "value"}', "application/json; charset=utf-8"),
+        ("get://localhost:9999", "plain text", None),
+        ("get://localhost:9999?+content-type=text/plain", "test", "text/plain"),
+    ],
+)
+def test_get_headers(url, body, expected_content_type):
+    """Test header extraction and content type detection."""
+    parsed_url = apprise_parse_url(url)
+    headers = _get_headers(parsed_url, body)
+
+    if expected_content_type:
+        assert headers.get("Content-Type") == expected_content_type
+
+
+@pytest.mark.parametrize(
+    "url,expected_params",
+    [
+        ("get://localhost:9999?param1=value1", {"param1": "value1"}),
+        ("get://localhost:9999?param1=value1&-param2=ignored", {"param1": "value1"}),
+        ("get://localhost:9999?param1=value1&+header=test", {"param1": "value1"}),
+        (
+            "get://localhost:9999?encoded%20param=encoded%20value",
+            {"encoded param": "encoded value"},
+        ),
+    ],
+)
+def test_get_params(url, expected_params):
+    """Test parameter extraction with URL encoding and exclusion logic."""
+    parsed_url = apprise_parse_url(url)
+    params = _get_params(parsed_url)
+    assert dict(params) == expected_params
+
+
+@pytest.mark.parametrize(
+    "url,schema,method",
+    [
+        ("get://localhost:9999", "get", "GET"),
+        ("post://localhost:9999", "post", "POST"),
+        ("delete://localhost:9999", "delete", "DELETE"),
+    ],
+)
+@patch("requests.request")
+def test_apprise_custom_api_call_success(mock_request, url, schema, method):
+    """Test successful API calls with different HTTP methods and schemas."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    meta = {"url": url, "schema": schema}
+    result = apprise_http_custom_handler(
+        body="test body", title="Test Title", notify_type="info", meta=meta
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+    assert call_args[1]["method"] == method.upper()
+    assert call_args[1]["url"].startswith("http")
+
+
+@patch("requests.request")
+def test_apprise_custom_api_call_with_auth(mock_request):
+    """Test API call with authentication."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = "get://user:pass@localhost:9999/secure"
+    meta = {"url": url, "schema": "get"}
+
+    result = apprise_http_custom_handler(
+        body=json.dumps({"key": "value"}),
+        title="Secure Test",
+        notify_type="info",
+        meta=meta,
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+    call_args = mock_request.call_args
+    assert call_args[1]["auth"] == ("user", "pass")
+
+
+@pytest.mark.parametrize(
+    "exception_type,expected_result",
+    [
+        (requests.RequestException, False),
+        (requests.HTTPError, False),
+        (Exception, False),
+    ],
+)
+@patch("requests.request")
+def test_apprise_custom_api_call_failure(mock_request, exception_type, expected_result):
+    """Test various failure scenarios."""
+    url = "get://localhost:9999/error"
+    meta = {"url": url, "schema": "get"}
+
+    # Simulate different types of exceptions
+    mock_request.side_effect = exception_type("Error occurred")
+
+    result = apprise_http_custom_handler(
+        body="error body", title="Error Test", notify_type="error", meta=meta
+    )
+
+    assert result == expected_result
+
+
+def test_invalid_url_parsing():
+    """Test handling of invalid URL parsing."""
+    meta = {"url": "invalid://url", "schema": "invalid"}
+    result = apprise_http_custom_handler(
+        body="test", title="Invalid URL", notify_type="info", meta=meta
+    )
+
+    assert result is False
+
+
+@pytest.mark.parametrize(
+    "schema,expected_method",
+    [
+        (http_method, http_method.upper())
+        for http_method in SUPPORTED_HTTP_METHODS
+    ],
+)
+@patch("requests.request")
+def test_http_methods(mock_request, schema, expected_method):
+    """Test all supported HTTP methods."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = f"{schema}://localhost:9999"
+
+    result = apprise_http_custom_handler(
+        body="test body",
+        title="Test Title",
+        notify_type="info",
+        meta={"url": url, "schema": schema},
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+    assert call_args[1]["method"] == expected_method
+
+
+@pytest.mark.parametrize(
+    "input_schema,expected_method",
+    [
+        (f"{http_method}s", http_method.upper())
+        for http_method in SUPPORTED_HTTP_METHODS
+    ],
+)
+@patch("requests.request")
+def test_https_method_conversion(
+    mock_request, input_schema, expected_method
+):
+    """Validate that methods ending with 's' use HTTPS and correct HTTP method."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = f"{input_schema}://localhost:9999"
+
+    result = apprise_http_custom_handler(
+        body="test body",
+        title="Test Title",
+        notify_type="info",
+        meta={"url": url, "schema": input_schema},
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+
+    assert call_args[1]["method"] == expected_method
+    assert call_args[1]["url"].startswith("https")

changedetectionio/tests/conftest.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import resource
+import psutil
 import time
 from threading import Thread
 
@@ -28,9 +28,10 @@ def reportlog(pytestconfig):
 
 
 def track_memory(memory_usage, ):
+    process = psutil.Process(os.getpid())
     while not memory_usage["stop"]:
-        max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-        memory_usage["peak"] = max(memory_usage["peak"], max_rss)
+        current_rss = process.memory_info().rss
+        memory_usage["peak"] = max(memory_usage["peak"], current_rss)
         time.sleep(0.01)  # Adjust the sleep time as needed
 
 @pytest.fixture(scope='function')

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -16,7 +16,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
     #####################
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_webdriver",
@@ -30,7 +30,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -42,13 +42,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
         # So the name should appear in the edit page under "Request" > "Fetch Method"
         res = client.get(
-            url_for("edit_page", uuid="first"),
+            url_for("ui.ui_edit.edit_page", uuid="first"),
             follow_redirects=True
         )
         assert b'custom browser URL' in res.data
 
         res = client.post(
-            url_for("edit_page", uuid="first"),
+            url_for("ui.ui_edit.edit_page", uuid="first"),
             data={
                 # 'run_customer_browser_url_tests.sh' will search for this string to know if we hit the right browser container or not
                   "url": f"https://changedetection.io/ci-test.html?custom-browser-search-string=1",
@@ -64,13 +64,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
         wait_for_all_checks(client)
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'cool it works' in res.data

changedetectionio/tests/fetchers/test_content.py

@@ -11,7 +11,7 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
 
     #####################
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_webdriver"},
@@ -22,7 +22,7 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": "https://changedetection.io/ci-test.html"},
         follow_redirects=True
     )
@@ -32,7 +32,7 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
 
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     logging.getLogger().info("Looking for correct fetched HTML (text) from server")

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -13,7 +13,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
@@ -21,7 +21,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
     res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
             "url": test_url,
             "tags": "",
@@ -41,7 +41,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
 
     # Check HTML conversion detected and workd
     res = client.get(
-        url_for("preview_page", uuid=uuid),
+        url_for("ui.ui_views.preview_page", uuid=uuid),
         follow_redirects=True
     )
     assert b"This text should be removed" not in res.data
@@ -51,6 +51,6 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"user-agent: mycustomagent" in res.data
 
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -11,7 +11,7 @@ def test_preferred_proxy(client, live_server, measure_memory_usage):
 
 
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
@@ -19,7 +19,7 @@ def test_preferred_proxy(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
     res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
                 "include_filters": "",
                 "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -13,12 +13,12 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
 
     # Should only be available when a proxy is setup
     res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1))
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1))
     assert b'No proxy' not in res.data
 
     # Setup a proxy
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
@@ -37,24 +37,24 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
 
     # Should be available as an option
     res = client.get(
-        url_for("settings_page", unpause_on_save=1))
+        url_for("settings.settings_page", unpause_on_save=1))
     assert b'No proxy' in res.data
 
 
     # This will add it paused
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1))
+        url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data
 
     res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
         data={
                 "include_filters": "",
                 "fetch_backend": "html_requests",
@@ -67,7 +67,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
     )
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)
 

changedetectionio/tests/proxy_list/test_proxy.py

@@ -8,7 +8,7 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         # Because a URL wont show in squid/proxy logs due it being SSLed
         # Use plain HTTP or a specific domain-name here
         data={"urls": "http://one.changedetection.io"},

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -11,7 +11,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
 
     # Goto settings, add our custom one
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
@@ -26,7 +26,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
     assert b"Settings updated." in res.data
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         # Because a URL wont show in squid/proxy logs due it being SSLed
         # Use plain HTTP or a specific domain-name here
         data={"urls": "https://changedetection.io/CHANGELOG.txt"},
@@ -36,11 +36,11 @@ def test_select_custom(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Proxy Authentication Required' not in res.data
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     # We should see something via proxy

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -25,7 +25,7 @@ def test_socks5(client, live_server, measure_memory_usage):
 
     # Setup a proxy
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
@@ -45,20 +45,20 @@ def test_socks5(client, live_server, measure_memory_usage):
     test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
     res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
     )
     # check the proxy is offered as expected
     assert b'ui-0socks5proxy' in res.data
 
     res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
             "include_filters": "",
             "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
@@ -73,7 +73,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -97,6 +97,6 @@ def test_socks5(client, live_server, measure_memory_usage):
     )
     assert b"OK" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -28,24 +28,24 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage)
     test_url = test_url.replace('localhost.localdomain', 'cdio')
     test_url = test_url.replace('localhost', 'cdio')
 
-    res = client.get(url_for("settings_page"))
+    res = client.get(url_for("settings.settings_page"))
     assert b'name="requests-proxy" type="radio" value="socks5proxy"' in res.data
 
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
     res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
     )
     # check the proxy is offered as expected
     assert b'name="proxy" type="radio" value="socks5proxy"' in res.data
 
     res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
             "include_filters": "",
             "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
@@ -60,7 +60,7 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage)
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/restock/test_restock.py

@@ -62,7 +62,7 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title "+default_notification_title,
               "application-notification_body": "fallback-body "+default_notification_body,
@@ -76,21 +76,21 @@ def test_restock_detection(client, live_server, measure_memory_usage):
 
 
     client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
         follow_redirects=True
     )
 
     # Is it correctly show as NOT in stock?
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' in res.data
 
     # Is it correctly shown as in stock
     set_back_in_stock_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
@@ -101,12 +101,12 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
     # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
     set_original_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(5)
     assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' in res.data, "Correctly showing NOT IN STOCK in the list after it changed from IN STOCK"
 

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -25,7 +25,6 @@ def test_setup(live_server):
 
 def get_last_message_from_smtp_server():
     import socket
-    global smtp_test_server
     port = 11080  # socket server port number
 
     client_socket = socket.socket()  # instantiate
@@ -44,13 +43,12 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     # live_server_setup(live_server)
     set_original_response()
 
-    global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": "fallback-body<br> " + default_notification_body,
@@ -64,7 +62,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
@@ -75,7 +73,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     set_longer_modified_response()
     time.sleep(2)
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
     time.sleep(3)
@@ -88,7 +86,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
     assert 'Content-Type: text/html' in msg
     assert '(added) So let\'s see what happens.<br>' in msg  # the html part
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
@@ -99,7 +97,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # https://github.com/caronc/apprise/issues/633
 
     set_original_response()
-    global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">
@@ -116,7 +113,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
@@ -130,7 +127,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
@@ -140,7 +137,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
     set_longer_modified_response()
     time.sleep(2)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
     time.sleep(3)
@@ -157,7 +154,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     set_original_response()
     # Now override as HTML format
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
             "notification_format": 'HTML',
@@ -182,5 +179,5 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     assert '&lt;' not in msg
     assert 'Content-Type: text/html' in msg
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_access_control.py

@@ -1,4 +1,4 @@
-from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
+from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
@@ -8,12 +8,12 @@ def test_check_access_control(app, client, live_server):
 
     with app.test_client(use_cookies=True) as c:
         # Check we don't have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
+        res = c.get(url_for("settings.settings_page"))
         assert b"Remove password" not in res.data
 
         # add something that we can hit via diff page later
         res = c.post(
-            url_for("import_page"),
+            url_for("imports.import_page"),
             data={"urls": url_for('test_random_content_endpoint', _external=True)},
             follow_redirects=True
         )
@@ -23,8 +23,9 @@ def test_check_access_control(app, client, live_server):
         # causes a 'Popped wrong request context.' error when client. is accessed?
         #wait_for_all_checks(client)
 
-        res = c.get(url_for("form_watch_checknow"), follow_redirects=True)
-        assert b'1 watches queued for rechecking.' in res.data
+        res = c.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+        assert b'Queued 1 watch for rechecking.' in res.data
+
         time.sleep(3)
         # causes a 'Popped wrong request context.' error when client. is accessed?
         #wait_for_all_checks(client)
@@ -32,7 +33,7 @@ def test_check_access_control(app, client, live_server):
 
         # Enable password check and diff page access bypass
         res = c.post(
-            url_for("settings_page"),
+            url_for("settings.settings_page"),
             data={"application-password": "foobar",
                   "application-shared_diff_access": "True",
                   "requests-time_between_check-minutes": 180,
@@ -43,14 +44,27 @@ def test_check_access_control(app, client, live_server):
         assert b"Password protection enabled." in res.data
 
         # Check we hit the login
-        res = c.get(url_for("index"), follow_redirects=True)
+        res = c.get(url_for("watchlist.index"), follow_redirects=True)
         # Should be logged out
         assert b"Login" in res.data
 
         # The diff page should return something valid when logged out
-        res = c.get(url_for("diff_history_page", uuid="first"))
+        res = c.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
+        # access to assets should work (check_authentication)
+        res = c.get(url_for('static_content', group='js', filename='jquery-3.6.0.min.js'))
+        assert res.status_code == 200
+        res = c.get(url_for('static_content', group='styles', filename='styles.css'))
+        assert res.status_code == 200
+        res = c.get(url_for('static_content', group='styles', filename='404-testetest.css'))
+        assert res.status_code == 404
+
+        # Access to screenshots should be limited by 'shared_diff_access'
+        path = url_for('static_content', group='screenshot', filename='random-uuid-that-will-404.png', _external=True)
+        res = c.get(path)
+        assert res.status_code == 404
+
         # Check wrong password does not let us in
         res = c.post(
             url_for("login"),
@@ -79,7 +93,7 @@ def test_check_access_control(app, client, live_server):
 
         # 598 - Password should be set and not accidently removed
         res = c.post(
-            url_for("settings_page"),
+            url_for("settings.settings_page"),
             data={
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
@@ -91,7 +105,7 @@ def test_check_access_control(app, client, live_server):
 
         assert b"Login" in res.data
 
-        res = c.get(url_for("settings_page"),
+        res = c.get(url_for("settings.settings_page"),
             follow_redirects=True)
 
 
@@ -110,7 +124,7 @@ def test_check_access_control(app, client, live_server):
         # Yes we are correctly logged in
         assert b"LOG OUT" in res.data
 
-        res = c.get(url_for("settings_page"))
+        res = c.get(url_for("settings.settings_page"))
 
         # Menu should be available now
         assert b"SETTINGS" in res.data
@@ -124,7 +138,7 @@ def test_check_access_control(app, client, live_server):
         # Remove password button, and check that it worked
         ##################################################
         res = c.post(
-            url_for("settings_page"),
+            url_for("settings.settings_page"),
             data={
                 "requests-time_between_check-minutes": 180,
                 "application-fetch_backend": "html_webdriver",
@@ -139,7 +153,7 @@ def test_check_access_control(app, client, live_server):
         # Be sure a blank password doesnt setup password protection
         ############################################################
         res = c.post(
-            url_for("settings_page"),
+            url_for("settings.settings_page"),
             data={"application-password": "",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
@@ -151,10 +165,10 @@ def test_check_access_control(app, client, live_server):
         # Now checking the diff access
         # Enable password check and diff page access bypass
         res = c.post(
-            url_for("settings_page"),
+            url_for("settings.settings_page"),
             data={"application-password": "foobar",
                   # Should be disabled
-#                  "application-shared_diff_access": "True",
+                  "application-shared_diff_access": "",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
@@ -163,10 +177,14 @@ def test_check_access_control(app, client, live_server):
         assert b"Password protection enabled." in res.data
 
         # Check we hit the login
-        res = c.get(url_for("index"), follow_redirects=True)
+        res = c.get(url_for("watchlist.index"), follow_redirects=True)
         # Should be logged out
         assert b"Login" in res.data
 
+        # Access to screenshots should be limited by 'shared_diff_access'
+        res = c.get(url_for('static_content', group='screenshot', filename='random-uuid-that-will-403.png'))
+        assert res.status_code == 403
+
         # The diff page should return something valid when logged out
-        res = c.get(url_for("diff_history_page", uuid="first"))
+        res = c.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
         assert b'Random content' not in res.data

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -45,7 +45,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -57,7 +57,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"trigger_text": 'The golden line',
               "url": test_url,
               'fetch_backend': "html_requests",
@@ -69,38 +69,38 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
 
     # Check in the processor here what's going on, its triggering empty-reply and no change.
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
     # Now add it back, and we should not get a trigger
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     set_original(excluding=None)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
@@ -111,7 +111,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
 
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               # triggered_text will contain multiple lines
               "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
@@ -128,7 +128,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -139,7 +139,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"trigger_text": 'Oh yes please',
               "url": test_url,
               'processor': 'text_json_diff',
@@ -153,18 +153,18 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     assert b'unviewed' in res.data
 
@@ -176,5 +176,5 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
         assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
+from .util import live_server_setup, wait_for_all_checks
 
 import json
 import uuid
@@ -57,16 +57,15 @@ def test_setup(client, live_server, measure_memory_usage):
 
 
 def test_api_simple(client, live_server, measure_memory_usage):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
-    api_key = extract_api_key_from_UI(client)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
     set_original_response()
 
     # Validate bad URL
-    test_url = url_for('test_endpoint', _external=True,
-                       headers={'x-api-key': api_key}, )
+    test_url = url_for('test_endpoint', _external=True )
     res = client.post(
         url_for("createwatch"),
         data=json.dumps({"url": "h://xxxxxxxxxom"}),
@@ -173,7 +172,7 @@ def test_api_simple(client, live_server, measure_memory_usage):
 
     assert watch.get('viewed') == False
     # Loading the most recent snapshot should force viewed to become true
-    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
+    client.get(url_for("ui.ui_views.diff_history_page", uuid="first"), follow_redirects=True)
 
     time.sleep(3)
     # Fetch the whole watch again, viewed should be true
@@ -259,7 +258,7 @@ def test_access_denied(client, live_server, measure_memory_usage):
 
     # Disable config_api_token_enabled and it should work
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-fetch_backend": "html_requests",
@@ -276,11 +275,11 @@ def test_access_denied(client, live_server, measure_memory_usage):
     assert res.status_code == 200
 
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-fetch_backend": "html_requests",
@@ -293,12 +292,11 @@ def test_access_denied(client, live_server, measure_memory_usage):
 def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
-    api_key = extract_api_key_from_UI(client)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
     set_original_response()
-    test_url = url_for('test_endpoint', _external=True,
-                       headers={'x-api-key': api_key}, )
+    test_url = url_for('test_endpoint', _external=True)
 
     # Create new
     res = client.post(
@@ -321,7 +319,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     # Check in the edit page just to be sure
     res = client.get(
-        url_for("edit_page", uuid=watch_uuid),
+        url_for("ui.ui_edit.edit_page", uuid=watch_uuid),
     )
     assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
     assert b"One" in res.data, "Tag 'One' was found"
@@ -344,7 +342,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     # Check in the edit page just to be sure
     res = client.get(
-        url_for("edit_page", uuid=watch_uuid),
+        url_for("ui.ui_edit.edit_page", uuid=watch_uuid),
     )
     assert b"new title" in res.data, "new title found in edit page"
     assert b"552" in res.data, "552 minutes found in edit page"
@@ -368,12 +366,13 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     assert b'Additional properties are not allowed' in res.data
 
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
 def test_api_import(client, live_server, measure_memory_usage):
-    api_key = extract_api_key_from_UI(client)
+    #live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     res = client.post(
         url_for("import") + "?tag=import-test",
@@ -384,11 +383,55 @@ def test_api_import(client, live_server, measure_memory_usage):
 
     assert res.status_code == 200
     assert len(res.json) == 2
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"https://website1.com" in res.data
     assert b"https://website2.com" in res.data
 
     # Should see the new tag in the tag/groups list
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
-    
+
+def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
+
+    #live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Enable password check and diff page access bypass
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-password": "foobar", # password is now set! API should still work!
+              "application-api_access_token_enabled": "y",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Password protection enabled." in res.data
+
+    # Create a watch
+    set_original_response()
+    test_url = url_for('test_endpoint', _external=True)
+
+    # Create new
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, "title": "My test URL" }),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+
+    assert res.status_code == 201
+
+
+    wait_for_all_checks(client)
+    url = url_for("createwatch")
+    # Get a listing, it will be the first one
+    res = client.get(
+        url,
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+
+    assert len(res.json)
+
+

changedetectionio/tests/test_api_search.py

@@ -0,0 +1,101 @@
+from copy import copy
+
+from flask import url_for
+import json
+import time
+from .util import live_server_setup, wait_for_all_checks
+
+
+def test_api_search(client, live_server):
+    live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    watch_data = {}
+    # Add some test watches
+    urls = [
+        'https://example.com/page1',
+        'https://example.org/testing',
+        'https://test-site.com/example'
+    ]
+
+    # Import the test URLs
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+    assert b"3 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Get a listing, it will be the first one
+    watches_response = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+
+
+    # Add a title to one watch for title search testing
+    for uuid, watch in watches_response.json.items():
+
+        watch_data = client.get(url_for("watch", uuid=uuid),
+                                follow_redirects=True,
+                                headers={'x-api-key': api_key}
+                                )
+
+        if urls[0] == watch_data.json['url']:
+            # HTTP PUT ( UPDATE an existing watch )
+            client.put(
+                url_for("watch", uuid=uuid),
+                headers={'x-api-key': api_key, 'content-type': 'application/json'},
+                data=json.dumps({'title': 'Example Title Test'}),
+            )
+
+    # Test search by URL
+    res = client.get(url_for("search")+"?q=https://example.com/page1", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[0]
+
+    # Test search by URL - partial should NOT match without ?partial=true flag
+    res = client.get(url_for("search")+"?q=https://example", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 0
+
+
+    # Test search by title
+    res = client.get(url_for("search")+"?q=Example Title Test", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[0]
+    assert list(res.json.values())[0]['title'] == 'Example Title Test'
+
+    # Test search that should return multiple results (partial = true)
+    res = client.get(url_for("search")+"?q=https://example&partial=true", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 2
+
+    # Test empty search
+    res = client.get(url_for("search")+"?q=", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert res.status_code == 400
+
+    # Add a tag to test search with tag filter
+    tag_name = 'test-tag'
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": tag_name}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 201
+    tag_uuid = res.json['uuid']
+
+    # Add the tag to one watch
+    for uuid, watch in watches_response.json.items():
+        if urls[2] == watch['url']:
+            client.put(
+                url_for("watch", uuid=uuid),
+                headers={'x-api-key': api_key, 'content-type': 'application/json'},
+                data=json.dumps({'tags': [tag_uuid]}),
+            )
+
+
+    # Test search with tag filter and q
+    res = client.get(url_for("search") + f"?q={urls[2]}&tag={tag_name}", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[2]
+

changedetectionio/tests/test_api_tags.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+import json
+
+def test_api_tags_listing(client, live_server, measure_memory_usage):
+    live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+    tag_title = 'Test Tag'
+
+    # Get a listing
+    res = client.get(
+        url_for("tags"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.text.strip() == "{}", "Should be empty list"
+    assert res.status_code == 200
+
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": tag_title}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 201
+
+    new_tag_uuid = res.json.get('uuid')
+
+    # List tags - should include our new tag
+    res = client.get(
+        url_for("tags"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert new_tag_uuid in res.text
+    assert res.json[new_tag_uuid]['title'] == tag_title
+    assert res.json[new_tag_uuid]['notification_muted'] == False
+
+    # Get single tag
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json['title'] == tag_title
+
+    # Update tag
+    res = client.put(
+        url_for("tag", uuid=new_tag_uuid),
+        data=json.dumps({"title": "Updated Tag"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert b'OK' in res.data
+
+    # Verify update worked
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json['title'] == 'Updated Tag'
+
+    # Mute tag notifications
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid) + "?muted=muted",
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert b'OK' in res.data
+
+    # Verify muted status
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json['notification_muted'] == True
+
+    # Unmute tag
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid) + "?muted=unmuted",
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert b'OK' in res.data
+
+    # Verify unmuted status
+    res = client.get(
+        url_for("tag", uuid=new_tag_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json['notification_muted'] == False
+
+    # Create a watch with the tag and check it matches UUID
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, "tag": "Updated Tag", "title": "Watch with tag"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+    assert res.status_code == 201
+    watch_uuid = res.json.get('uuid')
+
+    # Verify tag is associated with watch by name if need be
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert new_tag_uuid in res.json.get('tags', [])
+
+    # Delete tag
+    res = client.delete(
+        url_for("tag", uuid=new_tag_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 204
+
+    # Verify tag is gone
+    res = client.get(
+        url_for("tags"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert new_tag_uuid not in res.text
+
+    # Verify tag was removed from watch
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert new_tag_uuid not in res.json.get('tags', [])
+
+    # Delete the watch
+    res = client.delete(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    assert res.status_code == 204

changedetectionio/tests/test_auth.py

@@ -13,7 +13,7 @@ def test_basic_auth(client, live_server, measure_memory_usage):
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -22,7 +22,7 @@ def test_basic_auth(client, live_server, measure_memory_usage):
     time.sleep(1)
     # Check form validation
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -30,7 +30,7 @@ def test_basic_auth(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI, wait_for_all_checks
+from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
 
 
 def set_response_with_ldjson():
@@ -87,7 +87,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -95,22 +95,22 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     wait_for_all_checks(client)
 
     # Should get a notice that it's available
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Embedded price data' not in res.data
     assert b'tracking-ldjson-price-data' in res.data
 
     # and last snapshop (via API) should be just the price
-    api_key = extract_api_key_from_UI(client)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     res = client.get(
         url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
         headers={'x-api-key': api_key},
@@ -121,7 +121,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     # And not this cause its not the ld-json
     assert b"So let's see what happens" not in res.data
 
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
     ##########################################################################################
     # And we shouldnt see the offer
@@ -130,24 +130,24 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'ldjson-price-track-offer' not in res.data
     
     ##########################################################################################
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
 
 def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -160,7 +160,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
 
 
     ##########################################################################################
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
 
 def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):

changedetectionio/tests/test_backend.py

@@ -22,7 +22,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
@@ -33,13 +33,13 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
@@ -53,7 +53,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     # Check HTML conversion detected and workd
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     # Check this class does not appear (that we didnt see the actual source)
@@ -63,24 +63,24 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     set_modified_response()
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Check the 'get latest snapshot works'
-    res = client.get(url_for("watch_get_latest_html", uuid=uuid))
+    res = client.get(url_for("ui.ui_edit.watch_get_latest_html", uuid=uuid))
     assert b'which has this one new line' in res.data
 
     # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
     rss_token = extract_rss_token_from_UI(client)
-    res = client.get(url_for("rss", token=rss_token, _external=True))
+    res = client.get(url_for("rss.feed", token=rss_token, _external=True))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
 
@@ -91,12 +91,12 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert expected_url.encode('utf-8') in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
-    res = client.get(url_for("diff_history_page", uuid=uuid))
+    res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
     assert b'selected=""' in res.data, "Confirm diff history page loaded"
 
     # Check the [preview] pulls the right one
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'which has this one new line' in res.data
@@ -106,13 +106,13 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'unviewed' not in res.data
         assert b'Mark all viewed' not in res.data
         assert b'head title' not in res.data  # Should not be present because this is off by default
@@ -122,16 +122,16 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     # Enable auto pickup of <title> in settings
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
     assert b'Mark all viewed' in res.data
 
@@ -142,19 +142,19 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     time.sleep(1)
 
     # hit the mark all viewed link
-    res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
 
     assert b'Mark all viewed' not in res.data
     assert b'unviewed' not in res.data
 
     # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
-    client.get(url_for("clear_watch_history", uuid=uuid))
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.clear_watch_history", uuid=uuid))
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'preview/' in res.data
 
     #
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_backup.py

@@ -18,7 +18,7 @@ def test_backup(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": url_for('test_endpoint', _external=True)+"?somechar=őőőőőőőő"},
         follow_redirects=True
     )

changedetectionio/tests/test_block_while_text_present.py

@@ -71,7 +71,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -83,7 +83,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"text_should_not_be_present": ignore_text,
               "url": test_url,
               'fetch_backend': "html_requests"
@@ -96,18 +96,18 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     wait_for_all_checks(client)
     # Check it saved
     res = client.get(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
     )
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -115,33 +115,33 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     set_modified_original_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
     # 2548
     # Going back to the ORIGINAL should NOT trigger a change
     set_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
     set_modified_response_minus_block_text()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
 
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_clone.py

@@ -2,29 +2,39 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
-
-def test_trigger_functionality(client, live_server, measure_memory_usage):
+def test_clone_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html><body>Some content</body></html>")
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    test_url = url_for('test_endpoint', _external=True)
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
-        data={"urls": "https://changedetection.io"},
+        url_for("imports.import_page"),
+        data={"urls": test_url},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
 
+    # So that we can be sure the same history doesnt carry over
+    time.sleep(1)
 
     res = client.get(
-        url_for("form_clone", uuid="first"),
+        url_for("ui.form_clone", uuid="first"),
         follow_redirects=True
     )
+    existing_uuids = set()
 
-    assert b"Cloned." in res.data
+    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():
+        new_uuids = set(watch.history.keys())
+        duplicates = existing_uuids.intersection(new_uuids)
+        assert len(duplicates) == 0
+        existing_uuids.update(new_uuids)
+
+    assert b"Cloned" in res.data

changedetectionio/tests/test_conditions.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import json
+import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -53,7 +55,7 @@ def test_conditions_with_text_and_number(client, live_server):
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -64,7 +66,7 @@ def test_conditions_with_text_and_number(client, live_server):
     # 1. The page filtered text must contain "5" (first digit of value)
     # 2. The extracted number should be >= 20 and <= 100
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
             "fetch_backend": "html_requests",
@@ -103,31 +105,93 @@ def test_conditions_with_text_and_number(client, live_server):
     assert b"Updated watch." in res.data
 
     wait_for_all_checks(client)
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     wait_for_all_checks(client)
 
     # Case 1
     set_number_in_range_response("70.5")
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
+    time.sleep(2)
     # 75 is > 20 and < 100 and contains "5"
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
     # Case 2: Change with one condition violated
     # Number out of range (150) but contains '5'
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     set_number_out_of_range_response("150.5")
 
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
     # Should NOT be marked as having changes since not all conditions are met
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+# The 'validate' button next to each rule row
+def test_condition_validate_rule_row(client, live_server):
+
+    set_original_response("50")
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
+    # the front end submits the current form state which should override the watch in a temporary copy
+    res = client.post(
+        url_for("conditions.verify_condition_single_rule", watch_uuid=uuid),  # Base URL
+        query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
+        data={'include_filter': ""},
+        follow_redirects=True
+    )
+    assert res.status_code == 200
+    assert b'success' in res.data
+
+    # Now a number that does not equal what is found in the last fetch
+    res = client.post(
+        url_for("conditions.verify_condition_single_rule", watch_uuid=uuid),  # Base URL
+        query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "111111"})},
+        data={'include_filter': ""},
+        follow_redirects=True
+    )
+    assert res.status_code == 200
+    assert b'false' in res.data
+
+    # Now custom filter that exists
+    res = client.post(
+        url_for("conditions.verify_condition_single_rule", watch_uuid=uuid),  # Base URL
+        query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
+        data={'include_filter': ".number-container"},
+        follow_redirects=True
+    )
+    assert res.status_code == 200
+    assert b'success' in res.data
+
+    # Now custom filter that DOES NOT exists
+    res = client.post(
+        url_for("conditions.verify_condition_single_rule", watch_uuid=uuid),  # Base URL
+        query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
+        data={'include_filters': ".NOT-container"},
+        follow_redirects=True
+    )
+    assert res.status_code == 200
+    assert b'false' in res.data
+
+
+

changedetectionio/tests/test_css_selector.py

@@ -83,7 +83,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -95,7 +95,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -103,7 +103,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     time.sleep(1)
     # Check it saved
     res = client.get(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
     )
     assert bytes(include_filters.encode('utf-8')) in res.data
 
@@ -113,13 +113,13 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
@@ -140,7 +140,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -150,7 +150,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
               "tags": "",
@@ -164,7 +164,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -194,7 +194,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -204,7 +204,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
               "tags": "",
@@ -218,7 +218,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
 
 
     res = client.get(
-        url_for("index"),
+        url_for("watchlist.index"),
         follow_redirects=True
     )
 
@@ -236,11 +236,11 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
          </html>
         """)
 
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("index"),
+        url_for("watchlist.index"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_element_removal.py

@@ -156,7 +156,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
     res = client.post(
-        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+        url_for("imports.import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
@@ -165,7 +165,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
@@ -180,31 +180,31 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
 
     # Check it saved
     res = client.get(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
     )
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
 
     # so that we set the state to 'unviewed' after all the edits
-    client.get(url_for("diff_history_page", uuid="first"))
+    client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav
     set_modified_response()
 
     # Trigger a check
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # There should not be an unviewed change, as changes should be removed
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"unviewed" not in res.data
 
 # Re #2752
@@ -228,19 +228,19 @@ body > table > tr:nth-child(3) > td:nth-child(3)""",
 
     for selector_list in subtractive_selectors_data:
 
-        res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+        res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
         assert b'Deleted' in res.data
 
         # Add our URL to the import page
         test_url = url_for("test_endpoint", _external=True)
         res = client.post(
-            url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+            url_for("imports.import_page"), data={"urls": test_url}, follow_redirects=True
         )
         assert b"1 Imported" in res.data
         wait_for_all_checks(client)
 
         res = client.post(
-            url_for("edit_page", uuid="first"),
+            url_for("ui.ui_edit.edit_page", uuid="first"),
             data={
                 "subtractive_selectors": selector_list,
                 "url": test_url,
@@ -253,7 +253,7 @@ body > table > tr:nth-child(3) > td:nth-child(3)""",
         wait_for_all_checks(client)
 
         res = client.get(
-            url_for("preview_page", uuid="first"),
+            url_for("ui.ui_views.preview_page", uuid="first"),
             follow_redirects=True
         )
 

changedetectionio/tests/test_encoding.py

@@ -30,7 +30,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -44,7 +44,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -61,7 +61,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -69,7 +69,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_errorhandling.py

@@ -23,7 +23,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
                        _external=True)
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -32,7 +32,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     # no change
     assert b'unviewed' not in res.data
     assert bytes(expected_text.encode('utf-8')) in res.data
@@ -40,7 +40,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
 
     # Error viewing tabs should appear
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -50,7 +50,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     #assert b'Error Screenshot' in res.data
 
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
@@ -59,7 +59,7 @@ def test_http_error_handler(client, live_server, measure_memory_usage):
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
     _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
     _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 # Just to be sure error text is properly handled
@@ -69,7 +69,7 @@ def test_DNS_errors(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": "https://errorfuldomainthatnevereallyexists12356.com"},
         follow_redirects=True
     )
@@ -78,12 +78,12 @@ def test_DNS_errors(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 # Re 1513
@@ -99,7 +99,7 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
     test_url = url_for('test_endpoint', _external=True)
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": "https://dfkjasdkfjaidjfsdajfksdajfksdjfDOESNTEXIST.com"},
         follow_redirects=True
     )
@@ -107,13 +107,13 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
     wait_for_all_checks(client)
 
     # We should see the DNS error
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
 
     # Update with what should work
     client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
             "fetch_backend": "html_requests"},
@@ -122,9 +122,9 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
 
     # Now the error should be gone
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert not found_name_resolution_error
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_extract_csv.py

@@ -18,7 +18,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
 
     # Add our URL to the import page
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
@@ -36,11 +36,11 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
         with open("test-datastore/endpoint-content.txt", "w") as f:
             f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
 
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
     res = client.post(
-        url_for("diff_history_page", uuid="first"),
+        url_for("ui.ui_views.diff_history_page", uuid="first"),
         data={"extract_regex": "Now it's ([0-9\.]+)",
               "extract_submit_button": "Extract as CSV"},
         follow_redirects=False

changedetectionio/tests/test_extract_regex.py

@@ -77,7 +77,7 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -88,7 +88,7 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": '',
               # Test a regex and a plaintext
               'extract_text': '/something.+?6 billion.+?lines/si\r\nand this should be',
@@ -103,13 +103,13 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     # Issue 1828
     assert b'not at the start of the expression' not in res.data
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     # Plaintext that doesnt look like a regex should match also
@@ -131,7 +131,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -143,7 +143,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": include_filters,
               'extract_text': '/\d+ online/\r\n/\d+ guests/\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i\r\n/issue1828.+?2022/i',
               "url": test_url,
@@ -160,7 +160,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     #issue 1828
     assert b'not at the start of the expression' not in res.data
 
@@ -168,18 +168,18 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     # Check HTML conversion detected and workd
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -211,7 +211,7 @@ def test_regex_error_handling(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -219,7 +219,7 @@ def test_regex_error_handling(client, live_server, measure_memory_usage):
 
     ### test regex error handling
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"extract_text": '/something bad\d{3/XYZ',
               "url": test_url,
               "fetch_backend": "html_requests"},
@@ -228,5 +228,5 @@ def test_regex_error_handling(client, live_server, measure_memory_usage):
 
     assert b'is not a valid regular expression.' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_filter_exist_changes.py

@@ -55,7 +55,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'cinema'},
         follow_redirects=True
     )
@@ -97,7 +97,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         "fetch_backend": "html_requests"})
 
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data=notification_form_data,
         follow_redirects=True
     )
@@ -108,7 +108,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     assert not os.path.isfile("test-datastore/notification.txt")
     # Now the filter should exist
     set_response_with_filter()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     wait_for_notification_endpoint_output()
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -36,14 +36,14 @@ def run_filter_test(client, live_server, content_filter):
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
     if os.path.isfile("test-datastore/notification.txt"):
         os.unlink("test-datastore/notification.txt")
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -80,7 +80,7 @@ def run_filter_test(client, live_server, content_filter):
                   }
 
     res = client.post(
-        url_for("edit_page", uuid=uuid),
+        url_for("ui.ui_edit.edit_page", uuid=uuid),
         data=watch_data,
         follow_redirects=True
     )
@@ -91,7 +91,7 @@ def run_filter_test(client, live_server, content_filter):
     # Now add a filter, because recheck hours == 5, ONLY pressing of the [edit] or [recheck all] should trigger
     watch_data['include_filters'] = content_filter
     res = client.post(
-        url_for("edit_page", uuid=uuid),
+        url_for("ui.ui_edit.edit_page", uuid=uuid),
         data=watch_data,
         follow_redirects=True
     )
@@ -111,9 +111,9 @@ def run_filter_test(client, live_server, content_filter):
     ATTEMPT_THRESHOLD_SETTING = live_server.app.config['DATASTORE'].data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
     for i in range(0, ATTEMPT_THRESHOLD_SETTING - 2):
         checked += 1
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'Warning, no filters were found' in res.data
         assert not os.path.isfile("test-datastore/notification.txt")
         time.sleep(1)
@@ -122,7 +122,7 @@ def run_filter_test(client, live_server, content_filter):
 
     time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     wait_for_notification_endpoint_output()
 
@@ -141,7 +141,7 @@ def run_filter_test(client, live_server, content_filter):
 
     # Try several times, it should NOT have 'filter not found'
     for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
     wait_for_notification_endpoint_output()
@@ -157,7 +157,7 @@ def run_filter_test(client, live_server, content_filter):
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
     os.unlink("test-datastore/notification.txt")

changedetectionio/tests/test_group.py

@@ -71,13 +71,13 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'import-tag' in res.data
     assert b'extra-import-tag' in res.data
 
@@ -90,18 +90,18 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Warning, no filters were found' not in res.data
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'Should be only this' in res.data
     assert b'And never this' not in res.data
 
     res = client.get(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         follow_redirects=True
     )
     # 2307 the UI notice should appear in the placeholder
@@ -110,24 +110,24 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     # RSS Group tag filter
     # An extra one that should be excluded
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url + "?should-be-excluded=1 some-tag"},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
     set_modified_response()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     rss_token = extract_rss_token_from_UI(client)
     res = client.get(
-        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
+        url_for("rss.feed", token=rss_token, tag="extra-import-tag", _external=True),
         follow_redirects=True
     )
     assert b"should-be-excluded" not in res.data
     assert res.status_code == 200
     assert b"first-imported=1" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_tag_import_singular(client, live_server, measure_memory_usage):
@@ -135,7 +135,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage):
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
         follow_redirects=True
     )
@@ -147,7 +147,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage):
     )
     # Should be only 1 tag because they both had the same
     assert res.data.count(b'test-tag') == 1
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_tag_add_in_ui(client, live_server, measure_memory_usage):
@@ -164,7 +164,7 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage):
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_group_tag_notification(client, live_server, measure_memory_usage):
@@ -173,7 +173,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'test-tag, other-tag'},
         follow_redirects=True
     )
@@ -211,7 +211,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     set_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
 
     assert os.path.isfile("test-datastore/notification.txt")
@@ -232,7 +232,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
 
     #@todo Test that multiple notifications fired
     #@todo Test that each of multiple notifications with different settings
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_limit_tag_ui(client, live_server, measure_memory_usage):
@@ -248,14 +248,14 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
         urls.append(test_url+"?non-grouped="+str(i))
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": "\r\n".join(urls)},
         follow_redirects=True
     )
 
     assert b"40 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
 
     # All should be here
@@ -263,39 +263,41 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
 
     tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
 
-    res = client.get(url_for("index", tag=tag_uuid))
+    res = client.get(url_for("watchlist.index", tag=tag_uuid))
 
     # Just a subset should be here
     assert b'test-tag' in res.data
     assert res.data.count(b'processor-text_json_diff') == 20
     assert b"object at" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
+
 def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url + " test-tag, another-tag\r\n"},
         follow_redirects=True
     )
 
     assert b"1 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
     watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
+    res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
+    res = client.get(url_for("watchlist.index"))
     # 2 times plus the top link to tag
     assert res.data.count(b'test-tag') == 3
     assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
@@ -304,25 +306,26 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     test_url = url_for('test_endpoint', _external=True)
 
     res = client.post(
-        url_for("form_quick_watch_add"),
+        url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": ' test-tag, another-tag      '},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
     watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
+    res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True)
     assert b'Cloned' in res.data
+
+    res = client.get(url_for("watchlist.index"))
     # 2 times plus the top link to tag
     assert res.data.count(b'test-tag') == 3
     assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
@@ -387,7 +390,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -414,7 +417,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
             ]
 
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": '\n'.join(filters),
             "url": test_url,
             "tags": "test-tag-keep-order",
@@ -426,7 +429,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("preview_page", uuid="first"),
+        url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
 
@@ -476,5 +479,5 @@ the {test} appeared before. {test in res.data[:n]=}
         """
         n += t_index + len(test)
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_history_consistency.py

@@ -16,7 +16,7 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     for one in r:
         test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
         res = client.post(
-            url_for("import_page"),
+            url_for("imports.import_page"),
             data={"urls": test_url},
             follow_redirects=True
         )
@@ -27,7 +27,7 @@ def test_consistent_history(client, live_server, measure_memory_usage):
 
     # Essentially just triggers the DB write/update
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},

changedetectionio/tests/test_ignore.py

@@ -28,7 +28,7 @@ def test_ignore(client, live_server, measure_memory_usage):
     set_original_ignore_response()
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -39,12 +39,12 @@ def test_ignore(client, live_server, measure_memory_usage):
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     # use the highlighter endpoint
     res = client.post(
-        url_for("highlight_submit_ignore_url", uuid=uuid),
+        url_for("ui.ui_edit.highlight_submit_ignore_url", uuid=uuid),
         data={"mode": 'digit-regex', 'selection': 'oh yeah 123'},
         follow_redirects=True
     )
 
-    res = client.get(url_for("edit_page", uuid=uuid))
+    res = client.get(url_for("ui.ui_edit.edit_page", uuid=uuid))
     # should be a regex now
     assert b'/oh\ yeah\ \d+/' in res.data
 
@@ -52,7 +52,7 @@ def test_ignore(client, live_server, measure_memory_usage):
     assert b'href' in res.data
 
     # It should not be in the preview anymore
-    res = client.get(url_for("preview_page", uuid=uuid))
+    res = client.get(url_for("ui.ui_views.preview_page", uuid=uuid))
     assert b'<div class="ignored">oh yeah 456' not in res.data
 
     # Should be in base.html

changedetectionio/tests/test_ignore_text.py

@@ -96,7 +96,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -108,7 +108,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -116,18 +116,18 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
 
     # Check it saved
     res = client.get(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
     )
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -135,12 +135,12 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -148,20 +148,20 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
 
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("preview_page", uuid="first"))
+    res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
     # SHOULD BE be in the preview, it was added in set_modified_original_ignore_response()
     # and we have "new ignore stuff" in ignore_text
     # it is only ignored, it is not removed (it will be highlighted too)
     assert b'new ignore stuff' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 # When adding some ignore text, it should not trigger a change, even if something else on that line changes
@@ -172,7 +172,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     # Goto the settings page, add our ignore text
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
@@ -187,7 +187,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -198,7 +198,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     #Adding some ignore text should not trigger a change
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -206,15 +206,15 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     # Check it saved
     res = client.get(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
     )
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # It should report nothing found (no new 'unviewed' class), adding random ignore text should not cause a change
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 #####
@@ -224,22 +224,22 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -53,7 +53,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
 
     # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-fetch_backend": "html_requests",
@@ -65,32 +65,32 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
     res = client.post(
-        url_for("import_page"), data={"urls": test_url},
+        url_for("imports.import_page"), data={"urls": test_url},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
     time.sleep(sleep_time_for_fetch_thread)
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
     # We should not see the rendered anchor tag
-    res = client.get(url_for("preview_page", uuid="first"))
+    res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
     assert '(/modified_link)' not in res.data.decode()
 
     # Goto the settings page, ENABLE render anchor tag
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-render_anchor_tag_content": "true",
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     assert b"Settings updated." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -109,17 +109,17 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
 
 
     # check that the anchor tag content is rendered
-    res = client.get(url_for("preview_page", uuid="first"))
+    res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
     assert '(/modified_link)' in res.data.decode()
 
     # since the link has changed, and we chose to render anchor tag content,
     # we should detect a change (new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"unviewed" in res.data
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"),
+    res = client.get(url_for("ui.form_delete", uuid="all"),
                      follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -49,7 +49,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
 
     # Goto the settings page, add our ignore text
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_status_codes": "y",
@@ -62,7 +62,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -73,13 +73,13 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
     set_some_changed_response()
     wait_for_all_checks(client)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
     assert b'/test-endpoint' in res.data
 
@@ -96,7 +96,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     # Add our URL to the import page
     test_url = url_for('test_endpoint', status_code=403, _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -108,7 +108,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     # Goto the edit page, check our ignore option
     # Add our URL to the import page
     res = client.post(
-        url_for("edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -121,12 +121,12 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -59,7 +59,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
 
     # Goto the settings page, add our ignore text
     res = client.post(
-        url_for("settings_page"),
+        url_for("settings.settings_page"),
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
@@ -72,7 +72,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={"urls": test_url},
         follow_redirects=True
     )
@@ -80,17 +80,17 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data

changedetectionio/tests/test_import.py

@@ -16,7 +16,7 @@ def test_import(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={
             "distill-io": "",
             "urls": """https://example.com
@@ -28,11 +28,11 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
     # Clear flask alerts
-    res = client.get( url_for("index"))
-    res = client.get( url_for("index"))
+    res = client.get( url_for("watchlist.index"))
+    res = client.get( url_for("watchlist.index"))
 
 def xtest_import_skip_url(client, live_server, measure_memory_usage):
 
@@ -41,7 +41,7 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage):
     time.sleep(1)
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={
             "distill-io": "",
             "urls": """https://example.com
@@ -53,9 +53,9 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
     assert b"ht000000broken" in res.data
     assert b"1 Skipped" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
-    res = client.get( url_for("index"))
+    res = client.get( url_for("watchlist.index"))
 
 def test_import_distillio(client, live_server, measure_memory_usage):
 
@@ -82,9 +82,9 @@ def test_import_distillio(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data={
             "distill-io": distill_data,
             "urls" : ''
@@ -96,7 +96,7 @@ def test_import_distillio(client, live_server, measure_memory_usage):
     assert b"Unable to read JSON file, was it broken?" not in res.data
     assert b"1 Imported from Distill.io" in res.data
 
-    res = client.get( url_for("edit_page", uuid="first"))
+    res = client.get( url_for("ui.ui_edit.edit_page", uuid="first"))
 
     assert b"https://unraid.net/blog" in res.data
     assert b"Unraid | News" in res.data
@@ -113,15 +113,15 @@ def test_import_distillio(client, live_server, measure_memory_usage):
     assert b"xpath:(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]" in res.data
 
     # did the tags work?
-    res = client.get( url_for("index"))
+    res = client.get( url_for("watchlist.index"))
 
     # check tags
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
 def test_import_custom_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
@@ -146,7 +146,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage):
         }
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data=data,
         follow_redirects=True,
     )
@@ -156,7 +156,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage):
     assert b'Error processing row number 1' in res.data
 
     res = client.get(
-        url_for("index")
+        url_for("watchlist.index")
     )
 
     assert b'Somesite results ABC' in res.data
@@ -169,7 +169,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage):
             assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]'
             assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0}
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
@@ -186,7 +186,7 @@ def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
         }
 
     res = client.post(
-        url_for("import_page"),
+        url_for("imports.import_page"),
         data=data,
         follow_redirects=True,
     )
@@ -194,7 +194,7 @@ def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
     assert b'4 imported from Wachete .xlsx' in res.data
 
     res = client.get(
-        url_for("index")
+        url_for("watchlist.index")
     )
 
     assert b'Somesite results ABC' in res.data
@@ -214,5 +214,5 @@ def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
         if watch.get('title') == 'system default website':
             assert watch.get('fetch_backend') == 'system' # uses default if blank
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data