UI/Functionality - Ability to manage/apply filters and notifications across tags/groups

changedetectionio/__init__.py

@@ -317,23 +317,19 @@ def changedetection_app(config=None, datastore_o=None):
             return "Access denied, bad token", 403
 
         from . import diff
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-
+            if limit_tag and not limit_tag in watch['tags']:
-            if limit_tag != None:
+                    continue
-                # Support for comma separated list of tags.
-                for tag_in_watch in watch['tag'].split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        sorted_watches.append(watch)
-
-            else:
             watch['uuid'] = uuid
             sorted_watches.append(watch)
 
@@ -392,9 +388,17 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_optionally_required
     def index():
+        global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
             return redirect(url_for('rss', tag=limit_tag))
@@ -414,30 +418,15 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches = []
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-
+            if limit_tag and not limit_tag in watch['tags']:
-            if limit_tag:
-                # Support for comma separated list of tags.
-                if not watch.get('tag'):
                     continue
-                for tag_in_watch in watch.get('tag', '').split(','):
+
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
             else:
                 sorted_watches.append(watch)
 
-            else:
-                #watch['uuid'] = uuid
-                if search_q:
-                    if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                        sorted_watches.append(watch)
-                else:
-                    sorted_watches.append(watch)
-
-        existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
         total_count = len(sorted_watches)
@@ -452,6 +441,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 datastore=datastore,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -463,7 +453,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=existing_tags,
+                                 tags=datastore.data['settings']['application'].get('tags'),
                                  watches=sorted_watches
                                  )
 
@@ -606,9 +596,13 @@ def changedetection_app(config=None, datastore_o=None):
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
+                               data=default
                                )
 
+        # For the form widget tag uuid lookup
+        form.tags.datastore = datastore # in _value
+
+
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
         # form.browser_steps[0] can be assumed that we 'goto url' first
@@ -659,6 +653,16 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(name=t))
+                    extra_update_obj['tags'] = tag_uuids
 
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
@@ -713,7 +717,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
-                                     has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
+                                     has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1110,8 +1114,8 @@ def changedetection_app(config=None, datastore_o=None):
                 os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
             ) as f:
                 for uuid in datastore.data["watching"]:
-                    url = datastore.data["watching"][uuid]["url"]
+                    url = datastore.data["watching"][uuid].get('url')
-                    tag = datastore.data["watching"][uuid]["tag"]
+                    tag = datastore.data["watching"][uuid].get('tags', {})
                     f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
@@ -1199,7 +1203,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -1267,9 +1271,11 @@ def changedetection_app(config=None, datastore_o=None):
         elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch['tag']):
+                if (tag != None and tag in watch.get('tags', {})):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
+                        update_q.put(
+                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
+                        )
                         i += 1
 
         else:
@@ -1357,6 +1363,17 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
+        elif (op == 'assign-tag'):
+            op_extradata = request.form.get('op_extradata')
+            tag_uuid = datastore.add_tag(name=op_extradata)
+            if op_extradata and tag_uuid:
+                for uuid in uuids:
+                    uuid = uuid.strip()
+                    if datastore.data['watching'].get(uuid):
+                        datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+
+            flash("{} watches assigned tag".format(len(uuids)))
+
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1366,7 +1383,6 @@ def changedetection_app(config=None, datastore_o=None):
            the share-link can be imported/added"""
         import requests
         import json
-        tag = request.args.get('tag')
         uuid = request.args.get('uuid')
 
         # more for testing
@@ -1419,6 +1435,8 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.price_data_follower as price_data_follower
     app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
 
+    import changedetectionio.blueprint.tags as tags
+    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

changedetectionio/api/api_v1.py

@@ -218,6 +218,11 @@ class CreateWatch(Resource):
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         extras = copy.deepcopy(json_data)
+
+        # Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
+        if extras.get('tag'):
+            extras['tags'] = extras.get('tag')
+
         del extras['url']
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras)
@@ -259,13 +264,16 @@ class CreateWatch(Resource):
         """
         list = {}
 
-        tag_limit = request.args.get('tag', None)
+        tag_limit = request.args.get('tag', '').lower()
-        for k, watch in self.datastore.data['watching'].items():
+
-            if tag_limit:
+
-                if not tag_limit.lower() in watch.all_tags:
+        for uuid, watch in self.datastore.data['watching'].items():
+            # Watch tags by name (replace the other calls?)
+            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
+            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
                 continue
 
-            list[k] = {'url': watch['url'],
+            list[uuid] = {'url': watch['url'],
                        'title': watch['title'],
                        'last_checked': watch['last_checked'],
                        'last_changed': watch.last_changed,

changedetectionio/blueprint/tags/README.md

@@ -0,0 +1,9 @@
+# Groups tags
+
+## How it works
+
+Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
+
+The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
+
+So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -0,0 +1,131 @@
+from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
+
+    @tags_blueprint.route("/list", methods=['GET'])
+    @login_optionally_required
+    def tags_overview_page():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+        output = render_template("groups-overview.html",
+                                 form=add_form,
+                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
+                                 )
+
+        return output
+
+    @tags_blueprint.route("/add", methods=['POST'])
+    @login_optionally_required
+    def form_tag_add():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+
+        if not add_form.validate():
+            for widget, l in add_form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('tags.tags_overview_page'))
+
+        title = request.form.get('name').strip()
+
+        if datastore.tag_exists_by_name(title):
+            flash(f'The tag "{title}" already exists', "error")
+            return redirect(url_for('tags.tags_overview_page'))
+
+        datastore.add_tag(title)
+        flash("Tag added")
+
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def mute(uuid):
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def delete(uuid):
+        removed = 0
+        # Delete the tag, and any tag reference
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            del datastore.data['settings']['application']['tags'][uuid]
+
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                removed += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag deleted and removed from {removed} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def unlink(uuid):
+        unlinked = 0
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                unlinked += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag unlinked removed from {unlinked} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_tag_edit(uuid):
+        from changedetectionio import forms
+
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        form.datastore=datastore # needed?
+
+        output = render_template("edit-tag.html",
+                                 data=default,
+                                 form=form,
+                                 settings_application=datastore.data['settings']['application'],
+                                 )
+
+        return output
+
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def form_tag_edit_submit(uuid):
+        from changedetectionio import forms
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        # @todo subclass form so validation works
+        #if not form.validate():
+#            for widget, l in form.errors.items():
+#                flash(','.join(l), 'error')
+#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
+
+        datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.needs_write_urgent = True
+        flash("Updated")
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    def form_tag_delete(uuid):
+        return redirect(url_for('tags.tags_overview_page'))
+    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -0,0 +1,22 @@
+from wtforms import (
+    BooleanField,
+    Form,
+    IntegerField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    validators,
+)
+
+
+
+class SingleTag(Form):
+
+    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+
+

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -0,0 +1,131 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+
+/*{% if emailprefix %}*/
+    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
+/*{% endif %}*/
+
+
+</script>
+
+<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
+<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                    <div class="pure-control-group">
+                        {% set field = render_field(form.include_filters,
+                            rows=5,
+                            placeholder="#example
+xpath://body/div/span[contains(@class, 'example-class')]",
+                            class="m-d")
+                        %}
+                        {{ field }}
+                        {% if '/text()' in  field %}
+                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
+                        {% endif %}
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
+
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
+                            <ul>
+                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
+                                {% if jq_support %}
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
+                                {% else %}
+                                <li>jq support not installed</li>
+                                {% endif %}
+                            </ul>
+                        </li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
+                            <ul>
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
+                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
+                            </ul>
+                            </li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
+                </span>
+                    </div>
+                <fieldset class="pure-control-group">
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                </fieldset>
+
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_muted) }}
+                    </div>
+                    {% if is_html_webdriver %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
+                    <div class="field-group" id="notification-field-group">
+                        {% if has_default_notification_urls %}
+                        <div class="inline-warning">
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
+                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                        </div>
+                        {% endif %}
+                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
+
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -0,0 +1,60 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
+<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+
+<div class="box">
+    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <fieldset>
+            <legend>Add a new organisational tag</legend>
+            <div id="watch-add-wrapper-zone">
+                <div>
+                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.save_button, title="Save" ) }}
+                </div>
+            </div>
+            <br>
+            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
+        </fieldset>
+    </form>
+    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
+    <div id="watch-table-wrapper">
+
+        <table class="pure-table pure-table-striped watch-table group-overview-table">
+            <thead>
+            <tr>
+                <th></th>
+                <th>Tag / Label name</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+            <!--
+            @Todo - connect Last checked, Last Changed, Number of Watches etc
+            --->
+            {% if not available_tags|length %}
+            <tr>
+                <td colspan="3">No website organisational tags/groups configured</td>
+            </tr>
+            {% endif %}
+            {% for uuid, tag in available_tags.items()  %}
+            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
+                <td class="watch-controls">
+                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                </td>
+                <td class="title-col inline">{{tag.title}}</td>
+                <td>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+{% endblock %}

changedetectionio/forms.py

@@ -28,6 +28,8 @@ from changedetectionio.notification import (
 
 from wtforms.fields import FormField
 
+dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+
 valid_method = {
     'GET',
     'POST',
@@ -90,6 +92,29 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
+class StringTagUUID(StringField):
+
+   # process_formdata(self, valuelist) handled manually in POST handler
+
+    # Is what is shown when field <input> is rendered
+    def _value(self):
+        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
+        if self.data and type(self.data) is list:
+            tag_titles = []
+            for i in self.data:
+                tag = self.datastore.data['settings']['application']['tags'].get(i)
+                if tag:
+                    tag_title = tag.get('title')
+                    if tag_title:
+                        tag_titles.append(tag_title)
+
+            return ', '.join(tag_titles)
+
+        if not self.data:
+            return ''
+
+        return 'error'
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -347,7 +372,7 @@ class quickWatchForm(Form):
     from . import processors
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()])
+    tags = StringTagUUID('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
@@ -355,6 +380,7 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
+
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -382,7 +408,7 @@ class SingleBrowserStep(Form):
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()], default='')
+    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 

changedetectionio/importer.py

@@ -120,9 +120,9 @@ class import_distill_io_json(Importer):
                 except IndexError:
                     pass
 
-
+# Does this need to be here anymore?
                 if d.get('tags', False):
-                    extras['tag'] = ", ".join(d['tags'])
+                    extras['tags'] = ", ".join(d['tags'])
 
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
                                                extras=extras,

changedetectionio/model/App.py

@@ -43,6 +43,7 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
+                    'tags': {} #@todo use Tag.model initialisers
                 }
             }
         }

changedetectionio/model/Tag.py

@@ -0,0 +1,19 @@
+from .Watch import base_config
+import uuid
+
+class model(dict):
+
+    def __init__(self, *arg, **kw):
+
+        self.update(base_config)
+
+        self['uuid'] = str(uuid.uuid4())
+
+        if kw.get('default'):
+            self.update(kw['default'])
+            del kw['default']
+
+
+        # Goes at the end so we update the default object with the initialiser
+        super(model, self).__init__(*arg, **kw)
+

changedetectionio/model/Watch.py

@@ -52,7 +52,8 @@ base_config = {
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
     'subtractive_selectors': [],
-    'tag': None,
+    'tag': '', # Old system of text name for a tag, to be removed
+    'tags': [], # list of UUIDs to App.Tags
     'text_should_not_be_present': [],  # Text that should not present
     # Re #110, so then if this is set to None, we know to use the default value instead
     # Requires setting to None on submit if it's the same as the default
@@ -455,10 +456,6 @@ class model(dict):
 
         return csv_output_filename
 
-    @property
-    # Return list of tags, stripped and lowercase, used for searching
-    def all_tags(self):
-        return [s.strip().lower() for s in self.get('tag','').split(',')]
 
     def has_special_diff_filter_options_set(self):
 
@@ -473,40 +470,6 @@ class model(dict):
         # None is set
         return False
 
-    @property
-    def has_extra_headers_file(self):
-        if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')):
-            return True
-
-        for f in self.all_tags:
-            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
-            filepath = os.path.join(self.__datastore_path, fname)
-            if os.path.isfile(filepath):
-                return True
-
-        return False
-
-    def get_all_headers(self):
-        from .App import parse_headers_from_text_file
-        headers = self.get('headers', {}).copy()
-        # Available headers on the disk could 'headers.txt' in the watch data dir
-        filepath = os.path.join(self.watch_data_dir, 'headers.txt')
-        try:
-            if os.path.isfile(filepath):
-                headers.update(parse_headers_from_text_file(filepath))
-        except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        # Or each by tag, as tagname.txt in the main datadir
-        for f in self.all_tags:
-            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
-            filepath = os.path.join(self.__datastore_path, fname)
-            try:
-                if os.path.isfile(filepath):
-                    headers.update(parse_headers_from_text_file(filepath))
-            except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
-        return headers
 
     def get_last_fetched_before_filters(self):
         import brotli

changedetectionio/notification.py

@@ -186,8 +186,13 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid != '':
-        watch_title = datastore.data['watching'][uuid]['title']
+        watch_title = datastore.data['watching'][uuid].get('title', '')
-        watch_tag = datastore.data['watching'][uuid]['tag']
+        tag_list = []
+        tags = datastore.get_all_tags_for_watch(uuid)
+        if tags:
+            for tag_uuid, tag in tags.items():
+                tag_list.append(tag.get('title'))
+        watch_tag = ', '.join(tag_list)
     else:
         watch_title = 'Change Detection'
         watch_tag = ''

changedetectionio/processors/restock_diff.py

@@ -42,11 +42,10 @@ class perform_site_check(difference_detection_processor):
 
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
-        extra_headers = watch.get('headers', [])
 
-        # Tweak the base config with the per-watch ones
+        request_headers = watch.get('headers', [])
-        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(extra_headers)
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot

changedetectionio/processors/text_json_diff.py

@@ -57,7 +57,6 @@ class perform_site_check(difference_detection_processor):
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
-
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -71,9 +70,9 @@ class perform_site_check(difference_detection_processor):
         update_obj = {'last_notification_error': False, 'last_error': False}
 
         # Tweak the base config with the per-watch ones
-        extra_headers = watch.get_all_headers()
+        request_headers = watch.get('headers', [])
-        request_headers = self.datastore.get_all_headers()
+        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(extra_headers)
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -191,21 +190,23 @@ class perform_site_check(difference_detection_processor):
 
             fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
 
+        # Better would be if Watch.model could access the global data also
+        # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
+        # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
+        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
+        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
 
-        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
-        # include_filters_rule = watch['include_filters']
+                                 *watch.get("subtractive_selectors", []),
-        subtractive_selectors = watch.get(
+                                 *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
-            "subtractive_selectors", []
+                                 ]
-        ) + self.datastore.data["settings"]["application"].get(
-            "global_subtractive_selectors", []
-        )
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
             include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
 
-        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
+        has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
-        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+        has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
             include_filters_rule.append("json:$")

changedetectionio/static/js/watch-overview.js

@@ -4,6 +4,9 @@ $(function () {
     $(this).closest('.unviewed').removeClass('unviewed');
   });
 
+    $("#checkbox-assign-tag").click(function (e) {
+        $('#op_extradata').val(prompt("Enter a tag name"));
+    });
 
   $('.with-share-link > *').click(function () {
       $("#copied-clipboard").remove();

changedetectionio/store.py

@@ -16,6 +16,8 @@ import threading
 import time
 import uuid as uuid_builder
 
+dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -178,20 +180,6 @@ class ChangeDetectionStore:
 
         return self.__data
 
-    def get_all_tags(self):
-        tags = []
-        for uuid, watch in self.data['watching'].items():
-            if watch['tag'] is None:
-                continue
-            # Support for comma separated list of tags.
-            for tag in watch['tag'].split(','):
-                tag = tag.strip()
-                if tag not in tags:
-                    tags.append(tag)
-
-        tags.sort()
-        return tags
-
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
@@ -218,9 +206,9 @@ class ChangeDetectionStore:
     # Clone a watch by UUID
     def clone(self, uuid):
         url = self.data['watching'][uuid]['url']
-        tag = self.data['watching'][uuid]['tag']
+        tag = self.data['watching'][uuid].get('tags',[])
         extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
+        new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras)
         return new_uuid
 
     def url_exists(self, url):
@@ -255,10 +243,11 @@ class ChangeDetectionStore:
 
         self.needs_write_urgent = True
 
-    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
+    def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
 
         if extras is None:
             extras = {}
+
         # should always be str
         if tag is None or not tag:
             tag = ''
@@ -291,6 +280,7 @@ class ChangeDetectionStore:
                     'processor',
                     'subtractive_selectors',
                     'tag',
+                    'tags',
                     'text_should_not_be_present',
                     'title',
                     'trigger_text',
@@ -313,15 +303,24 @@ class ChangeDetectionStore:
             flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
-        with self.lock:
-            # #Re 569
-            new_watch = Watch.model(datastore_path=self.datastore_path, default={
-                'url': url,
-                'tag': tag,
-                'date_created': int(time.time())
-            })
 
-            new_uuid = new_watch['uuid']
+        # #Re 569
+        # Could be in 'tags',  var or extras, smash them together and strip
+        apply_extras['tags'] = []
+        if tag or extras.get('tags'):
+            tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(',')))))
+            for t in list(map(str.strip, tags)):
+                # for each stripped tag, add tag as UUID
+                apply_extras['tags'].append(self.add_tag(t))
+
+        # Or if UUIDs given directly
+        if tag_uuids:
+            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+
+        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
+
+        new_uuid = new_watch.get('uuid')
+
         logging.debug("Added URL {} - {}".format(url, new_uuid))
 
         for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
@@ -329,9 +328,9 @@ class ChangeDetectionStore:
                 del apply_extras[k]
 
         new_watch.update(apply_extras)
+        new_watch.ensure_data_dir_exists()
         self.__data['watching'][new_uuid] = new_watch
 
-        self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
@@ -511,10 +510,19 @@ class ChangeDetectionStore:
         filepath = os.path.join(self.datastore_path, 'headers.txt')
         return os.path.isfile(filepath)
 
-    def get_all_headers(self):
+    def get_all_base_headers(self):
         from .model.App import parse_headers_from_text_file
-        headers = copy(self.data['settings'].get('headers', {}))
+        headers = {}
+        # Global app settings
+        headers.update(self.data['settings'].get('headers', {}))
 
+        return headers
+
+    def get_all_headers_in_textfile_for_watch(self, uuid):
+        from .model.App import parse_headers_from_text_file
+        headers = {}
+
+        # Global in /datastore/headers.txt
         filepath = os.path.join(self.datastore_path, 'headers.txt')
         try:
             if os.path.isfile(filepath):
@@ -522,8 +530,76 @@ class ChangeDetectionStore:
         except Exception as e:
             print(f"ERROR reading headers.txt at {filepath}", str(e))
 
+        watch = self.data['watching'].get(uuid)
+        if watch:
+
+            # In /datastore/xyz-xyz/headers.txt
+            filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
+            try:
+                if os.path.isfile(filepath):
+                    headers.update(parse_headers_from_text_file(filepath))
+            except Exception as e:
+                print(f"ERROR reading headers.txt at {filepath}", str(e))
+
+            # In /datastore/tag-name.txt
+            tags = self.get_all_tags_for_watch(uuid=uuid)
+            for tag_uuid, tag in tags.items():
+                fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
+                filepath = os.path.join(self.datastore_path, fname)
+                try:
+                    if os.path.isfile(filepath):
+                        headers.update(parse_headers_from_text_file(filepath))
+                except Exception as e:
+                    print(f"ERROR reading headers.txt at {filepath}", str(e))
+
         return headers
 
+    def get_tag_overrides_for_watch(self, uuid, attr):
+        tags = self.get_all_tags_for_watch(uuid=uuid)
+        ret = []
+
+        if tags:
+            for tag_uuid, tag in tags.items():
+                if attr in tag and tag[attr]:
+                    ret=[*ret, *tag[attr]]
+
+        return ret
+
+    def add_tag(self, name):
+        print (">>> Adding new tag -", name)
+        # If name exists, return that
+        n = name.strip().lower()
+        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
+            if n == tag.get('title', '').lower().strip():
+                print (f">>> Tag {name} already exists")
+                return uuid
+
+        # Eventually almost everything todo with a watch will apply as a Tag
+        # So we use the same model as a Watch
+        with self.lock:
+            new_tag = Watch.model(datastore_path=self.datastore_path, default={
+                'title': name.strip(),
+                'date_created': int(time.time())
+            })
+
+            new_uuid = new_tag.get('uuid')
+
+            self.__data['settings']['application']['tags'][new_uuid] = new_tag
+
+        return new_uuid
+
+    def get_all_tags_for_watch(self, uuid):
+        """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
+        watch = self.data['watching'].get(uuid)
+
+        # Should return a dict of full tag info linked by UUID
+        if watch:
+            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
+
+        return {}
+
+    def tag_exists_by_name(self, tag_name):
+        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
@@ -710,3 +786,16 @@ class ChangeDetectionStore:
             i+=1
         return
 
+    # Create tag objects and their references from existing tag text
+    def update_12(self):
+        i = 0
+        for uuid, watch in self.data['watching'].items():
+            # Split out and convert old tag string
+            tag = watch.get('tag')
+            if tag:
+                tag_uuids = []
+                for t in tag.split(','):
+                    tag_uuids.append(self.add_tag(name=t))
+
+                self.data['watching'][uuid]['tags'] = tag_uuids
+

changedetectionio/templates/base.html

@@ -58,6 +58,9 @@
           {% if current_user.is_authenticated or not has_password %}
             {% if not
             current_diff_url %}
+              <li class="pure-menu-item">
+                <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
+              </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
@@ -86,7 +89,7 @@
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
               <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
+              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
               <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
                 {% include "svgs/search-icon.svg" %}
               </button>

changedetectionio/templates/edit.html

@@ -75,7 +75,7 @@
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.tag) }}
+                        {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">

changedetectionio/templates/watch-overview.html

@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -30,12 +30,14 @@
 
     <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
     <div id="checkbox-operations">
         <button class="pure-button button-secondary button-xsmall"  name="op" value="pause">Pause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute">UnMute</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
@@ -47,9 +49,9 @@
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for tag in tags %}
+        {% for uuid, tag in tags.items() %}
             {% if tag != "" %}
-                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -143,9 +145,11 @@
                     </span>
                     {% endif %}
 
-                    {% if not active_tag %}
+
-                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
-                    {% endif %}
+                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
+                    {% endfor %}
+
                 </td>
                 <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@@ -178,7 +182,7 @@
             {% endif %}
             <li>
                <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
-                "tag": "",
+                "tags": "",
                 "url": url,
               },
         follow_redirects=True

changedetectionio/tests/restock/test_restock.py

@@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
+        data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
         follow_redirects=True
     )
 

changedetectionio/tests/test_access_control.py

@@ -45,6 +45,15 @@ def test_check_access_control(app, client, live_server):
         res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
+        # Check wrong password does not let us in
+        res = c.post(
+            url_for("login"),
+            data={"password": "WRONG PASSWORD"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" not in res.data
+        assert b"Incorrect password" in res.data
 
 
         # Menu should not be available yet

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 
@@ -39,7 +39,6 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_check_removed_line_contains_trigger(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -54,7 +53,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -67,20 +66,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -89,14 +88,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
     client.get(url_for("mark_all_viewed"), follow_redirects=True)
     set_original(excluding=None)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -105,8 +104,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
 
 
 def test_check_add_line_contains_trigger(client, live_server):
-
+    #live_server_setup(live_server)
-    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -136,8 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
-
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -150,23 +147,25 @@ def test_check_add_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # Takes a moment for apprise to fire
+    time.sleep(3)
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI
+from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
 
 import json
 import uuid
@@ -57,6 +57,7 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_api_simple(client, live_server):
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -86,7 +87,7 @@ def test_api_simple(client, live_server):
     watch_uuid = res.json.get('uuid')
     assert res.status_code == 201
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Verify its in the list and that recheck worked
     res = client.get(
@@ -107,7 +108,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
@@ -115,7 +116,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch", recheck_all='1'),
         headers={'x-api-key': api_key},
     )
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Did the recheck fire?
     res = client.get(
@@ -297,6 +298,8 @@ def test_api_watch_PUT_update(client, live_server):
         url_for("edit_page", uuid=watch_uuid),
     )
     assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
+    assert b"One" in res.data, "Tag 'One' was found"
+    assert b"Two" in res.data, "Tag 'Two' was found"
 
     # HTTP PUT ( UPDATE an existing watch )
     res = client.put(
@@ -319,7 +322,8 @@ def test_api_watch_PUT_update(client, live_server):
     )
     assert b"new title" in res.data, "new title found in edit page"
     assert b"552" in res.data, "552 minutes found in edit page"
-    assert b"One, Two" in res.data, "Tag 'One, Two' was found"
+    assert b"One" in res.data, "Tag 'One' was found"
+    assert b"Two" in res.data, "Tag 'Two' was found"
     assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
 
     ######################################################

changedetectionio/tests/test_auth.py

@@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 def set_original_ignore_response():
@@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
 
 
 def test_check_block_changedetection_text_NOT_present(client, live_server):
-    sleep_time_for_fetch_thread = 3
+
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
@@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
@@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_css_selector.py

@@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "headers": "",
             "fetch_backend": "html_requests",
         },

changedetectionio/tests/test_extract_regex.py

@@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
         data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },
@@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
         data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },

changedetectionio/tests/test_filter_exist_changes.py

@@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'cinema'},
+        data={"url": test_url, "tags": 'cinema'},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag",
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,7 +1,7 @@
 import os
 import time
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client
+from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
 from changedetectionio.model import App
 
 
@@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -71,8 +71,8 @@ def run_filter_test(client, content_filter):
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag",
-        "title": "my title",
+        "title": "my title 123",
         "headers": "",
         "filter_failure_notification_send": 'y',
         "include_filters": content_filter,
@@ -85,43 +85,55 @@ def run_filter_test(client, content_filter):
     )
 
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
+    # -2 because we would have checked twice above (on adding and on edit)
+    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
         res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        time.sleep(3)
+        wait_for_all_checks(client)
+        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
 
     # We should see something in the frontend
     assert b'Warning, no filters were found' in res.data
 
+    # One more check should trigger it (see -2 above)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
-    notification = False
+
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
+
     assert 'CSS/xPath filter was not present in the page' in notification
     assert content_filter.replace('"', '\\"') in notification
 
-    # Remove it and prove that it doesnt trigger when not expected
+    # Remove it and prove that it doesn't trigger when not expected
+    # It should register a change, but no 'filter not found'
     os.unlink("test-datastore/notification.txt")
     set_response_with_filter()
 
+    # Try several times, it should NOT have 'filter not found'
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        time.sleep(3)
+        wait_for_all_checks(client)
 
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
-    # but it should not contain the info about the failed filter
+    # but it should not contain the info about a failed filter (because there was none in this case)
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced
+    # Re #1247 - All tokens got replaced correctly in the notification
+    res = client.get(url_for("index"))
     uuid = extract_UUID_from_client(client)
+    # UUID is correct, but notification contains tag uuid as UUIID wtf
     assert uuid in notification
 
     # cleanup for the next
@@ -137,7 +149,7 @@ def test_setup(live_server):
 
 def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
-    time.sleep(1)
+    wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):

changedetectionio/tests/test_group.py

@@ -0,0 +1,262 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name
+import os
+
+
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p id="only-this">Should be only this</p>
+     <br>
+     <p id="not-this">And never this</p>     
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p id="only-this">Should be REALLY only this</p>
+     <br>
+     <p id="not-this">And never this</p>     
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def test_setup_group_tag(client, live_server):
+    #live_server_setup(live_server)
+    set_original_response()
+
+    # Add a tag with some config, import a tag and it should roughly work
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+    assert b"test-tag" in res.data
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid="first"),
+        data={"name": "test-tag",
+              "include_filters": '#only-this',
+              "subtractive_selectors": '#not-this'},
+        follow_redirects=True
+    )
+    assert b"Updated" in res.data
+    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
+    res = client.get(
+        url_for("tags.form_tag_edit", uuid="first")
+    )
+    assert b"#only-this" in res.data
+    assert b"#not-this" in res.data
+
+    # Tag should be setup and ready, now add a watch
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'import-tag' in res.data
+    assert b'extra-import-tag' in res.data
+
+    res = client.get(
+        url_for("tags.tags_overview_page"),
+        follow_redirects=True
+    )
+    assert b'import-tag' in res.data
+    assert b'extra-import-tag' in res.data
+
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+    assert b'Warning, no filters were found' not in res.data
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'Should be only this' in res.data
+    assert b'And never this' not in res.data
+
+
+    # RSS Group tag filter
+    # An extra one that should be excluded
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + "?should-be-excluded=1 some-tag"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    set_modified_response()
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    rss_token = extract_rss_token_from_UI(client)
+    res = client.get(
+        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
+        follow_redirects=True
+    )
+    assert b"should-be-excluded" not in res.data
+    assert res.status_code == 200
+    assert b"first-imported=1" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_tag_import_singular(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
+        follow_redirects=True
+    )
+    assert b"2 Imported" in res.data
+
+    res = client.get(
+        url_for("tags.tags_overview_page"),
+        follow_redirects=True
+    )
+    # Should be only 1 tag because they both had the same
+    assert res.data.count(b'test-tag') == 1
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_tag_add_in_ui(client, live_server):
+    #live_server_setup(live_server)
+#
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "new-test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+    assert b"new-test-tag" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_group_tag_notification(client, live_server):
+    #live_server_setup(live_server)
+    set_original_response()
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'test-tag, other-tag'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_form_data = {"notification_urls": notification_url,
+                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Added: {{diff_added}}\n"
+                                                   "Diff Removed: {{diff_removed}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
+                                                   ":-)",
+                              "notification_screenshot": True,
+                              "notification_format": "Text",
+                              "title": "test-tag"}
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
+        data=notification_form_data,
+        follow_redirects=True
+    )
+    assert b"Updated" in res.data
+
+    wait_for_all_checks(client)
+
+    set_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+
+    assert os.path.isfile("test-datastore/notification.txt")
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+    os.unlink("test-datastore/notification.txt")
+
+    # Did we see the URL that had a change, in the notification?
+    # Diff was correctly executed
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
+    assert "Diff Full: Some initial text" in notification_submission
+    assert "New GROUP TAG ChangeDetection.io" in notification_submission
+    assert "test-tag" in notification_submission
+    assert "other-tag" in notification_submission
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    #@todo Test that multiple notifications fired
+    #@todo Test that each of multiple notifications with different settings
+
+
+def test_limit_tag_ui(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+    urls=[]
+
+    for i in range(20):
+        urls.append(test_url+"?x="+str(i)+" test-tag")
+
+    for i in range(20):
+        urls.append(test_url+"?non-grouped="+str(i))
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+
+    assert b"40 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+
+    # All should be here
+    assert res.data.count(b'processor-text_json_diff') == 40
+
+    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
+
+    res = client.get(url_for("index", tag=tag_uuid))
+
+    # Just a subset should be here
+    assert b'test-tag' in res.data
+    assert res.data.count(b'processor-text_json_diff') == 20
+    assert b"object at" not in res.data

changedetectionio/tests/test_ignore_text.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 def test_setup(live_server):
@@ -84,7 +84,6 @@ def set_modified_ignore_response():
 
 
 def test_check_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -103,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -124,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -137,7 +136,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -151,7 +150,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -167,7 +166,6 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -198,7 +196,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
 
     # Goto the edit page of the item, add our ignore text
@@ -220,7 +218,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # so that we are sure everything is viewed and in a known 'nothing changed' state
     res = client.get(url_for("diff_history_page", uuid="first"))
@@ -237,7 +235,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -247,7 +245,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -40,7 +40,7 @@ def set_some_changed_response():
 
 
 def test_normal_page_check_works_with_ignore_status_code(client, live_server):
-    sleep_time_for_fetch_thread = 3
+
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     set_some_changed_response()
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     #  Make a change
     set_some_changed_response()
@@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id

changedetectionio/tests/test_jinja2.py

@@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
                               "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": full_url, "tag": "test"},
+        data={"url": full_url, "tags": "test"},
         follow_redirects=True
     )
     assert b"Watch added" in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -208,7 +208,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -238,7 +238,7 @@ def check_json_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -246,7 +246,7 @@ def check_json_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -261,14 +261,14 @@ def check_json_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(4)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     res = client.get(url_for("index"))
@@ -306,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -322,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("diff_history_page", uuid="first"))
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
@@ -366,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -374,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -389,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_ext_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(4)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed'
     res = client.get(url_for("index"))
@@ -428,14 +428,14 @@ def test_ignore_json_order(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -446,7 +446,7 @@ def test_ignore_json_order(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_notification.py

@@ -3,7 +3,7 @@ import os
 import time
 import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -21,10 +21,11 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
+    #live_server_setup(live_server)
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(3)
+    time.sleep(1)
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
@@ -62,13 +63,13 @@ def test_check_notification(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # We write the PNG to disk, but a JPEG should appear in the notification
     # Write the last screenshot png
@@ -105,7 +106,7 @@ def test_check_notification(client, live_server):
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag, my second tag",
         "title": "my title",
         "headers": "",
         "fetch_backend": "html_requests"})
@@ -128,7 +129,7 @@ def test_check_notification(client, live_server):
 
 
     ## Now recheck, and it should have sent the notification
-    time.sleep(3)
+    wait_for_all_checks(client)
     set_modified_response()
 
     # Trigger a check
@@ -150,7 +151,7 @@ def test_check_notification(client, live_server):
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
     assert "Watch title: my title" in notification_submission
-    assert "Watch tag: my tag" in notification_submission
+    assert "Watch tag: my tag, my second tag" in notification_submission
     assert "diff/" in notification_submission
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
@@ -193,11 +194,11 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     res = client.get(url_for("notification_logs"))
@@ -209,7 +210,7 @@ def test_check_notification(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag",
         "title": "my title",
         "notification_urls": '',
         "notification_title": '',
@@ -243,7 +244,7 @@ def test_notification_validation(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'nice one'},
+        data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
 
@@ -303,13 +304,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'nice one'},
+        data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_notification_errors.py

@@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
               "notification_body": "xxxxx",
               "notification_format": "Text",
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",

changedetectionio/tests/test_request.py

@@ -25,7 +25,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("import_page"),
@@ -43,7 +43,7 @@ def test_headers_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
               "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
@@ -95,14 +95,14 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # add the first 'version'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": "something something"},
@@ -110,7 +110,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Now the change which should trigger a change
     body_value = 'Test Body Value'
@@ -118,7 +118,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": body_value},
@@ -126,7 +126,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # The service should echo back the body
     res = client.get(
@@ -163,7 +163,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": "invalid"},
@@ -187,7 +187,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -195,14 +195,14 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "fetch_backend": "html_requests",
             "method": "invalid"},
         follow_redirects=True
@@ -214,7 +214,7 @@ def test_method_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "fetch_backend": "html_requests",
             "method": "PATCH"},
         follow_redirects=True
@@ -222,7 +222,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # The service should echo back the request verb
     res = client.get(
@@ -233,7 +233,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:
@@ -265,7 +265,7 @@ def test_headers_textfile_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
 
 
     # Add some headers to a request
@@ -273,7 +273,7 @@ def test_headers_textfile_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "testtag",
+              "tags": "testtag",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
               "headers": "xxx:ooo\ncool:yeah\r\n"},
         follow_redirects=True

changedetectionio/tests/test_search.py

@@ -28,7 +28,7 @@ def test_basic_search(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0], "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -62,7 +62,7 @@ def test_search_in_tag_limit(client, live_server):
     # By Title
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tag": urls[0].split(' ')[1], "headers": "",
+        data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True
     )

changedetectionio/tests/test_security.py

@@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": 'javascript:alert(document.domain)',
-              "tag": "",
+              "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": ""},
@@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '            javascript:alert(123)', "tag": ''},
+        data={"url": '            javascript:alert(123)', "tags": ''},
         follow_redirects=True
     )
 
@@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
+        data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
         follow_redirects=True
     )
 
@@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
+        data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
         follow_redirects=True
     )
 
@@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tag": ''},
+        data={"url": 'file:///tasty/disk/drive', "tags": ''},
         follow_redirects=True
     )
     time.sleep(1)

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_source.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 
 sleep_time_for_fetch_thread = 3
 
@@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
 
-    time.sleep(5)
+    wait_for_all_checks(client)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 # `subtractive_selectors` should still work in `source:` type requests
 def test_check_ignore_elements(client, live_server):
     set_original_response()
-    time.sleep(2)
+    time.sleep(1)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
     res = client.post(
@@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     #####################
     # We want <span> and <p> ONLY, but ignore span with .foobar-detection
 
     client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_watch_fields_storage.py

@@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
                "title" : "My title",
                "ignore_text" : "ignore this",
                "url": test_url,
-               "tag": "woohoo",
+               "tags": "woohoo",
                "headers": "curl:foo",
                'fetch_backend': "html_requests"
                },

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/util.py

@@ -70,6 +70,16 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
+
+# kinda funky, but works for now
+def get_UUID_for_tag_name(client, name):
+    app_config = client.application.config.get('DATASTORE').data
+    for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
+        if name == tag.get('title', '').lower().strip():
+            return uuid
+    return None
+
+
 # kinda funky, but works for now
 def extract_rss_token_from_UI(client):
     import re

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
@@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_webdriver",
               'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'

changedetectionio/update_worker.py

@@ -26,43 +26,13 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def send_content_changed_notification(self, t, watch_uuid):
+    def queue_notification_for_watch(self, n_object, watch):
 
         from changedetectionio import diff
 
-        from changedetectionio.notification import (
-            default_notification_format_for_watch
-        )
-
-        n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
-        if not watch:
-            return
-
         watch_history = watch.history
         dates = list(watch_history.keys())
-        # Theoretically it's possible that this could be just 1 long,
-        # - In the case that the timestamp key was not unique
-        if len(dates) == 1:
-            raise ValueError(
-                "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
-            )
 
-        n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
-            self.datastore.data['settings']['application']['notification_urls']
-
-        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
-            self.datastore.data['settings']['application']['notification_title']
-
-        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
-            self.datastore.data['settings']['application']['notification_body']
-
-        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
-            self.datastore.data['settings']['application']['notification_format']
-
-
-        # Only prepare to notify if the rules above matched
-        if 'notification_urls' in n_object and n_object['notification_urls']:
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object['notification_format'] == 'HTML':
             line_feed_sep = "<br>"
@@ -89,18 +59,91 @@ class update_worker(threading.Thread):
             'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
             'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
-                'uuid': watch_uuid,
+            'uuid': watch.get('uuid'),
-                'watch_url': watch['url'],
+            'watch_url': watch.get('url'),
         })
         logging.info (">> SENDING NOTIFICATION")
         self.notification_q.put(n_object)
-        else:
+
-            logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
+
+    def send_content_changed_notification(self, watch_uuid):
+
+        from changedetectionio.notification import (
+            default_notification_format_for_watch
+        )
+
+        n_object = {}
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        if not watch:
+            return
+
+        watch_history = watch.history
+        dates = list(watch_history.keys())
+        # Theoretically it's possible that this could be just 1 long,
+        # - In the case that the timestamp key was not unique
+        if len(dates) == 1:
+            raise ValueError(
+                "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
+            )
+
+        # Should be a better parent getter in the model object
+        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
+        n_object['notification_urls'] = watch.get('notification_urls')
+
+        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
+            self.datastore.data['settings']['application']['notification_title']
+
+        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
+            self.datastore.data['settings']['application']['notification_body']
+
+        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
+            self.datastore.data['settings']['application']['notification_format']
+
+        # (Individual watch) Only prepare to notify if the rules above matched
+        sent = False
+        if 'notification_urls' in n_object and n_object['notification_urls']:
+            sent = True
+            self.queue_notification_for_watch(n_object, watch)
+
+        # (Group tags) try by group tag
+        if not sent:
+            # Else, Try by tag, and use system default vars for format, body etc as fallback
+            tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
+            for tag_uuid, tag in tags.items():
+                n_object = {}
+                n_object['notification_urls'] = tag.get('notification_urls')
+
+                n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
+                    self.datastore.data['settings']['application']['notification_title']
+
+                n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
+                    self.datastore.data['settings']['application']['notification_body']
+
+                n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
+                    self.datastore.data['settings']['application']['notification_format']
+
+                if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
+                    sent = True
+                    self.queue_notification_for_watch(n_object, watch)
+
+        # (Group tags) try by global
+        if not sent:
+            # leave this as is, but repeat in a loop for each tag also
+            n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
+            n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
+            n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
+            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
+            if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
+                sent = True
+                self.queue_notification_for_watch(n_object, watch)
+
+        return sent
+
 
     def send_filter_failure_notification(self, watch_uuid):
 
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
 
@@ -177,7 +220,7 @@ class update_worker(threading.Thread):
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
 
-                if uuid in list(self.datastore.data['watching'].keys()):
+                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
@@ -360,7 +403,7 @@ class update_worker(threading.Thread):
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
-                                        self.send_content_changed_notification(self, watch_uuid=uuid)
+                                        self.send_content_changed_notification(watch_uuid=uuid)
 
 
                         except Exception as e: