Merge branch 'master' into fetch-backend-ui-default

Added hook update to be sure the default is really set

.github/workflows/containers.yml

@@ -98,6 +98,7 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
       - name: Build and push :tag
@@ -116,6 +117,7 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/test-container-build.yml

@@ -10,11 +10,13 @@ on:
     paths:
       - requirements.txt
       - Dockerfile
+      - .github/workflows/*
 
   pull_request:
     paths:
       - requirements.txt
       - Dockerfile
+      - .github/workflows/*
 
   # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
   # @todo: some kind of path filter for requirements.txt and Dockerfile

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.1.0'
+__version__ = '0.40.1.1'
 
 datastore = None
 
@@ -417,6 +417,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  has_unviewed=datastore.has_unviewed,
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
                                  tags=existing_tags,
                                  watches=sorted_watches
                                  )
@@ -638,8 +639,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
 
-            # Only works reliably with Playwright
-            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver'
 
             # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
             jq_support = True
@@ -655,6 +654,9 @@ def changedetection_app(config=None, datastore_o=None):
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
                 is_html_webdriver = True
 
+            # Only works reliably with Playwright
+            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
+
             output = render_template("edit.html",
                                      browser_steps_config=browser_step_ui_config,
                                      current_base_url=datastore.data['settings']['application']['base_url'],

changedetectionio/changedetection.py

@@ -31,11 +31,13 @@ def sigterm_handler(_signo, _stack_frame):
 def main():
     global datastore
     global app
-    ssl_mode = False
-    host = ''
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
+
     datastore_path = None
+    do_cleanup = False
+    host = ''
+    ipv6_enabled = False
+    port = os.environ.get('PORT') or 5000
+    ssl_mode = False
 
     # On Windows, create and use a default path.
     if os.name == 'nt':
@@ -46,7 +48,7 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
         sys.exit(2)
@@ -66,6 +68,10 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
+        if opt == '-6':
+            print ("Enabling IPv6 listen support")
+            ipv6_enabled = True
+
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -133,13 +139,15 @@ def main():
         from werkzeug.middleware.proxy_fix import ProxyFix
         app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
+    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), socket.AF_INET6),
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
                                                certfile='cert.pem',
                                                keyfile='privkey.pem',
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), socket.AF_INET6), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
 

changedetectionio/model/Watch.py

@@ -18,7 +18,7 @@ base_config = {
     'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
     'extract_text': [],  # Extract text by regex after filters
     'extract_title_as_title': False,
-    'fetch_backend': None,
+    'fetch_backend': 'system',
     'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
     'has_ldjson_price_data': None,
     'track_ldjson_price_data': None,

changedetectionio/store.py

@@ -673,3 +673,13 @@ class ChangeDetectionStore:
                 self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
 
         return
+
+    # Some setups may have missed the correct default, so it shows the wrong config in the UI, although it will default to system-wide
+    def update_10(self):
+        for uuid, watch in self.data['watching'].items():
+            try:
+                if not watch.get('fetch_backend', ''):
+                    watch['fetch_backend'] = 'system'
+            except:
+                continue
+        return

changedetectionio/templates/watch-overview.html

@@ -91,7 +91,12 @@
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
                     <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img class="status-icon"  src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" /></a>
 
-                    {%if watch.get_fetch_backend == "html_webdriver" %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
+                    {% if watch.get_fetch_backend == "html_webdriver"
+                         or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                    %}
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />
+                    {% endif %}
+
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" />{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}