Dont clear when the same, only clear the error on edit

* improving unique key fix

* also bump timestamp along 1 sec

.github/workflows/test-only.yml

@@ -4,17 +4,10 @@ name: ChangeDetection.io App Test
 on: [push, pull_request]
 
 jobs:
-  test-application:
+  lint-code:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      # Mainly just for link/flake8
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
       - name: Lint with flake8
         run: |
           pip3 install flake8
@@ -23,202 +16,24 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-      - name: Spin up ancillary testable services
-        run: |
-          
-          docker network create changedet-network
-          
-          # Selenium
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Build changedetection.io container for testing
-        run: |         
-          # Build a changedetection.io container and start testing inside
-          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
-
-      - name: Show docker container state and other debug info
-        run: |
-          set -x
-          echo "Running processes in docker..."
-          docker ps
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
-          # Unit tests
-          echo "run test with unittest"
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
-          # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
-
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+  test-application-3-10:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.10'
 
 
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+  test-application-3-11:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.11'
+      skip-pypuppeteer: true
 
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+  test-application-3-12:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.12'
+      skip-pypuppeteer: true
 
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# SELENIUM
-      - name: Specific tests in built container for Selenium
-        run: |
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
-      - name: Specific tests in built container for headers and requests checks with Selenium
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-# OTHER STUFF
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test proxy SOCKS5 style interaction
-        run: |
-          cd changedetectionio
-          ./run_socks_proxy_tests.sh
-          cd ..
-
-      - name: Test custom browser URL
-        run: |
-          cd changedetectionio
-          ./run_custom_browser_url_tests.sh
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
-          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
-          
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
-
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
-
-      - name: Store container log
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output
-          path: output-logs

.github/workflows/test-stack-reusable-workflow.yml

@@ -0,0 +1,239 @@
+name: ChangeDetection.io App Test
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+        default: '3.10'
+      skip-pypuppeteer:
+        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  test-application:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # Mainly just for link/flake8
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
+        run: |
+          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          # Build a changedetection.io container and start testing inside
+          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'         
+
+      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+        run: |         
+          docker run test-changedetectionio  bash -c 'python3 --version'
+
+      - name: Spin up ancillary testable services
+        run: |
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+          docker ps
+
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
+          # Unit tests
+          echo "run test with unittest"
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Playwright via Sockpuppetbrowser fetch 
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+      - name: Specific tests in built container for Selenium
+        run: |
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          # "mailserver" hostname defined above
+          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE is came from STDERR
+          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
+          
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
+
+      - name: Store container log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: output-logs

Dockerfile

@@ -2,7 +2,10 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.10-slim-bookworm as builder
+
+ARG PYTHON_VERSION=3.10
+
+FROM python:${PYTHON_VERSION}-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -32,7 +35,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bookworm
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.22'
+__version__ = '0.45.23'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/content_fetchers/base.py

@@ -28,7 +28,7 @@ def manage_user_agent(headers, current_ua=''):
     :return:
     """
     # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = next((v for k, v in headers.items() if k.lower() == "user-agent"), None)
+    ua_in_custom_headers = headers.get('User-Agent')
     if ua_in_custom_headers:
         return ua_in_custom_headers
 

changedetectionio/content_fetchers/puppeteer.py

@@ -9,7 +9,6 @@ from loguru import logger
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
 
-
 class fetcher(Fetcher):
     fetcher_description = "Puppeteer/direct {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -93,15 +92,38 @@ class fetcher(Fetcher):
                                                        ignoreHTTPSErrors=True
                                                        )
         except websockets.exceptions.InvalidStatusCode as e:
-            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)")
+            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
         except websockets.exceptions.InvalidURI:
             raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
         except Exception as e:
             raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
-        else:
-            self.page = await browser.newPage()
 
-        await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
+        # Better is to launch chrome with the URL as arg
+        # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
+        # headless - ask a new page
+        self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
+
+        try:
+            from pyppeteerstealth import inject_evasions_into_page
+        except ImportError:
+            logger.debug("pyppeteerstealth module not available, skipping")
+            pass
+        else:
+            # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page)
+            # But I could never get it to fire reliably, so we just inject it straight after
+            await inject_evasions_into_page(self.page)
+
+        # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
+        user_agent = None
+        if request_headers and request_headers.get('User-Agent'):
+            # Request_headers should now be CaaseInsensitiveDict
+            # Remove it so it's not sent again with headers after
+            user_agent = request_headers.pop('User-Agent').strip()
+            await self.page.setUserAgent(user_agent)
+
+        if not user_agent:
+            # Attempt to strip 'HeadlessChrome' etc
+            await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
 
         await self.page.setBypassCSP(True)
         if request_headers:

changedetectionio/content_fetchers/requests.py

@@ -30,11 +30,6 @@ class fetcher(Fetcher):
         if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
 
-        # Make requests use a more modern looking user-agent
-        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
-            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
-                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
-
         proxies = {}
 
         # Allows override the proxy on a per-request basis

changedetectionio/flask_app.py

@@ -124,10 +124,10 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
 
 @app.template_filter('format_timestamp_timeago')
 def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    if timestamp == False:
+    if not timestamp:
         return 'Not yet'
 
-    return timeago.format(timestamp, time.time())
+    return timeago.format(int(timestamp), time.time())
 
 
 @app.template_filter('pagination_slice')
@@ -339,7 +339,7 @@ def changedetection_app(config=None, datastore_o=None):
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
             # @todo tag notification_muted skip also (improve Watch model)
-            if watch.get('notification_muted'):
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
                 continue
             if limit_tag and not limit_tag in watch['tags']:
                 continue
@@ -472,7 +472,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=active_tag,
                                  active_tag_uuid=active_tag_uuid,
-                                 app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
                                  datastore=datastore,
                                  errored_count=errored_count,
                                  form=form,
@@ -679,7 +679,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         if request.method == 'POST' and form.validate():
 
-            extra_update_obj = {}
+            extra_update_obj = {
+                'consecutive_filter_failures': 0,
+                'last_error' : False
+            }
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
@@ -771,7 +774,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                                      settings_application=datastore.data['settings']['application'],
-                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
+                                     using_global_webdriver_wait=not default['webdriver_delay'],
                                      uuid=uuid,
                                      visualselector_enabled=visualselector_enabled,
                                      watch=watch

changedetectionio/forms.py

@@ -526,6 +526,10 @@ class SingleExtraBrowser(Form):
     browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
     # @todo do the validation here instead
 
+class DefaultUAInputForm(Form):
+    html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
+    if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"):
+        html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
@@ -537,6 +541,8 @@ class globalSettingsRequestForm(Form):
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
+    default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides")
+
     def validate_extra_proxies(self, extra_validators=None):
         for e in self.data['extra_proxies']:
             if e.get('proxy_name') or e.get('proxy_url'):
@@ -566,6 +572,8 @@ class globalSettingsApplicationForm(commonSettingsForm):
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
+    rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
+                                      validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/model/App.py

@@ -5,7 +5,9 @@ from changedetectionio.notification import (
     default_notification_title,
 )
 
+# Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
+DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 class model(dict):
     base_config = {
@@ -22,6 +24,10 @@ class model(dict):
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
+                    'default_ua': {
+                        'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT),
+                        'html_webdriver': None,
+                    }
                 },
                 'application': {
                     # Custom notification content
@@ -41,6 +47,8 @@ class model(dict):
                     'pager_size': 50,
                     'password': False,
                     'render_anchor_tag_content': False,
+                    'rss_access_token': None,
+                    'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text

changedetectionio/model/Watch.py

@@ -333,7 +333,9 @@ class model(dict):
         # Small hack so that we sleep just enough to allow 1 second  between history snapshots
         # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            time.sleep(timestamp - self.__newest_history_key)
+            logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+            timestamp = str(int(timestamp) + 1)
+            time.sleep(1)
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))

changedetectionio/processors/__init__.py

@@ -1,10 +1,10 @@
 from abc import abstractmethod
-import os
-import hashlib
-import re
-from copy import deepcopy
 from changedetectionio.strtobool import strtobool
+from copy import deepcopy
 from loguru import logger
+import hashlib
+import os
+import re
 
 class difference_detection_processor():
 
@@ -21,7 +21,7 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
 
     def call_browser(self):
-
+        from requests.structures import CaseInsensitiveDict
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
@@ -93,7 +93,13 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.watch.get('headers', [])
+        request_headers = CaseInsensitiveDict()
+
+        ua = self.datastore.data['settings']['requests'].get('default_ua')
+        if ua and ua.get(prefer_fetch_backend):
+            request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})
+
+        request_headers.update(self.watch.get('headers', {}))
         request_headers.update(self.datastore.get_all_base_headers())
         request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
 

changedetectionio/static/js/preview.js

@@ -40,9 +40,13 @@ if (selectElement) {
     if (selectedOption) {
         if (selectedOption.previousElementSibling) {
             document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value;
+        } else {
+            document.getElementById('btn-previous').remove()
         }
         if (selectedOption.nextElementSibling) {
             document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value;
+        } else {
+            document.getElementById('btn-next').remove()
         }
 
     }

changedetectionio/static/styles/scss/styles.scss

@@ -1021,6 +1021,11 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none;
+  button {
+    /* some space if they wrap the page */
+    margin-bottom: 3px;
+    margin-top: 3px;
+  }
 }
 
 .checkbox-uuid {

changedetectionio/static/styles/styles.css

@@ -1127,6 +1127,10 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none; }
+  #checkbox-operations button {
+    /* some space if they wrap the page */
+    margin-bottom: 3px;
+    margin-top: 3px; }
 
 .checkbox-uuid > * {
   vertical-align: middle; }

changedetectionio/store.py

@@ -124,12 +124,12 @@ class ChangeDetectionStore:
                 self.__data['app_guid'] = str(uuid_builder.uuid4())
 
         # Generate the URL access token for RSS feeds
-        if not 'rss_access_token' in self.__data['settings']['application']:
+        if not self.__data['settings']['application'].get('rss_access_token'):
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
         # Generate the API access token
-        if not 'api_access_token' in self.__data['settings']['application']:
+        if not self.__data['settings']['application'].get('api_access_token'):
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
@@ -178,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.viewed == False:
+            if watch.history_n >= 2 and watch.viewed == False:
                 return True
         return False
 
@@ -554,7 +554,6 @@ class ChangeDetectionStore:
         return os.path.isfile(filepath)
 
     def get_all_base_headers(self):
-        from .model.App import parse_headers_from_text_file
         headers = {}
         # Global app settings
         headers.update(self.data['settings'].get('headers', {}))

changedetectionio/templates/base.html

@@ -6,7 +6,9 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" >
     <meta name="description" content="Self hosted website change detection." >
     <title>Change Detection{{extra_title}}</title>
-    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" >
+    {% if app_rss_token %}
+      <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss', tag=active_tag_uuid , token=app_rss_token)}}" >
+    {% endif %}
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}?v={{ get_css_version() }}" >
     {% if extra_stylesheets %}
@@ -83,8 +85,8 @@
           <li class="pure-menu-item pure-form" id="search-menu-item">
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
+              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag_uuid %}in '{{ active_tag.title }}'{% endif %}" required="" type="text" value="">
+              <input name="tags" type="hidden" value="{% if active_tag_uuid %}{{active_tag_uuid}}{% endif %}">
               <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
                 {% include "svgs/search-icon.svg" %}
               </button>

changedetectionio/templates/edit.html

@@ -433,7 +433,8 @@ Unavailable") }}
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
                             <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed, this tool is a helper to manage filters in the  "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab.
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br>
+                                This tool is a helper to manage filters in the  "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab.
                             </span>
 
                             <div id="selector-header">

changedetectionio/templates/preview.html

@@ -66,7 +66,7 @@
         </div>
 
         <div class="tab-pane-inner" id="text">
-            <div class="snapshot-age">{{ watch.snapshot_text_ctime|format_timestamp_timeago }}</div>
+            <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div>
             <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
             <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
 

changedetectionio/templates/settings.html

@@ -62,6 +62,9 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
@@ -108,8 +111,6 @@
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
-                    <br>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
                     <div class="pure-form-message-inline">
@@ -121,6 +122,18 @@
                         {{ render_field(form.application.form.webdriver_delay) }}
                     </div>
                 </fieldset>
+                <div class="pure-control-group inline-radio">
+                    {{ render_field(form.requests.form.default_ua) }}
+                    <span class="pure-form-message-inline">
+                        Applied to all requests.<br><br>
+                        Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>.
+                    </span>
+                </div>
+                <div class="pure-control-group">
+                                        <br>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+
+                </div>
             </div>
 
             <div class="tab-pane-inner" id="filters">
@@ -190,7 +203,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}">
+                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>

changedetectionio/templates/watch-overview.html

@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
 
                     {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }}
                     {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
@@ -46,7 +46,7 @@
     {% endif %}
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
-        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
 
     <!-- tag list -->
     {% for uuid, tag in tags %}
@@ -67,18 +67,18 @@
             <tr>
                 {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
                 {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
                 <th></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th></th>
             </tr>
             </thead>
             <tbody>
             {% if not watches|length %}
             <tr>
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+                <td colspan="6" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
@@ -95,11 +95,11 @@
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -204,7 +204,7 @@
                 all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+                <a href="{{ url_for('rss', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
         {{ pagination.links }}

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,42 +1,51 @@
 #!/usr/bin/python3
-import smtpd
-import asyncore
+import asyncio
+from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
 
 # Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
 
 last_received_message = b"Nothing"
 
 
-class CustomSMTPServer(smtpd.SMTPServer):
-
-    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+class CustomSMTPHandler:
+    async def handle_DATA(self, server, session, envelope):
         global last_received_message
-        last_received_message = data
-        print('Receiving message from:', peer)
-        print('Message addressed from:', mailfrom)
-        print('Message addressed to  :', rcpttos)
-        print('Message length        :', len(data))
-        print(data.decode('utf8'))
-        return
+        last_received_message = envelope.content
+        print('Receiving message from:', session.peer)
+        print('Message addressed from:', envelope.mail_from)
+        print('Message addressed to  :', envelope.rcpt_tos)
+        print('Message length        :', len(envelope.content))
+        print(envelope.content.decode('utf8'))
+        return '250 Message accepted for delivery'
 
 
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
-    def __init__(self, host, port):
-        asyncore.dispatcher.__init__(self)
-        self.create_socket()
-        self.set_reuse_addr()
-        self.bind((host, port))
-        self.listen(5)
-
-    def handle_accepted(self, sock, addr):
+class EchoServerProtocol(asyncio.Protocol):
+    def connection_made(self, transport):
         global last_received_message
-        print('Incoming connection from %s' % repr(addr))
-        sock.send(last_received_message)
+        self.transport = transport
+        peername = transport.get_extra_info('peername')
+        print('Incoming connection from {}'.format(peername))
+        self.transport.write(last_received_message)
+
         last_received_message = b''
+        self.transport.close()
 
 
-server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
-asyncore.loop()
+async def main():
+    # Start the SMTP server
+    controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
+    controller.start()
+
+    # Start the TCP Echo server
+    loop = asyncio.get_running_loop()
+    server = await loop.create_server(
+        lambda: EchoServerProtocol(),
+        '0.0.0.0', 11080
+    )
+    async with server:
+        await server.serve_forever()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -32,6 +32,8 @@ def get_last_message_from_smtp_server():
     client_socket.connect((smtp_test_server, port))  # connect to the server
 
     data = client_socket.recv(50024).decode()  # receive response
+    logging.info("get_last_message_from_smtp_server..")
+    logging.info(data)
     client_socket.close()  # close the connection
     return data
 
@@ -71,6 +73,8 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
+
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -81,7 +85,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
     assert 'Content-Type: text/html' in msg
     assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -135,6 +139,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -147,7 +152,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # The email should not have two bodies, should be TEXT only
 
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
 
     set_original_response()
     # Now override as HTML format
@@ -168,7 +173,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
     assert 'Content-Type: text/html' in msg
     assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 

changedetectionio/tests/test_auth.py

@@ -2,13 +2,12 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
+
 
 def test_basic_auth(client, live_server):
 
     live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -19,8 +18,8 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
     time.sleep(1)
-
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -29,7 +28,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -100,11 +100,12 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # Accept it
     uuid = extract_UUID_from_client(client)
-
+    time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
     # Trigger a check
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone

changedetectionio/tests/test_backend.py

@@ -62,9 +62,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Make a change
     set_modified_response()
 
-    res = urlopen(url_for('test_endpoint', _external=True))
-    assert b'which has this one new line' in res.read()
-
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
@@ -135,6 +132,9 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
+    # Be sure the last_viewed is going to be greater than the last snapshot
+    time.sleep(1)
+
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 

changedetectionio/tests/test_encoding.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 import pytest
 
 
@@ -27,9 +27,6 @@ def set_html_response():
 def test_check_encoding_detection(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -39,7 +36,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Give the thread time to pick it up
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -56,9 +53,6 @@ def test_check_encoding_detection(client, live_server):
 def test_check_encoding_detection_missing_content_type_header(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(
@@ -67,8 +61,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
-    # Give the thread time to pick it up
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_extract_csv.py

@@ -29,6 +29,7 @@ def test_check_extract_text_from_diff(client, live_server):
     # Load in 5 different numbers/changes
     last_date=""
     for n in range(5):
+        time.sleep(1)
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())

changedetectionio/tests/test_filter_failure_notification.py

@@ -21,10 +21,11 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, content_filter):
+def run_filter_test(client, live_server, content_filter):
+
+    # Response WITHOUT the filter ID element
+    set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -79,6 +80,7 @@ def run_filter_test(client, content_filter):
         "include_filters": content_filter,
         "fetch_backend": "html_requests"})
 
+    # A POST here will also reset the filter failure counter (filter_failure_notification_threshold_attempts)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data=notification_form_data,
@@ -91,20 +93,21 @@ def run_filter_test(client, content_filter):
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # -2 because we would have checked twice above (on adding and on edit)
+    # recheck it up to just before the threshold, including the fact that in the previous POST it would have rechecked (and incremented)
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
-        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
+        time.sleep(2) # delay for apprise to fire
+        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i} when threshold is {App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT}"
 
     # We should see something in the frontend
+    res = client.get(url_for("index"))
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger it (see -2 above)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
+    time.sleep(2)  # delay for apprise to fire
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -149,13 +152,9 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
-    set_original_response()
-    wait_for_all_checks(client)
-    run_filter_test(client, '#nope-doesnt-exist')
+    run_filter_test(client, live_server,'#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
-    set_original_response()
-    time.sleep(1)
-    run_filter_test(client, '//*[@id="nope-doesnt-exist"]')
+    run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent

changedetectionio/tests/test_notification_errors.py

@@ -9,9 +9,6 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/test_request.py

@@ -253,15 +253,99 @@ def test_method_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
+def test_ua_global_override(client, live_server):
+    # live_server_setup(live_server)
+    test_url = url_for('test_headers', _external=True)
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            "requests-default_ua-html_requests": "html-requests-user-agent"
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"html-requests-user-agent" in res.data
+    # default user-agent should have shown by now
+    # now add a custom one in the headers
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tags": "testtag",
+            "fetch_backend": 'html_requests',
+            # Important - also test case-insensitive
+            "headers": "User-AGent: agent-from-watch"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"agent-from-watch" in res.data
+    assert b"html-requests-user-agent" not in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
 def test_headers_textfile_in_request(client, live_server):
     #live_server_setup(live_server)
     # Add our URL to the import page
+
+    webdriver_ua = "Hello fancy webdriver UA 1.0"
+    requests_ua = "Hello basic requests UA 1.1"
+
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
         # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
-    print ("TEST URL IS ",test_url)
+    form_data = {
+        "application-fetch_backend": "html_requests",
+        "application-minutes_between_check": 180,
+        "requests-default_ua-html_requests": requests_ua
+    }
+
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        form_data["requests-default_ua-html_webdriver"] = webdriver_ua
+
+    res = client.post(
+        url_for("settings_page"),
+        data=form_data,
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    res = client.get(url_for("settings_page"))
+
+    # Only when some kind of real browser is setup
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        assert b'requests-default_ua-html_webdriver' in res.data
+
+    # Field should always be there
+    assert b"requests-default_ua-html_requests" in res.data
+
     # Add the test URL twice, we will check
     res = client.post(
         url_for("import_page"),
@@ -272,15 +356,14 @@ def test_headers_textfile_in_request(client, live_server):
 
     wait_for_all_checks(client)
 
-
     # Add some headers to a request
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
-              "url": test_url,
-              "tags": "testtag",
-              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "xxx:ooo\ncool:yeah\r\n"},
+            "url": test_url,
+            "tags": "testtag",
+            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "headers": "xxx:ooo\ncool:yeah\r\n"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -292,7 +375,7 @@ def test_headers_textfile_in_request(client, live_server):
     with open('test-datastore/headers.txt', 'w') as f:
         f.write("global-header: nice\r\nnext-global-header: nice")
 
-    with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f:
+    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
         f.write("watch-header: nice")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -306,7 +389,7 @@ def test_headers_textfile_in_request(client, live_server):
     # Not needed anymore
     os.unlink('test-datastore/headers.txt')
     os.unlink('test-datastore/headers-testtag.txt')
-    os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt')
+
     # The service should echo back the request verb
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -319,7 +402,12 @@ def test_headers_textfile_in_request(client, live_server):
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
 
+    # Check the custom UA from system settings page made it through
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data
+    else:
+        assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data
 
-    #unlink headers.txt on start/stop
+    # unlink headers.txt on start/stop
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+    assert b'Deleted' in res.data

changedetectionio/tests/test_rss.py

@@ -69,6 +69,7 @@ def test_rss_and_token(client, live_server):
 
     wait_for_all_checks(client)
     set_modified_response()
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -87,7 +88,7 @@ def test_rss_and_token(client, live_server):
     assert b"Access denied, bad token" not in res.data
     assert b"Random content" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 def test_basic_cdata_rss_markup(client, live_server):
     #live_server_setup(live_server)

changedetectionio/update_worker.py

@@ -348,7 +348,7 @@ class update_worker(threading.Thread):
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
+                            logger.warning(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
                                 if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_filter_failure_notification(uuid)
@@ -362,7 +362,6 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
                     except content_fetchers.exceptions.BrowserConnectError as e:
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet==0.33.3 # related to dnspython fixes
+eventlet==0.35.2 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -29,9 +29,7 @@ chardet>2.3.0
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
-# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
-dnspython==2.3.0 # related to eventlet fixes
+dnspython==2.6.1 # related to eventlet fixes
 
 # jq not available on Windows so must be installed manually
 
@@ -73,12 +71,10 @@ openpyxl
 jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
 jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
-# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
-pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
-# experimental release
 pyppeteer-ng==2.0.0rc5
+pyppeteerstealth>=0.0.4
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
@@ -88,3 +84,5 @@ pytest-flask ~=1.2
 jsonschema==4.17.3
 
 loguru
+# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
+greenlet >= 3.0.3