cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Filters can now accept a list/multiple filters (#1064) #623

Browse Source
This commit is contained in:
dgtlmoon
2022-11-03 12:13:54 +01:00
committed by GitHub
parent d0efeb9770
commit 359fc48fb4
23 changed files with 237 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
changedetectionio/tests/proxy_list/test_multiple_proxy.py
View File

@@ -24,7 +24,7 @@ def test_preferred_proxy(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={
"css_filter": "",
"include_filters": "",
"fetch_backend": "html_requests",
"headers": "",
"proxy": "proxy-two",

2
changedetectionio/tests/test_auth.py
View File

@@ -23,7 +23,7 @@ def test_basic_auth(client, live_server):
# Check form validation
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data

74
changedetectionio/tests/test_css_selector.py
View File

@@ -46,22 +46,23 @@ def set_modified_response():
# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
def test_css_filter_output():
from changedetectionio import fetch_site_status
def test_include_filters_output():
from inscriptis import get_text
# Check text with sub-parts renders correctly
content = """<html> <body><div id="thingthing" > Some really <b>bold</b> text </div> </body> </html>"""
html_blob = css_filter(css_filter="#thingthing", html_content=content)
html_blob = include_filters(include_filters="#thingthing", html_content=content)
text = get_text(html_blob)
assert text == " Some really bold text"
content = """<html> <body>
<p>foo bar blah</p>
<div class="parts">Block A</div> <div class="parts">Block B</div></body>
<DIV class="parts">Block A</DiV> <div class="parts">Block B</DIV></body>
</html>
"""
html_blob = css_filter(css_filter=".parts", html_content=content)
# in xPath this would be //*[@class='parts']
html_blob = include_filters(include_filters=".parts", html_content=content)
text = get_text(html_blob)
# Divs are converted to 4 whitespaces by inscriptis
@@ -69,10 +70,10 @@ def test_css_filter_output():
# Tests the whole stack works with the CSS Filter
def test_check_markup_css_filter_restriction(client, live_server):
def test_check_markup_include_filters_restriction(client, live_server):
sleep_time_for_fetch_thread = 3
css_filter = "#sametext"
include_filters = "#sametext"
set_original_response()
@@ -98,7 +99,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -107,7 +108,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
res = client.get(
url_for("edit_page", uuid="first"),
)
assert bytes(css_filter.encode('utf-8')) in res.data
assert bytes(include_filters.encode('utf-8')) in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -126,3 +127,58 @@ def test_check_markup_css_filter_restriction(client, live_server):
# Because it should be looking at only that 'sametext' id
res = client.get(url_for("index"))
assert b'unviewed' in res.data
# Tests the whole stack works with the CSS Filter
def test_check_multiple_filters(client, live_server):
sleep_time_for_fetch_thread = 3
include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("""<html><body>
<div id="blob-a">Blob A</div>
<div id="blob-b">Blob B</div>
<div id="blob-c">Blob C</div>
</body>
</html>
""")
# Give the endpoint time to spin up
time.sleep(1)
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(1)
# Goto the edit page, add our ignore text
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": include_filters,
"url": test_url,
"tag": "",
"headers": "",
'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
res = client.get(
url_for("preview_page", uuid="first"),
follow_redirects=True
)
# Only the two blobs should be here
assert b"Blob A" in res.data # CSS was ok
assert b"Blob B" in res.data # xPath was ok
assert b"Blob C" not in res.data # Should not be included

6
changedetectionio/tests/test_extract_regex.py
View File

@@ -88,7 +88,7 @@ def test_check_filter_multiline(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": '',
data={"include_filters": '',
'extract_text': '/something.+?6 billion.+?lines/si',
"url": test_url,
"tag": "",
@@ -116,7 +116,7 @@ def test_check_filter_multiline(client, live_server):
def test_check_filter_and_regex_extract(client, live_server):
sleep_time_for_fetch_thread = 3
css_filter = ".changetext"
include_filters = ".changetext"
set_original_response()
@@ -143,7 +143,7 @@ def test_check_filter_and_regex_extract(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": css_filter,
data={"include_filters": include_filters,
'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
"url": test_url,
"tag": "",

2
changedetectionio/tests/test_filter_exist_changes.py
View File

@@ -92,7 +92,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
"tag": "my tag",
"title": "my title",
"headers": "",
"css_filter": '.ticket-available',
"include_filters": '.ticket-available',
"fetch_backend": "html_requests"})
res = client.post(

6
changedetectionio/tests/test_filter_failure_notification.py
View File

@@ -76,7 +76,7 @@ def run_filter_test(client, content_filter):
"title": "my title",
"headers": "",
"filter_failure_notification_send": 'y',
"css_filter": content_filter,
"include_filters": content_filter,
"fetch_backend": "html_requests"})
res = client.post(
@@ -95,7 +95,7 @@ def run_filter_test(client, content_filter):
time.sleep(3)
# We should see something in the frontend
assert b'Warning, filter' in res.data
assert b'Warning, no filters were found' in res.data
# Now it should exist and contain our "filter not found" alert
assert os.path.isfile("test-datastore/notification.txt")
@@ -131,7 +131,7 @@ def run_filter_test(client, content_filter):
def test_setup(live_server):
live_server_setup(live_server)
def test_check_css_filter_failure_notification(client, live_server):
def test_check_include_filters_failure_notification(client, live_server):
set_original_response()
time.sleep(1)
run_filter_test(client, '#nope-doesnt-exist')

31
changedetectionio/tests/test_jsonpath_jq_selector.py
View File

@@ -132,7 +132,7 @@ def set_original_response():
return None
def set_response_with_html():
def set_json_response_with_html():
test_return_data = """
{
"test": [
@@ -176,7 +176,7 @@ def set_modified_response():
def test_check_json_without_filter(client, live_server):
# Request a JSON document from a application/json source containing HTML
# and be sure it doesn't get chewed up by instriptis
set_response_with_html()
set_json_response_with_html()
# Give the endpoint time to spin up
time.sleep(1)
@@ -189,9 +189,6 @@ def test_check_json_without_filter(client, live_server):
follow_redirects=True
)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
@@ -200,6 +197,7 @@ def test_check_json_without_filter(client, live_server):
follow_redirects=True
)
# Should still see '"html": "<b>"'
assert b'&#34;&lt;b&gt;' in res.data
assert res.data.count(b'{\n') >= 2
@@ -221,9 +219,6 @@ def check_json_filter(json_filter, client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
@@ -231,7 +226,7 @@ def check_json_filter(json_filter, client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": json_filter,
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"headers": "",
@@ -247,9 +242,6 @@ def check_json_filter(json_filter, client, live_server):
)
assert bytes(escape(json_filter).encode('utf-8')) in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
# Make a change
@@ -301,7 +293,7 @@ def check_json_filter_bool_val(json_filter, client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": json_filter,
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"headers": "",
@@ -311,11 +303,6 @@ def check_json_filter_bool_val(json_filter, client, live_server):
)
assert b"Updated watch." in res.data
time.sleep(3)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
# Make a change
@@ -360,9 +347,6 @@ def check_json_ext_filter(json_filter, client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
@@ -370,7 +354,7 @@ def check_json_ext_filter(json_filter, client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": json_filter,
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"headers": "",
@@ -386,9 +370,6 @@ def check_json_ext_filter(json_filter, client, live_server):
)
assert bytes(escape(json_filter).encode('utf-8')) in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
# Make a change

12
changedetectionio/tests/test_share_watch.py
View File

@@ -14,7 +14,7 @@ def test_share_watch(client, live_server):
live_server_setup(live_server)
test_url = url_for('test_endpoint', _external=True)
css_filter = ".nice-filter"
include_filters = ".nice-filter"
# Add our URL to the import page
res = client.post(
@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -37,7 +37,7 @@ def test_share_watch(client, live_server):
res = client.get(
url_for("edit_page", uuid="first"),
)
assert bytes(css_filter.encode('utf-8')) in res.data
assert bytes(include_filters.encode('utf-8')) in res.data
# click share the link
res = client.get(
@@ -73,4 +73,8 @@ def test_share_watch(client, live_server):
res = client.get(
url_for("edit_page", uuid="first"),
)
assert bytes(css_filter.encode('utf-8')) in res.data
assert bytes(include_filters.encode('utf-8')) in res.data
# Check it saved the URL
res = client.get(url_for("index"))
assert bytes(test_url.encode('utf-8')) in res.data

8
changedetectionio/tests/test_source.py
View File

@@ -57,10 +57,9 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
# `subtractive_selectors` should still work in `source:` type requests
def test_check_ignore_elements(client, live_server):
set_original_response()
time.sleep(2)
test_url = 'source:'+url_for('test_endpoint', _external=True)
# Add our URL to the import page
@@ -77,9 +76,9 @@ def test_check_ignore_elements(client, live_server):
#####################
# We want <span> and <p> ONLY, but ignore span with .foobar-detection
res = client.post(
client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
follow_redirects=True
)
@@ -89,7 +88,6 @@ def test_check_ignore_elements(client, live_server):
url_for("preview_page", uuid="first"),
follow_redirects=True
)
assert b'foobar-detection' not in res.data
assert b'&lt;br' not in res.data
assert b'&lt;p' in res.data

2
changedetectionio/tests/test_trigger_regex_with_filter.py
View File

@@ -49,7 +49,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server):
url_for("edit_page", uuid="first"),
data={"trigger_text": "/cool.stuff/",
"url": test_url,
"css_filter": '#in-here',
"include_filters": '#in-here',
"fetch_backend": "html_requests"},
follow_redirects=True
)

2
changedetectionio/tests/test_watch_fields_storage.py
View File

@@ -22,7 +22,7 @@ def test_check_watch_field_storage(client, live_server):
url_for("edit_page", uuid="first"),
data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n",
"time_between_check-minutes": 126,
"css_filter" : ".fooclass",
"include_filters" : ".fooclass",
"title" : "My title",
"ignore_text" : "ignore this",
"url": test_url,

16
changedetectionio/tests/test_xpath_selector.py
View File

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
time.sleep(1)
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
time.sleep(1)
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -182,9 +182,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
@@ -192,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -230,10 +227,11 @@ def test_xpath_validation(client, live_server):
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(2)
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"is not a valid XPath expression" in res.data
@@ -242,7 +240,7 @@ def test_xpath_validation(client, live_server):
# actually only really used by the distll.io importer, but could be handy too
def test_check_with_prefix_css_filter(client, live_server):
def test_check_with_prefix_include_filters(client, live_server):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
@@ -263,7 +261,7 @@ def test_check_with_prefix_css_filter(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)