cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move history data to a textfile, improves memory handling (#638)

Browse Source
This commit is contained in:
dgtlmoon
2022-05-31 23:43:50 +02:00
committed by GitHub
parent dbb5468cdc
commit aa3c8a9370
14 changed files with 330 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
changedetectionio/__init__.py
View File

@@ -178,6 +178,10 @@ def changedetection_app(config=None, datastore_o=None):
global datastore global datastore
datastore = datastore_o datastore = datastore_o
# so far just for read-only via tests, but this will be moved eventually to be the main source
# (instead of the global var)
app.config['DATASTORE']=datastore_o
#app.config.update(config or {}) #app.config.update(config or {})
login_manager = flask_login.LoginManager(app) login_manager = flask_login.LoginManager(app)
@@ -317,25 +321,19 @@ def changedetection_app(config=None, datastore_o=None):
for watch in sorted_watches: for watch in sorted_watches:
dates = list(watch['history'].keys()) dates = list(watch.history.keys())
# Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected. # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
if len(dates) < 2: if len(dates) < 2:
continue continue
# Convert to int, sort and back to str again prev_fname = watch.history[dates[-2]]
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
prev_fname = watch['history'][dates[1]]
if not watch['viewed']: if not watch.viewed:
# Re #239 - GUID needs to be individual for each event # Re #239 - GUID needs to be individual for each event
# @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
guid = "{}/{}".format(watch['uuid'], watch['last_changed']) guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
fe = fg.add_entry() fe = fg.add_entry()
# Include a link to the diff page, they will have to login here to see if password protection is enabled. # Include a link to the diff page, they will have to login here to see if password protection is enabled.
# Description is the page you watch, link takes you to the diff JS UI page # Description is the page you watch, link takes you to the diff JS UI page
base_url = datastore.data['settings']['application']['base_url'] base_url = datastore.data['settings']['application']['base_url']
@@ -350,13 +348,13 @@ def changedetection_app(config=None, datastore_o=None):
watch_title = watch.get('title') if watch.get('title') else watch.get('url') watch_title = watch.get('title') if watch.get('title') else watch.get('url')
fe.title(title=watch_title) fe.title(title=watch_title)
latest_fname = watch['history'][dates[0]] latest_fname = watch.history[dates[-1]]
html_diff = diff.render_diff(prev_fname, latest_fname, include_equal=False, line_feed_sep="</br>") html_diff = diff.render_diff(prev_fname, latest_fname, include_equal=False, line_feed_sep="</br>")
fe.description(description="<![CDATA[<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff)) fe.description(description="<![CDATA[<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff))
fe.guid(guid, permalink=False) fe.guid(guid, permalink=False)
dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key'])) dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
dt = dt.replace(tzinfo=pytz.UTC) dt = dt.replace(tzinfo=pytz.UTC)
fe.pubDate(dt) fe.pubDate(dt)
@@ -491,10 +489,10 @@ def changedetection_app(config=None, datastore_o=None):
# 0 means that theres only one, so that there should be no 'unviewed' history available # 0 means that theres only one, so that there should be no 'unviewed' history available
if newest_history_key == 0: if newest_history_key == 0:
newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0] newest_history_key = list(datastore.data['watching'][uuid].history.keys())[0]
if newest_history_key: if newest_history_key:
with open(datastore.data['watching'][uuid]['history'][newest_history_key], with open(datastore.data['watching'][uuid].history[newest_history_key],
encoding='utf-8') as file: encoding='utf-8') as file:
raw_content = file.read() raw_content = file.read()
@@ -588,12 +586,12 @@ def changedetection_app(config=None, datastore_o=None):
# Reset the previous_md5 so we process a new snapshot including stripping ignore text. # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
if form_ignore_text: if form_ignore_text:
if len(datastore.data['watching'][uuid]['history']): if len(datastore.data['watching'][uuid].history):
extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
# Reset the previous_md5 so we process a new snapshot including stripping ignore text. # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']: if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
if len(datastore.data['watching'][uuid]['history']): if len(datastore.data['watching'][uuid].history):
extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
# Be sure proxy value is None # Be sure proxy value is None
@@ -754,7 +752,7 @@ def changedetection_app(config=None, datastore_o=None):
# Save the current newest history as the most recently viewed # Save the current newest history as the most recently viewed
for watch_uuid, watch in datastore.data['watching'].items(): for watch_uuid, watch in datastore.data['watching'].items():
datastore.set_last_viewed(watch_uuid, watch['newest_history_key']) datastore.set_last_viewed(watch_uuid, watch.newest_history_key)
flash("Cleared all statuses.") flash("Cleared all statuses.")
return redirect(url_for('index')) return redirect(url_for('index'))
@@ -774,20 +772,17 @@ def changedetection_app(config=None, datastore_o=None):
flash("No history found for the specified link, bad link?", "error") flash("No history found for the specified link, bad link?", "error")
return redirect(url_for('index')) return redirect(url_for('index'))
dates = list(watch['history'].keys()) history = watch.history
# Convert to int, sort and back to str again dates = list(history.keys())
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
if len(dates) < 2: if len(dates) < 2:
flash("Not enough saved change detection snapshots to produce a report.", "error") flash("Not enough saved change detection snapshots to produce a report.", "error")
return redirect(url_for('index')) return redirect(url_for('index'))
# Save the current newest history as the most recently viewed # Save the current newest history as the most recently viewed
datastore.set_last_viewed(uuid, dates[0]) datastore.set_last_viewed(uuid, time.time())
newest_file = watch['history'][dates[0]]
newest_file = history[dates[-1]]
try: try:
with open(newest_file, 'r') as f: with open(newest_file, 'r') as f:
@@ -797,10 +792,10 @@ def changedetection_app(config=None, datastore_o=None):
previous_version = request.args.get('previous_version') previous_version = request.args.get('previous_version')
try: try:
previous_file = watch['history'][previous_version] previous_file = history[previous_version]
except KeyError: except KeyError:
# Not present, use a default value, the second one in the sorted list. # Not present, use a default value, the second one in the sorted list.
previous_file = watch['history'][dates[1]] previous_file = history[dates[-2]]
try: try:
with open(previous_file, 'r') as f: with open(previous_file, 'r') as f:
@@ -817,7 +812,7 @@ def changedetection_app(config=None, datastore_o=None):
extra_stylesheets=extra_stylesheets, extra_stylesheets=extra_stylesheets,
versions=dates[1:], versions=dates[1:],
uuid=uuid, uuid=uuid,
newest_version_timestamp=dates[0], newest_version_timestamp=dates[-1],
current_previous_version=str(previous_version), current_previous_version=str(previous_version),
current_diff_url=watch['url'], current_diff_url=watch['url'],
extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']), extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
@@ -845,9 +840,9 @@ def changedetection_app(config=None, datastore_o=None):
flash("No history found for the specified link, bad link?", "error") flash("No history found for the specified link, bad link?", "error")
return redirect(url_for('index')) return redirect(url_for('index'))
if len(watch['history']): if watch.history_n >0:
timestamps = sorted(watch['history'].keys(), key=lambda x: int(x)) timestamps = sorted(watch.history.keys(), key=lambda x: int(x))
filename = watch['history'][timestamps[-1]] filename = watch.history[timestamps[-1]]
try: try:
with open(filename, 'r') as f: with open(filename, 'r') as f:
tmp = f.readlines() tmp = f.readlines()
@@ -1141,6 +1136,7 @@ def changedetection_app(config=None, datastore_o=None):
# copy it to memory as trim off what we dont need (history) # copy it to memory as trim off what we dont need (history)
watch = deepcopy(datastore.data['watching'][uuid]) watch = deepcopy(datastore.data['watching'][uuid])
# For older versions that are not a @property
if (watch.get('history')): if (watch.get('history')):
del (watch['history']) del (watch['history'])
@@ -1249,6 +1245,7 @@ def notification_runner():
# Thread runner to check every minute, look for new watches to feed into the Queue. # Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks(): def ticker_thread_check_time_launch_checks():
from changedetectionio import update_worker from changedetectionio import update_worker
import logging
# Spin up Workers that do the fetching # Spin up Workers that do the fetching
# Can be overriden by ENV or use the default settings # Can be overriden by ENV or use the default settings
@@ -1267,9 +1264,10 @@ def ticker_thread_check_time_launch_checks():
running_uuids.append(t.current_uuid) running_uuids.append(t.current_uuid)
# Re #232 - Deepcopy the data incase it changes while we're iterating through it all # Re #232 - Deepcopy the data incase it changes while we're iterating through it all
watch_uuid_list = []
while True: while True:
try: try:
copied_datastore = deepcopy(datastore) watch_uuid_list = datastore.data['watching'].keys()
except RuntimeError as e: except RuntimeError as e:
# RuntimeError: dictionary changed size during iteration # RuntimeError: dictionary changed size during iteration
time.sleep(0.1) time.sleep(0.1)
@@ -1286,7 +1284,12 @@ def ticker_thread_check_time_launch_checks():
recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
recheck_time_system_seconds = datastore.threshold_seconds recheck_time_system_seconds = datastore.threshold_seconds
for uuid, watch in copied_datastore.data['watching'].items(): for uuid in watch_uuid_list:
watch = datastore.data['watching'].get(uuid)
if not watch:
logging.error("Watch: {} no longer present.".format(uuid))
continue
# No need todo further processing if it's paused # No need todo further processing if it's paused
if watch['paused']: if watch['paused']:

11
changedetectionio/api/api_v1.py
View File

@@ -28,8 +28,7 @@ class Watch(Resource):
return "OK", 200 return "OK", 200
# Return without history, get that via another API call # Return without history, get that via another API call
watch['history_n'] = len(watch['history']) watch['history_n'] = watch.history_n
del (watch['history'])
return watch return watch
@auth.check_token @auth.check_token
@@ -52,7 +51,7 @@ class WatchHistory(Resource):
watch = self.datastore.data['watching'].get(uuid) watch = self.datastore.data['watching'].get(uuid)
if not watch: if not watch:
abort(404, message='No watch exists with the UUID of {}'.format(uuid)) abort(404, message='No watch exists with the UUID of {}'.format(uuid))
return watch['history'], 200 return watch.history, 200
class WatchSingleHistory(Resource): class WatchSingleHistory(Resource):
@@ -69,13 +68,13 @@ class WatchSingleHistory(Resource):
if not watch: if not watch:
abort(404, message='No watch exists with the UUID of {}'.format(uuid)) abort(404, message='No watch exists with the UUID of {}'.format(uuid))
if not len(watch['history']): if not len(watch.history):
abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid)) abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
if timestamp == 'latest': if timestamp == 'latest':
timestamp = list(watch['history'].keys())[-1] timestamp = list(watch.history.keys())[-1]
with open(watch['history'][timestamp], 'r') as f: with open(watch.history[timestamp], 'r') as f:
content = f.read() content = f.read()
response = make_response(content, 200) response = make_response(content, 200)

1
changedetectionio/fetch_site_status.py
View File

@@ -224,6 +224,7 @@ class perform_site_check():
result = html_tools.strip_ignore_text(content=str(stripped_text_from_html), result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
wordlist=watch['trigger_text'], wordlist=watch['trigger_text'],
mode="line numbers") mode="line numbers")
# If it returned any lines that matched..
if result: if result:
blocked_by_not_found_trigger_text = False blocked_by_not_found_trigger_text = False

108
changedetectionio/model/Watch.py
View File

@@ -1,5 +1,4 @@
import os import os
import uuid as uuid_builder import uuid as uuid_builder
minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
@@ -12,22 +11,24 @@ from changedetectionio.notification import (
class model(dict): class model(dict):
base_config = { __newest_history_key = None
__history_n=0
__base_config = {
'url': None, 'url': None,
'tag': None, 'tag': None,
'last_checked': 0, 'last_checked': 0,
'last_changed': 0, 'last_changed': 0,
'paused': False, 'paused': False,
'last_viewed': 0, # history key value of the last viewed via the [diff] link 'last_viewed': 0, # history key value of the last viewed via the [diff] link
'newest_history_key': 0, #'newest_history_key': 0,
'title': None, 'title': None,
'previous_md5': False, 'previous_md5': False,
# UUID not needed, should be generated only as a key 'uuid': str(uuid_builder.uuid4()),
# 'uuid':
'headers': {}, # Extra headers to send 'headers': {}, # Extra headers to send
'body': None, 'body': None,
'method': 'GET', 'method': 'GET',
'history': {}, # Dict of timestamp and output stripped filename #'history': {}, # Dict of timestamp and output stripped filename
'ignore_text': [], # List of text to ignore when calculating the comparison checksum 'ignore_text': [], # List of text to ignore when calculating the comparison checksum
# Custom notification content # Custom notification content
'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise) 'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
@@ -48,10 +49,103 @@ class model(dict):
} }
def __init__(self, *arg, **kw): def __init__(self, *arg, **kw):
self.update(self.base_config) import uuid
self.update(self.__base_config)
self.__datastore_path = kw['datastore_path']
self['uuid'] = str(uuid.uuid4())
del kw['datastore_path']
if kw.get('default'):
self.update(kw['default'])
del kw['default']
# goes at the end so we update the default object with the initialiser # goes at the end so we update the default object with the initialiser
super(model, self).__init__(*arg, **kw) super(model, self).__init__(*arg, **kw)
@property
def viewed(self):
if int(self.newest_history_key) <= int(self['last_viewed']):
return True
return False
@property
def history_n(self):
return self.__history_n
@property
def history(self):
tmp_history = {}
import logging
import time
# Read the history file as a dict
fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
if os.path.isfile(fname):
logging.debug("Disk IO accessed " + str(time.time()))
with open(fname, "r") as f:
tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
if len(tmp_history):
self.__newest_history_key = list(tmp_history.keys())[-1]
self.__history_n = len(tmp_history)
return tmp_history
@property
def has_history(self):
fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
return os.path.isfile(fname)
# Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
@property
def newest_history_key(self):
if self.__newest_history_key is not None:
return self.__newest_history_key
if len(self.history) <= 1:
return 0
bump = self.history
return self.__newest_history_key
# Save some text file to the appropriate path and bump the history
# result_obj from fetch_site_status.run()
def save_history_text(self, contents, timestamp):
import uuid
from os import mkdir, path, unlink
import logging
output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
# Incase the operator deleted it, check and create.
if not os.path.isdir(output_path):
mkdir(output_path)
snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
logging.debug("Saving history text {}".format(snapshot_fname))
with open(snapshot_fname, 'wb') as f:
f.write(contents)
f.close()
# Append to index
# @todo check last char was \n
index_fname = "{}/history.txt".format(output_path)
with open(index_fname, 'a') as f:
f.write("{},{}\n".format(timestamp, snapshot_fname))
f.close()
self.__newest_history_key = timestamp
self.__history_n+=1
#@todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
return snapshot_fname
@property @property
def has_empty_checktime(self): def has_empty_checktime(self):

106
changedetectionio/store.py
View File

@@ -40,7 +40,7 @@ class ChangeDetectionStore:
# Base definition for all watchers # Base definition for all watchers
# deepcopy part of #569 - not sure why its needed exactly # deepcopy part of #569 - not sure why its needed exactly
self.generic_definition = deepcopy(Watch.model()) self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
if path.isfile('changedetectionio/source.txt'): if path.isfile('changedetectionio/source.txt'):
with open('changedetectionio/source.txt') as f: with open('changedetectionio/source.txt') as f:
@@ -71,13 +71,10 @@ class ChangeDetectionStore:
if 'application' in from_disk['settings']: if 'application' in from_disk['settings']:
self.__data['settings']['application'].update(from_disk['settings']['application']) self.__data['settings']['application'].update(from_disk['settings']['application'])
# Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future. # Convert each existing watch back to the Watch.model object
# @todo pretty sure theres a python we todo this with an abstracted(?) object!
for uuid, watch in self.__data['watching'].items(): for uuid, watch in self.__data['watching'].items():
_blank = deepcopy(self.generic_definition) watch['uuid']=uuid
_blank.update(watch) self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
self.__data['watching'].update({uuid: _blank})
self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
print("Watching:", uuid, self.__data['watching'][uuid]['url']) print("Watching:", uuid, self.__data['watching'][uuid]['url'])
# First time ran, doesnt exist. # First time ran, doesnt exist.
@@ -130,22 +127,6 @@ class ChangeDetectionStore:
# Finally start the thread that will manage periodic data saves to JSON # Finally start the thread that will manage periodic data saves to JSON
save_data_thread = threading.Thread(target=self.save_datastore).start() save_data_thread = threading.Thread(target=self.save_datastore).start()
# Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
def get_newest_history_key(self, uuid):
if len(self.__data['watching'][uuid]['history']) == 1:
return 0
dates = list(self.__data['watching'][uuid]['history'].keys())
# Convert to int, sort and back to str again
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
if len(dates):
# always keyed as str
return str(dates[0])
return 0
def set_last_viewed(self, uuid, timestamp): def set_last_viewed(self, uuid, timestamp):
self.data['watching'][uuid].update({'last_viewed': int(timestamp)}) self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
self.needs_write = True self.needs_write = True
@@ -170,7 +151,6 @@ class ChangeDetectionStore:
del (update_obj[dict_key]) del (update_obj[dict_key])
self.__data['watching'][uuid].update(update_obj) self.__data['watching'][uuid].update(update_obj)
self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
self.needs_write = True self.needs_write = True
@@ -188,14 +168,14 @@ class ChangeDetectionStore:
@property @property
def data(self): def data(self):
has_unviewed = False has_unviewed = False
for uuid, v in self.__data['watching'].items(): for uuid, watch in self.__data['watching'].items():
self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid) #self.__data['watching'][uuid]['viewed']=True
if int(v['newest_history_key']) <= int(v['last_viewed']): # if int(watch.newest_history_key) <= int(watch['last_viewed']):
self.__data['watching'][uuid]['viewed'] = True # self.__data['watching'][uuid]['viewed'] = True
else: # else:
self.__data['watching'][uuid]['viewed'] = False # self.__data['watching'][uuid]['viewed'] = False
has_unviewed = True # has_unviewed = True
# #106 - Be sure this is None on empty string, False, None, etc # #106 - Be sure this is None on empty string, False, None, etc
# Default var for fetch_backend # Default var for fetch_backend
@@ -239,11 +219,11 @@ class ChangeDetectionStore:
# GitHub #30 also delete history records # GitHub #30 also delete history records
for uuid in self.data['watching']: for uuid in self.data['watching']:
for path in self.data['watching'][uuid]['history'].values(): for path in self.data['watching'][uuid].history.values():
self.unlink_history_file(path) self.unlink_history_file(path)
else: else:
for path in self.data['watching'][uuid]['history'].values(): for path in self.data['watching'][uuid].history.values():
self.unlink_history_file(path) self.unlink_history_file(path)
del self.data['watching'][uuid] del self.data['watching'][uuid]
@@ -275,13 +255,14 @@ class ChangeDetectionStore:
def scrub_watch(self, uuid): def scrub_watch(self, uuid):
import pathlib import pathlib
self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'newest_history_key': 0, 'previous_md5': False}) self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': False})
self.needs_write_urgent = True self.needs_write_urgent = True
for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"): for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
unlink(item) unlink(item)
def add_watch(self, url, tag="", extras=None, write_to_disk_now=True): def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
if extras is None: if extras is None:
extras = {} extras = {}
# should always be str # should always be str
@@ -317,16 +298,15 @@ class ChangeDetectionStore:
return False return False
with self.lock: with self.lock:
# @todo use a common generic version of this
new_uuid = str(uuid_builder.uuid4())
# #Re 569 # #Re 569
# Not sure why deepcopy was needed here, sometimes new watches would appear to already have 'history' set new_watch = Watch.model(datastore_path=self.datastore_path, default={
# I assumed this would instantiate a new object but somehow an existing dict was getting used
new_watch = deepcopy(Watch.model({
'url': url, 'url': url,
'tag': tag 'tag': tag
})) })
new_uuid = new_watch['uuid']
logging.debug("Added URL {} - {}".format(url, new_uuid))
for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']: for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
if k in apply_extras: if k in apply_extras:
@@ -346,23 +326,6 @@ class ChangeDetectionStore:
self.sync_to_json() self.sync_to_json()
return new_uuid return new_uuid
# Save some text file to the appropriate path and bump the history
# result_obj from fetch_site_status.run()
def save_history_text(self, watch_uuid, contents):
import uuid
output_path = "{}/{}".format(self.datastore_path, watch_uuid)
# Incase the operator deleted it, check and create.
if not os.path.isdir(output_path):
mkdir(output_path)
fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
with open(fname, 'wb') as f:
f.write(contents)
f.close()
return fname
def get_screenshot(self, watch_uuid): def get_screenshot(self, watch_uuid):
output_path = "{}/{}".format(self.datastore_path, watch_uuid) output_path = "{}/{}".format(self.datastore_path, watch_uuid)
fname = "{}/last-screenshot.png".format(output_path) fname = "{}/last-screenshot.png".format(output_path)
@@ -448,8 +411,8 @@ class ChangeDetectionStore:
index=[] index=[]
for uuid in self.data['watching']: for uuid in self.data['watching']:
for id in self.data['watching'][uuid]['history']: for id in self.data['watching'][uuid].history:
index.append(self.data['watching'][uuid]['history'][str(id)]) index.append(self.data['watching'][uuid].history[str(id)])
import pathlib import pathlib
@@ -520,3 +483,28 @@ class ChangeDetectionStore:
# Only upgrade individual watch time if it was set # Only upgrade individual watch time if it was set
if watch.get('minutes_between_check', False): if watch.get('minutes_between_check', False):
self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check'] self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']
# Move the history list to a flat text file index
# Better than SQLite because this list is only appended to, and works across NAS / NFS type setups
def update_2(self):
# @todo test running this on a newly updated one (when this already ran)
for uuid, watch in self.data['watching'].items():
history = []
if watch.get('history', False):
for d, p in watch['history'].items():
d = int(d) # Used to be keyed as str, we'll fix this now too
history.append("{},{}\n".format(d,p))
if len(history):
target_path = os.path.join(self.datastore_path, uuid)
if os.path.exists(target_path):
with open(os.path.join(target_path, "history.txt"), "w") as f:
f.writelines(history)
else:
logging.warning("Datastore history directory {} does not exist, skipping history import.".format(target_path))
# No longer needed, dynamically pulled from the disk when needed.
# But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.
# In the distant future we can remove this entirely
self.data['watching'][uuid]['history'] = {}

8
changedetectionio/templates/watch-overview.html
View File

@@ -46,7 +46,7 @@
{% if watch.last_error is defined and watch.last_error != False %}error{% endif %} {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
{% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %} {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
{% if watch.paused is defined and watch.paused != False %}paused{% endif %} {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
{% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %} {% if watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}unviewed{% endif %}
{% if watch.uuid in queued_uuids %}queued{% endif %}"> {% if watch.uuid in queued_uuids %}queued{% endif %}">
<td class="inline">{{ loop.index }}</td> <td class="inline">{{ loop.index }}</td>
<td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td> <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
@@ -68,7 +68,7 @@
{% endif %} {% endif %}
</td> </td>
<td class="last-checked">{{watch|format_last_checked_time}}</td> <td class="last-checked">{{watch|format_last_checked_time}}</td>
<td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %} <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed %}
{{watch.last_changed|format_timestamp_timeago}} {{watch.last_changed|format_timestamp_timeago}}
{% else %} {% else %}
Not yet Not yet
@@ -78,10 +78,10 @@
<a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a> class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
<a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a> <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
{% if watch.history|length >= 2 %} {% if watch.history_n >= 2 %}
<a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a> <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
{% else %} {% else %}
{% if watch.history|length == 1 %} {% if watch.history_n == 1 %}
<a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a> <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
{% endif %} {% endif %}
{% endif %} {% endif %}

17
changedetectionio/tests/test_api.py
View File

@@ -2,7 +2,7 @@
import time import time
from flask import url_for from flask import url_for
from .util import live_server_setup from .util import live_server_setup, extract_api_key_from_UI
import json import json
import uuid import uuid
@@ -53,23 +53,10 @@ def is_valid_uuid(val):
return False return False
# kinda funky, but works for now
def _extract_api_key_from_UI(client):
import re
res = client.get(
url_for("settings_page"),
)
# <span id="api-key">{{api_key}}</span>
m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
api_key = m.group(1)
return api_key.strip()
def test_api_simple(client, live_server): def test_api_simple(client, live_server):
live_server_setup(live_server) live_server_setup(live_server)
api_key = _extract_api_key_from_UI(client) api_key = extract_api_key_from_UI(client)
# Create a watch # Create a watch
set_original_response() set_original_response()

5
changedetectionio/tests/test_element_removal.py
View File

@@ -150,9 +150,8 @@ def test_element_removal_full(client, live_server):
# Give the thread time to pick it up # Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# No change yet - first check # so that we set the state to 'unviewed' after all the edits
res = client.get(url_for("index")) client.get(url_for("diff_history_page", uuid="first"))
assert b"unviewed" not in res.data
# Make a change to header/footer/nav # Make a change to header/footer/nav
set_modified_response() set_modified_response()

84
changedetectionio/tests/test_history_consistency.py Normal file
View File

@@ -0,0 +1,84 @@
#!/usr/bin/python3
import time
import os
import json
import logging
from flask import url_for
from .util import live_server_setup
from urllib.parse import urlparse, parse_qs
def test_consistent_history(client, live_server):
live_server_setup(live_server)
# Give the endpoint time to spin up
time.sleep(1)
r = range(1, 50)
for one in r:
test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(3)
while True:
res = client.get(url_for("index"))
logging.debug("Waiting for 'Checking now' to go away..")
if b'Checking now' not in res.data:
break
time.sleep(0.5)
time.sleep(3)
# Essentially just triggers the DB write/update
res = client.post(
url_for("settings_page"),
data={"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Settings updated." in res.data
# Give it time to write it out
time.sleep(3)
json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
json_obj = None
with open(json_db_file, 'r') as f:
json_obj = json.load(f)
# assert the right amount of watches was found in the JSON
assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"
# each one should have a history.txt containing just one line
for w in json_obj['watching'].keys():
history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
assert os.path.isfile(history_txt_index_file), "History.txt should exist where I expect it - {}".format(history_txt_index_file)
# Same like in model.Watch
with open(history_txt_index_file, "r") as f:
tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
assert len(tmp_history) == 1, "History.txt should contain 1 line"
# Should be two files,. the history.txt , and the snapshot.txt
files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,
w))
# Find the snapshot one
for fname in files_in_watch_dir:
if fname != 'history.txt':
# contents should match what we requested as content returned from the test url
with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
contents = snapshot_f.read()
watch_url = json_obj['watching'][w]['url']
u = urlparse(watch_url)
q = parse_qs(u[4])
assert q['content'][0] == contents.strip(), "Snapshot file {} should contain {}".format(fname, q['content'][0])
assert len(files_in_watch_dir) == 2, "Should be just two files in the dir, history.txt and the snapshot"

9
changedetectionio/tests/test_trigger.py
View File

@@ -78,9 +78,6 @@ def test_trigger_functionality(client, live_server):
# Trigger a check # Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True) client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
# Goto the edit page, add our ignore text # Goto the edit page, add our ignore text
# Add our URL to the import page # Add our URL to the import page
res = client.post( res = client.post(
@@ -98,6 +95,12 @@ def test_trigger_functionality(client, live_server):
) )
assert bytes(trigger_text.encode('utf-8')) in res.data assert bytes(trigger_text.encode('utf-8')) in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
# so that we set the state to 'unviewed' after all the edits
client.get(url_for("diff_history_page", uuid="first"))
# Trigger a check # Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True) client.get(url_for("form_watch_checknow"), follow_redirects=True)

11
changedetectionio/tests/test_trigger_regex.py
View File

@@ -42,9 +42,6 @@ def test_trigger_regex_functionality(client, live_server):
) )
assert b"1 Imported" in res.data assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up # Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
@@ -60,7 +57,9 @@ def test_trigger_regex_functionality(client, live_server):
"fetch_backend": "html_requests"}, "fetch_backend": "html_requests"},
follow_redirects=True follow_redirects=True
) )
time.sleep(sleep_time_for_fetch_thread)
# so that we set the state to 'unviewed' after all the edits
client.get(url_for("diff_history_page", uuid="first"))
with open("test-datastore/endpoint-content.txt", "w") as f: with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("some new noise") f.write("some new noise")
@@ -79,3 +78,7 @@ def test_trigger_regex_functionality(client, live_server):
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
res = client.get(url_for("index")) res = client.get(url_for("index"))
assert b'unviewed' in res.data assert b'unviewed' in res.data
# Cleanup everything
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data

24
changedetectionio/tests/test_trigger_regex_with_filter.py
View File

@@ -22,10 +22,9 @@ def set_original_ignore_response():
def test_trigger_regex_functionality(client, live_server): def test_trigger_regex_functionality_with_filter(client, live_server):
live_server_setup(live_server) live_server_setup(live_server)
sleep_time_for_fetch_thread = 3 sleep_time_for_fetch_thread = 3
set_original_ignore_response() set_original_ignore_response()
@@ -42,26 +41,24 @@ def test_trigger_regex_functionality(client, live_server):
) )
assert b"1 Imported" in res.data assert b"1 Imported" in res.data
# Trigger a check # it needs time to save the original version
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# It should report nothing found (just a new one shouldnt have anything)
res = client.get(url_for("index"))
assert b'unviewed' not in res.data
### test regex with filter ### test regex with filter
res = client.post( res = client.post(
url_for("edit_page", uuid="first"), url_for("edit_page", uuid="first"),
data={"trigger_text": "/cool.stuff\d/", data={"trigger_text": "/cool.stuff/",
"url": test_url, "url": test_url,
"css_filter": '#in-here', "css_filter": '#in-here',
"fetch_backend": "html_requests"}, "fetch_backend": "html_requests"},
follow_redirects=True follow_redirects=True
) )
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
client.get(url_for("diff_history_page", uuid="first"))
# Check that we have the expected text.. but it's not in the css filter we want # Check that we have the expected text.. but it's not in the css filter we want
with open("test-datastore/endpoint-content.txt", "w") as f: with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("<html>some new noise with cool stuff2 ok</html>") f.write("<html>some new noise with cool stuff2 ok</html>")
@@ -73,6 +70,7 @@ def test_trigger_regex_functionality(client, live_server):
res = client.get(url_for("index")) res = client.get(url_for("index"))
assert b'unviewed' not in res.data assert b'unviewed' not in res.data
# now this should trigger something
with open("test-datastore/endpoint-content.txt", "w") as f: with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>") f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
@@ -81,4 +79,6 @@ def test_trigger_regex_functionality(client, live_server):
res = client.get(url_for("index")) res = client.get(url_for("index"))
assert b'unviewed' in res.data assert b'unviewed' in res.data
# Cleanup everything
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data

19
changedetectionio/tests/util.py
View File

@@ -1,6 +1,7 @@
#!/usr/bin/python3 #!/usr/bin/python3
from flask import make_response, request from flask import make_response, request
from flask import url_for
def set_original_response(): def set_original_response():
test_return_data = """<html> test_return_data = """<html>
@@ -55,14 +56,32 @@ def set_more_modified_response():
return None return None
# kinda funky, but works for now
def extract_api_key_from_UI(client):
import re
res = client.get(
url_for("settings_page"),
)
# <span id="api-key">{{api_key}}</span>
m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
api_key = m.group(1)
return api_key.strip()
def live_server_setup(live_server): def live_server_setup(live_server):
@live_server.app.route('/test-endpoint') @live_server.app.route('/test-endpoint')
def test_endpoint(): def test_endpoint():
ctype = request.args.get('content_type') ctype = request.args.get('content_type')
status_code = request.args.get('status_code') status_code = request.args.get('status_code')
content = request.args.get('content') or None
try: try:
if content is not None:
resp = make_response(content, status_code)
resp.headers['Content-Type'] = ctype if ctype else 'text/html'
return resp
# Tried using a global var here but didn't seem to work, so reading from a file instead. # Tried using a global var here but didn't seem to work, so reading from a file instead.
with open("test-datastore/endpoint-content.txt", "r") as f: with open("test-datastore/endpoint-content.txt", "r") as f:
resp = make_response(f.read(), status_code) resp = make_response(f.read(), status_code)

16
changedetectionio/update_worker.py
View File

@@ -75,9 +75,7 @@ class update_worker(threading.Thread):
# For the FIRST time we check a site, or a change detected, save the snapshot. # For the FIRST time we check a site, or a change detected, save the snapshot.
if changed_detected or not watch['last_checked']: if changed_detected or not watch['last_checked']:
# A change was detected # A change was detected
fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents) fname = watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
# Should always be keyed by string(timestamp)
self.datastore.update_watch(uuid, {"history": {str(round(time.time())): fname}})
# Generally update anything interesting returned # Generally update anything interesting returned
self.datastore.update_watch(uuid=uuid, update_obj=update_obj) self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
@@ -88,16 +86,10 @@ class update_worker(threading.Thread):
print (">> Change detected in UUID {} - {}".format(uuid, watch['url'])) print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
# Notifications should only trigger on the second time (first time, we gather the initial snapshot) # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
if len(watch['history']) > 1: if watch.history_n >= 2:
dates = list(watch['history'].keys()) dates = list(watch.history.keys())
# Convert to int, sort and back to str again prev_fname = watch.history[dates[-2]]
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
prev_fname = watch['history'][dates[1]]
# Did it have any notification alerts to hit? # Did it have any notification alerts to hit?