cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import speed improvements, and adding an import URL batch size of 5,000 to stop accidental CPU overload (#549)

Browse Source
This commit is contained in:
dgtlmoon
2022-04-19 23:15:32 +02:00
committed by GitHub
parent 6c3c5dc28a
commit 2a9fb12451
4 changed files with 20 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
changedetectionio/__init__.py
View File

@@ -661,13 +661,19 @@ def changedetection_app(config=None, datastore_o=None):
good = 0
if request.method == 'POST':
now=time.time()
urls = request.values.get('urls').split("\n")
if (len(urls) > 5000):
flash("Importing 5,000 of the first URLs from your list, the rest can be imported again.")
for url in urls:
url = url.strip()
url, *tags = url.split(" ")
# Flask wtform validators wont work with basic auth, use validators package
if len(url) and validators.url(url.replace('source:', '')):
new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags))
# Up to 5000 per batch so we dont flood the server
if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags), write_to_disk_now=False)
# Straight into the queue.
update_q.put(new_uuid)
good += 1
@@ -675,7 +681,8 @@ def changedetection_app(config=None, datastore_o=None):
if len(url):
remaining_urls.append(url)
flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
flash("{} Imported in {:.2f}s, {} Skipped.".format(good, time.time()-now,len(remaining_urls)))
datastore.needs_write = True
if len(remaining_urls) == 0:
# Looking good, redirect to index.