Move history data to a textfile, improves memory handling (#638)

changedetectionio/model/Watch.py

@@ -1,5 +1,4 @@
 import os
-
 import uuid as uuid_builder
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
@@ -12,22 +11,24 @@ from changedetectionio.notification import (
 
 
 class model(dict):
-    base_config = {
+    __newest_history_key = None
+    __history_n=0
+
+    __base_config = {
             'url': None,
             'tag': None,
             'last_checked': 0,
             'last_changed': 0,
             'paused': False,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'newest_history_key': 0,
+            #'newest_history_key': 0,
             'title': None,
             'previous_md5': False,
-#           UUID not needed, should be generated only as a key
-#            'uuid':
+            'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
             'body': None,
             'method': 'GET',
-            'history': {},  # Dict of timestamp and output stripped filename
+            #'history': {},  # Dict of timestamp and output stripped filename
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             # Custom notification content
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
@@ -48,10 +49,103 @@ class model(dict):
         }
 
     def __init__(self, *arg, **kw):
-        self.update(self.base_config)
+        import uuid
+        self.update(self.__base_config)
+        self.__datastore_path = kw['datastore_path']
+
+        self['uuid'] = str(uuid.uuid4())
+
+        del kw['datastore_path']
+
+        if kw.get('default'):
+            self.update(kw['default'])
+            del kw['default']
+
         # goes at the end so we update the default object with the initialiser
         super(model, self).__init__(*arg, **kw)
 
+    @property
+    def viewed(self):
+        if int(self.newest_history_key) <= int(self['last_viewed']):
+            return True
+
+        return False
+
+    @property
+    def history_n(self):
+        return self.__history_n
+
+    @property
+    def history(self):
+        tmp_history = {}
+        import logging
+        import time
+
+        # Read the history file as a dict
+        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
+        if os.path.isfile(fname):
+            logging.debug("Disk IO accessed " + str(time.time()))
+            with open(fname, "r") as f:
+                tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
+
+        if len(tmp_history):
+            self.__newest_history_key = list(tmp_history.keys())[-1]
+
+        self.__history_n = len(tmp_history)
+
+        return tmp_history
+
+    @property
+    def has_history(self):
+        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
+        return os.path.isfile(fname)
+
+    # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
+    @property
+    def newest_history_key(self):
+        if self.__newest_history_key is not None:
+            return self.__newest_history_key
+
+        if len(self.history) <= 1:
+            return 0
+
+
+        bump = self.history
+        return self.__newest_history_key
+
+
+    # Save some text file to the appropriate path and bump the history
+    # result_obj from fetch_site_status.run()
+    def save_history_text(self, contents, timestamp):
+        import uuid
+        from os import mkdir, path, unlink
+        import logging
+
+        output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
+
+        # Incase the operator deleted it, check and create.
+        if not os.path.isdir(output_path):
+            mkdir(output_path)
+
+        snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
+        logging.debug("Saving history text {}".format(snapshot_fname))
+
+        with open(snapshot_fname, 'wb') as f:
+            f.write(contents)
+            f.close()
+
+        # Append to index
+        # @todo check last char was \n
+        index_fname = "{}/history.txt".format(output_path)
+        with open(index_fname, 'a') as f:
+            f.write("{},{}\n".format(timestamp, snapshot_fname))
+            f.close()
+
+        self.__newest_history_key = timestamp
+        self.__history_n+=1
+
+        #@todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
+        return snapshot_fname
 
     @property
     def has_empty_checktime(self):