From 486e245c144d02c3ce4ace1ea89532437152596f Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Fri, 18 Aug 2023 08:59:35 +1000 Subject: [PATCH] Duplicate Plugin Objects bug work --- front/plugins/README.md | 3 +++ pialert/__main__.py | 8 ++++---- pialert/database.py | 19 +++++++++++++++++++ pialert/plugin.py | 23 ++++++++++++++++------- pialert/scheduler.py | 4 ++++ 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/front/plugins/README.md b/front/plugins/README.md index 54ae6d5c..c980b5f6 100755 --- a/front/plugins/README.md +++ b/front/plugins/README.md @@ -95,6 +95,8 @@ More on specifics below. | 7 | `Extra` | no | Any other data you want to pass and display in PiAlert and the notifications | | 8 | `ForeignKey` | no | A foreign key that can be used to link to the parent object (usually a MAC address) | +> [!NOTE] +> De-duplication is run once an hour on the `Plugins_Objects` database table and duplicate entries with the same value in columns `Object_PrimaryID`, `Object_SecondaryID`, `Plugin` (auto-filled based on `unique_prefix` of the plugin), `UserData` (can be populated with the `"type": "textboxsave"` column type) are removed. # config.json structure @@ -540,6 +542,7 @@ The UI will adjust how columns are displayed in the UI based on the definition o - `devicemac` - The value is considered to be a Mac address and a link pointing to the device with the given Mac address is generated. - `deviceip` - The value is considered to be an IP address and a link pointing to the device with the given IP is generated. The IP is checked against the last detected IP addresses and translated into a Mac address that is then used for the link itself. - `url` - The value is considered to be a URL so a link is generated. +- `textboxsave` - An editable and saveable text box is generated that saves values in the database. Primarily intended for the `UserData` database column in the `Plugins_Objects` table. ```json diff --git a/pialert/__main__.py b/pialert/__main__.py index 8709eade..7d729aee 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -113,10 +113,10 @@ def main (): # check if new version is available / only check once an hour - if conf.last_version_check + datetime.timedelta(hours=1) < loop_start_time : + if conf.last_version_check + datetime.timedelta(hours=1) < conf.loop_start_time : # if newVersionAvailable is already true the function does nothing and returns true again mylog('debug', [f"[Version check] Last version check timestamp: {conf.last_version_check}"]) - conf.last_version_check = loop_start_time + conf.last_version_check = conf.loop_start_time conf.newVersionAvailable = isNewVersion(conf.newVersionAvailable) # Handle plugins executed ONCE @@ -232,8 +232,8 @@ def main (): send_notifications(db) # clean up the DB once an hour - if last_cleanup + datetime.timedelta(hours = 1) < loop_start_time: - last_cleanup = loop_start_time + if conf.last_cleanup + datetime.timedelta(hours = 1) < loop_start_time: + conf.last_cleanup = loop_start_time conf.cycle = 'cleanup' mylog('verbose', ['[MAIN] cycle:',conf.cycle]) db.cleanup_database(startTime, conf.DAYS_TO_KEEP_EVENTS, conf.PHOLUS_DAYS_DATA, conf.HRS_TO_KEEP_NEWDEV, conf.PLUGINS_KEEP_HIST) diff --git a/pialert/database.py b/pialert/database.py index 437b9ea2..90eb504a 100755 --- a/pialert/database.py +++ b/pialert/database.py @@ -50,6 +50,11 @@ class DB(): return True #------------------------------------------------------------------------------- + def rollbackDB(self): + if self.sql_connection: + self.sql_connection.rollback() + + #------------------------------------------------------------------------------- def get_sql_array(self, query): if self.sql_connection == None : mylog('debug','getQueryArray: databse is not open') @@ -114,6 +119,20 @@ class DB(): self.sql.execute (f"""DELETE FROM Devices WHERE dev_NewDevice = 1 AND dev_FirstConnection < date('now', '+{str(HRS_TO_KEEP_NEWDEV)} hour')""") + + # De-dupe (de-duplicate) from the Plugins_Objects table + mylog('verbose', ['[DB Cleanup] Plugins_Objects: Delete all duplicates']) + self.sql.execute(""" + DELETE FROM Plugins_Objects + WHERE rowid > ( + SELECT MIN(rowid) FROM Plugins_Objects p2 + WHERE Plugins_Objects.Plugin = p2.Plugin + AND Plugins_Objects.Object_PrimaryID = p2.Object_PrimaryID + AND Plugins_Objects.Object_SecondaryID = p2.Object_SecondaryID + AND Plugins_Objects.UserData = p2.UserData + ) + """) + # De-Dupe (de-duplicate - remove duplicate entries) from the Pholus_Scan table mylog('verbose', ['[DB Cleanup] Pholus_Scan: Delete all duplicates']) self.sql.execute ("""DELETE FROM Pholus_Scan diff --git a/pialert/plugin.py b/pialert/plugin.py index 58573249..6d4ee50f 100755 --- a/pialert/plugin.py +++ b/pialert/plugin.py @@ -42,14 +42,11 @@ def run_plugin_scripts(db, runType, pluginsState = None): shouldRun = True elif runType == "schedule": # run if overdue scheduled time - # check schedules if any contains a unique plugin prefix matching the current plugin + # check schedules if any contains a unique plugin prefix matching the current plugin for schd in conf.mySchedules: if schd.service == prefix: # Check if schedule overdue shouldRun = schd.runScheduleCheck() - if shouldRun: - # note the last time the scheduled plugin run was executed - schd.last_run = timeNowTZ() if shouldRun: # Header @@ -58,6 +55,12 @@ def run_plugin_scripts(db, runType, pluginsState = None): print_plugin_info(plugin, ['display_name']) mylog('debug', ['[Plugins] CMD: ', get_plugin_setting(plugin, "CMD")["value"]]) pluginsState = execute_plugin(db, plugin, pluginsState) + # update last run time + if runType == "schedule": + for schd in conf.mySchedules: + if schd.service == prefix: + # note the last time the scheduled plugin run was executed + schd.last_run = timeNowTZ() return pluginsState @@ -103,7 +106,7 @@ def get_plugin_setting(plugin, function_key): result = set if result == None: - mylog('none', ['[Plugins] Setting with "function":"', function_key, '" is missing in plugin: ', get_plugin_string(plugin, 'display_name')]) + mylog('debug', ['[Plugins] Setting with "function":"', function_key, '" is missing in plugin: ', get_plugin_string(plugin, 'display_name')]) return result @@ -298,8 +301,14 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ): if len(sqlParams) > 0: sql.executemany ("""INSERT INTO Plugins_Events ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) db.commitDB() - sql.executemany ("""INSERT INTO Plugins_History ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) - db.commitDB() + + try: + sql.executemany("""INSERT INTO Plugins_History ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) + db.commitDB() + except sqlite3.Error as e: + db.rollbackDB() # Rollback changes in case of an error + mylog('none', ['[Plugins] ERROR inserting into Plugins_History:', e]) + # create objects pluginsState = process_plugin_events(db, plugin, pluginsState) diff --git a/pialert/scheduler.py b/pialert/scheduler.py index 2350ded0..f61333f5 100755 --- a/pialert/scheduler.py +++ b/pialert/scheduler.py @@ -27,12 +27,16 @@ class schedule_class: # Run the schedule if the current time is past the schedule time we saved last time and # (maybe the following check is unnecessary:) # if the last run is past the last time we run a scheduled Pholus scan + # if nowTime > self.last_next_schedule and self.last_run < self.last_next_schedule: if nowTime > self.last_next_schedule and self.last_run < self.last_next_schedule: mylog('debug',f'[Scheduler] - Scheduler run for {self.service}: YES') self.was_last_schedule_used = True result = True else: mylog('debug',f'[Scheduler] - Scheduler run for {self.service}: NO') + mylog('debug',f'[Scheduler] - nowTime {nowTime}') + mylog('debug',f'[Scheduler] - self.last_next_schedule {self.last_next_schedule}') + mylog('debug',f'[Scheduler] - self.last_run {self.last_run}') if self.was_last_schedule_used: self.was_last_schedule_used = False