From be4e0acdfc869f1b4ada89efb8e80671acbb91dd Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Sun, 8 Oct 2023 14:52:22 +1100 Subject: [PATCH] Notification rework - Apprise v1 - working --- back/report_template.txt | 6 +- back/webhook_json_sample.json | 156 +++++++++---------- front/plugins/README.md | 3 +- front/plugins/_publisher_apprise/config.json | 4 +- front/pluginsCore.php | 5 + pialert/__main__.py | 6 +- pialert/helper.py | 32 ++-- pialert/notification.py | 25 +-- pialert/plugin.py | 55 +++---- pialert/plugin_utils.py | 19 +-- pialert/reporting.py | 23 +-- 11 files changed, 152 insertions(+), 182 deletions(-) diff --git a/back/report_template.txt b/back/report_template.txt index 85f8575a..80c9562c 100755 --- a/back/report_template.txt +++ b/back/report_template.txt @@ -1,7 +1,7 @@ Report Date: Server: - - - + + + diff --git a/back/webhook_json_sample.json b/back/webhook_json_sample.json index 940630d0..a782fa16 100755 --- a/back/webhook_json_sample.json +++ b/back/webhook_json_sample.json @@ -1,90 +1,76 @@ [ - { - "headers": { - "host": "192.168.1.82:5678", - "user-agent": "curl/7.74.0", - "accept": "*/*", - "content-type": "application/json", - "content-length": "872" - }, - "params": {}, - "query": {}, - "body": { - "username": "Pi.Alert", - "text": "There are new notifications", - "attachments": [ - { - "title": "Pi.Alert Notifications", - "title_link": "", - "text": { - "internet": [], - "new_devices": [{ - "MAC": "74:ac:74:ac:74:ac", - "Datetime": "2023-01-30 22:15:09", - "IP": "192.168.1.1", - "Event Type": "New Device", - "Device name": "(name not found)", - "Comments": null, - "Device Vendor": null - }], - "down_devices": [], - "events": [{ - "MAC": "74:ac:74:ac:74:ac", - "Datetime": "2023-01-30 22:15:09", - "IP": "192.168.1.92", - "Event Type": "Disconnected", - "Device name": "(name not found)", - "Comments": null, - "Device Vendor": null - }, { - "MAC": "74:ac:74:ac:74:ac", - "Datetime": "2023-01-30 22:15:09", - "IP": "192.168.1.150", - "Event Type": "Disconnected", - "Device name": "(name not found)", - "Comments": null, - "Device Vendor": null - }], - "ports": [{ - "new": { - "Name": "New device", - "MAC": "74:ac:74:ac:74:ac", - "Port": "22/tcp", - "State": "open", - "Service": "ssh", - "Extra": "" - } - }, { - "new": { - "Name": "New device", - "MAC": "74:ac:74:ac:74:ac", - "Port": "53/tcp", - "State": "open", - "Service": "domain", - "Extra": "" - } - }, { - "new": { - "Name": "New device", - "MAC": "74:ac:74:ac:74:ac", - "Port": "80/tcp", - "State": "open", - "Service": "http", - "Extra": "" - } - }, { - "new": { - "Name": "New device", - "MAC": "74:ac:74:ac:74:ac", - "Port": "443/tcp", - "State": "open", - "Service": "https", - "Extra": "" - } - }] - } - } + { + "headers": { + "host": "192.168.1.82:5678", + "user-agent": "curl/7.74.0", + "accept": "*/*", + "content-type": "application/json", + "content-length": "872" + }, + "params": {}, + "query": {}, + "body": { + "username": "Pi.Alert", + "text": "There are new notifications", + "attachments": [ + { + "title": "Pi.Alert Notifications", + "title_link": "", + "text": { + "internet": [], + "new_devices": [ + { + "MAC": "74:ac:74:ac:74:ac", + "Datetime": "2023-01-30 22:15:09", + "IP": "192.168.1.1", + "Event Type": "New Device", + "Device name": "(name not found)", + "Comments": null, + "Device Vendor": null + } + ], + "down_devices": [], + "events": [ + { + "MAC": "74:ac:74:ac:74:ac", + "Datetime": "2023-01-30 22:15:09", + "IP": "192.168.1.92", + "Event Type": "Disconnected", + "Device name": "(name not found)", + "Comments": null, + "Device Vendor": null + }, + { + "MAC": "74:ac:74:ac:74:ac", + "Datetime": "2023-01-30 22:15:09", + "IP": "192.168.1.150", + "Event Type": "Disconnected", + "Device name": "(name not found)", + "Comments": null, + "Device Vendor": null + } + ], + "plugins": [ + { + "Index": 138, + "Plugin": "INTRSPD", + "Object_PrimaryID": "Speedtest", + "Object_SecondaryID": "2023-10-08 02:01:16+02:00", + "DateTimeCreated": "2023-10-08 02:01:16", + "DateTimeChanged": "2023-10-08 02:32:15", + "Watched_Value1": "-1", + "Watched_Value2": "-1", + "Watched_Value3": "null", + "Watched_Value4": "null", + "Status": "missing-in-last-scan", + "Extra": "null", + "UserData": "null", + "ForeignKey": "null" + } ] + } } + ] } + } ] \ No newline at end of file diff --git a/front/plugins/README.md b/front/plugins/README.md index 53353fdc..9764f70d 100755 --- a/front/plugins/README.md +++ b/front/plugins/README.md @@ -619,7 +619,8 @@ The UI will adjust how columns are displayed in the UI based on the resolvers de | Supported Types | Description | | -------------- | ----------- | | `label` | Displays a column only. | -| `text` | Makes a column editable, and a save icon is displayed next to it. See below for information on `threshold`, `replace`. | +| `textarea_readonly` | Generates a read only text area and cleans up the text to display it somewhat formatted with new lines preserved. | +| See below for information on `threshold`, `replace`. | | | | | | `options` Property | Used in conjunction with types like `threshold`, `replace`, `regex`. | | `threshold` | The `options` array contains objects ordered from the lowest `maximum` to the highest. The corresponding `hexColor` is used for the value background color if it's less than the specified `maximum` but more than the previous one in the `options` array. | diff --git a/front/plugins/_publisher_apprise/config.json b/front/plugins/_publisher_apprise/config.json index 1d890b7c..99b05b97 100755 --- a/front/plugins/_publisher_apprise/config.json +++ b/front/plugins/_publisher_apprise/config.json @@ -137,9 +137,9 @@ }, { "column": "Watched_Value2", - "css_classes": "col-sm-2", + "css_classes": "col-sm-8", "show": true, - "type": "label", + "type": "textarea_readonly", "default_value":"", "options": [], "localized": ["name"], diff --git a/front/pluginsCore.php b/front/pluginsCore.php index e30f986f..e0d6fdb6 100755 --- a/front/pluginsCore.php +++ b/front/pluginsCore.php @@ -89,6 +89,11 @@ function processColumnValue(dbColumnDef, value, index, type) { case 'label': value = `${value}`; break; + case 'textarea_readonly': + value = ``; + break; case 'textbox_save': value = value == 'null' ? '' : value; // hide 'null' values diff --git a/pialert/__main__.py b/pialert/__main__.py index 8a07f430..6eedc75e 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -155,10 +155,10 @@ def main (): # Write the notifications into the DB notification = Notification_obj(db) - hasNotification = notification.create(notiStructure.json, notiStructure.text, notiStructure.html, "") + notificationObj = notification.create(notiStructure.json, notiStructure.text, notiStructure.html, "") # run all enabled publisher gateways - if hasNotification: + if notificationObj.HasNotifications: pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) notification.setAllProcessed() @@ -177,6 +177,8 @@ def main (): # DEBUG - print number of rows updated mylog('minimal', ['[Notification] Notifications changes: ', sql.rowcount]) + else: + mylog('verbose', ['[Notification] No changes to report']) # Commit SQL db.commitDB() diff --git a/pialert/helper.py b/pialert/helper.py index 19e1aabe..4a36c8b5 100755 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -244,27 +244,24 @@ def write_file(pPath, pText): # Return whole setting touple def get_setting(key): - settingsFile = apiPath + '/table_settings.json' + settingsFile = apiPath + 'table_settings.json' try: with open(settingsFile, 'r') as json_file: data = json.load(json_file) - # if not isinstance(data, list): - # mylog('minimal', [f' [Settings] Data is not a list of dictionaries (file: {settingsFile})']) - for item in data.get("data",[]): if item.get("Code_Name") == key: return item - mylog('minimal', [f'[Settings] Error - setting_missing - Setting not found for key: {key} in file {settingsFile}']) + mylog('debug', [f'[Settings] Error - setting_missing - Setting not found for key: {key} in file {settingsFile}']) return None except (FileNotFoundError, json.JSONDecodeError, ValueError) as e: # Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format - mylog('minimal', [f'[Settings] Error - JSONDecodeError or FileNotFoundError for file {settingsFile}']) + mylog('none', [f'[Settings] Error - JSONDecodeError or FileNotFoundError for file {settingsFile}']) return None @@ -274,19 +271,32 @@ def get_setting(key): # Return setting value def get_setting_value(key): - set = get_setting(key) + setting = get_setting(key) - if get_setting(key) is not None: + if setting is not None: - setVal = set["Value"] # setting value - setTyp = set["Type"] # setting type + set_value = setting["Value"] # Setting value + set_type = setting["Type"] # Setting type - return setVal + # Handle different types of settings + if set_type in ['text', 'string', 'password', 'readonly', 'text.select']: + return str(set_value) + elif set_type in ['boolean', 'integer.checkbox']: + return bool(set_value) + elif set_type in ['integer.select', 'integer']: + return int(set_value) + elif set_type in ['text.multiselect', 'list', 'subnets']: + # Assuming set_value is a list in this case + return set_value + elif set_type == '.template': + # Assuming set_value is a JSON object in this case + return json.loads(set_value) return '' + #------------------------------------------------------------------------------- # IP validation methods #------------------------------------------------------------------------------- diff --git a/pialert/notification.py b/pialert/notification.py index b9d8a052..82c474a6 100644 --- a/pialert/notification.py +++ b/pialert/notification.py @@ -40,11 +40,8 @@ class Notification_obj: # Check if nothing to report, end if JSON["internet"] == [] and JSON["new_devices"] == [] and JSON["down_devices"] == [] and JSON["events"] == [] and JSON["plugins"] == []: self.HasNotifications = False - # end if nothing to report - return self.HasNotifications - - # continue and save into DB if notifications available - self.HasNotifications = True + else: + self.HasNotifications = True self.GUID = str(uuid.uuid4()) self.DateTimeCreated = timeNowTZ() @@ -56,9 +53,10 @@ class Notification_obj: self.PublishedVia = "" self.Extra = Extra - self.upsert() + if self.HasNotifications: + self.upsert() - return self.HasNotifications + return self # Only updates the status def updateStatus(self, newStatus): @@ -69,9 +67,7 @@ class Notification_obj: def updatePublishedVia(self, newPublishedVia): self.PublishedVia = newPublishedVia self.DateTimePushed = timeNowTZ() - self.upsert() - - # TODO Index vs hash to minimize SQL calls, finish CRUD operations, expose via API, use API in plugins + self.upsert() # create or update a notification def upsert(self): @@ -82,6 +78,15 @@ class Notification_obj: self.save() + # Remove notification object by GUID + def remove(self, GUID): + # Execute an SQL query to delete the notification with the specified GUID + self.db.sql.execute(""" + DELETE FROM Notifications + WHERE GUID = ? + """, (GUID,)) + self.save() + # Get all with the "new" status def getNew(self): self.db.sql.execute(""" diff --git a/pialert/plugin.py b/pialert/plugin.py index 5e77e61a..e0a2f8f1 100755 --- a/pialert/plugin.py +++ b/pialert/plugin.py @@ -9,11 +9,12 @@ from collections import namedtuple # pialert modules import conf -from const import pluginsPath, logPath +from const import pluginsPath, logPath, pialertPath from logger import mylog from helper import timeNowTZ, updateState, get_file_content, write_file, get_setting, get_setting_value from api import update_api -from plugin_utils import logEventStatusCounts, get_plugin_string, get_plugin_setting, print_plugin_info, list_to_csv, combine_plugin_objects, resolve_wildcards_arr, get_plugin_setting_value, handle_empty, custom_plugin_decoder +from plugin_utils import logEventStatusCounts, get_plugin_string, get_plugin_setting, print_plugin_info, list_to_csv, combine_plugin_objects, resolve_wildcards_arr, handle_empty, custom_plugin_decoder +from notification import Notification_obj #------------------------------------------------------------------------------- @@ -484,10 +485,7 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr): history_to_insert = [] objects_to_update = [] - statuses_to_report_on = get_plugin_setting_value(plugin, "REPORT_ON") - - mylog('debug', ['[Plugins] statuses_to_report_on: ', statuses_to_report_on]) - + for plugObj in pluginObjects: # keep old createdTime time if the plugObj already was created before createdTime = plugObj.changed if plugObj.status == 'new' else plugObj.created @@ -505,6 +503,8 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr): objects_to_update.append(values + (plugObj.index,)) # Include index for UPDATE # only generate events that we want to be notified on + statuses_to_report_on = get_setting_value(plugObj.pluginPref + "_REPORT_ON") + if plugObj.status in statuses_to_report_on: events_to_insert.append(values) @@ -777,39 +777,24 @@ def handle_run(runType, db, pluginsState): #------------------------------------------------------------------------------- def handle_test(runType, db, pluginsState): - mylog('minimal', ['[', timeNowTZ(), '] START Test: ', testType]) + mylog('minimal', ['[', timeNowTZ(), '] [Test] START Test: ', runType]) - # TODO finish + # Prepare test samples + sample_txt = get_file_content(pialertPath + '/back/report_sample.txt') + sample_html = get_file_content(pialertPath + '/back/report_sample.html') + sample_json = json.loads(get_file_content(pialertPath + '/back/webhook_json_sample.json'))[0]["body"]["attachments"][0]["text"] + + # Create fake notification + notification = Notification_obj(db) + notificationObj = notification.create(sample_json, sample_txt, sample_html, "") - # # Open text sample - # sample_txt = get_file_content(pialertPath + '/back/report_sample.txt') + # Run test + pluginsState = handle_run(runType, db, pluginsState) - # # Open html sample - # sample_html = get_file_content(pialertPath + '/back/report_sample.html') + # Remove sample notification + notificationObj.remove(notificationObj.GUID) - # # Open json sample and get only the payload part - # sample_json_payload = json.loads(get_file_content(pialertPath + '/back/webhook_json_sample.json'))[0]["body"]["attachments"][0]["text"] - - # sample_msg = noti_obj(sample_json_payload, sample_txt, sample_html, "test_sample") - - - pluginsState = handle_run(param, db, pluginsState) - - - # if testType == 'Email': - # send_email(sample_msg) - # elif testType == 'Webhooks': - # send_webhook (sample_msg) - # elif testType == 'Apprise': - # send_apprise (sample_msg) - # elif testType == 'NTFY': - # send_ntfy (sample_msg) - # elif testType == 'PUSHSAFER': - # send_pushsafer (sample_msg) - # else: - # mylog('none', ['[Test Publishers] No test matches: ', testType]) - - # mylog('minimal', ['[Test Publishers] END Test: ', testType]) + mylog('minimal', ['[Test] END Test: ', runType]) return pluginsState diff --git a/pialert/plugin_utils.py b/pialert/plugin_utils.py index 258f76bc..fc914353 100755 --- a/pialert/plugin_utils.py +++ b/pialert/plugin_utils.py @@ -77,11 +77,8 @@ def list_to_csv(arr): arrayItemStr = '' mylog('debug', '[Plugins] Flattening the below array') - mylog('debug', arr) - - - mylog('debug', f'[Plugins] isinstance(arr, list) : {isinstance(arr, list)}') - mylog('debug', f'[Plugins] isinstance(arr, str) : {isinstance(arr, str)}') + mylog('debug', arr) + mylog('debug', f'[Plugins] isinstance(arr, list) : {isinstance(arr, list)} | isinstance(arr, str) : {isinstance(arr, str)}') if isinstance(arr, str): return arr.replace('[','').replace(']','').replace("'", '') # removing brackets and single quotes (not allowed) @@ -172,18 +169,6 @@ def get_plugins_configs(): return pluginsList # Return the list of plugin configurations - -#------------------------------------------------------------------------------- -# Gets the setting value -def get_plugin_setting_value(plugin, function_key): - - resultObj = get_plugin_setting(plugin, function_key) - - if resultObj != None: - return resultObj["value"] - - return None - #------------------------------------------------------------------------------- def custom_plugin_decoder(pluginDict): return namedtuple('X', pluginDict.keys())(*pluginDict.values()) diff --git a/pialert/reporting.py b/pialert/reporting.py index 3ed380a8..35e63803 100755 --- a/pialert/reporting.py +++ b/pialert/reporting.py @@ -192,7 +192,7 @@ def get_notifications (db): # collect "new_devices" for the webhook json json_new_devices = notiStruc.json["data"] - mail_text = mail_text.replace ('', notiStruc.text + '\n') + mail_text = mail_text.replace ('', notiStruc.text + '\n') mail_html = mail_html.replace ('', notiStruc.html) mylog('verbose', ['[Notification] New Devices sections done.']) @@ -208,7 +208,7 @@ def get_notifications (db): # collect "down_devices" for the webhook json json_down_devices = notiStruc.json["data"] - mail_text = mail_text.replace ('', notiStruc.text + '\n') + mail_text = mail_text.replace ('', notiStruc.text + '\n') mail_html = mail_html.replace ('', notiStruc.html) mylog('verbose', ['[Notification] Down Devices sections done.']) @@ -225,7 +225,7 @@ def get_notifications (db): # collect "events" for the webhook json json_events = notiStruc.json["data"] - mail_text = mail_text.replace ('', notiStruc.text + '\n') + mail_text = mail_text.replace ('', notiStruc.text + '\n') mail_html = mail_html.replace ('', notiStruc.html) mylog('verbose', ['[Notification] Events sections done.']) @@ -256,7 +256,7 @@ def get_notifications (db): final_text = removeDuplicateNewLines(mail_text) # Create clickable MAC links - final_html = generate_mac_links (mail_html, deviceUrl) + final_html = generate_mac_links (mail_html, deviceUrl) # Write output emails for debug write_file (logPath + '/report_output.json', json.dumps(final_json)) @@ -265,12 +265,7 @@ def get_notifications (db): return noti_obj(final_json, final_text, final_html) - # # Notify is something to report - # if hasNotifications: - - # mylog('none', ['[Notification] Changes detected, sending reports']) - - # msg = noti_obj(json_final, mail_text, mail_html) + # mylog('minimal', ['[Notification] Udating API files']) # send_api() @@ -281,12 +276,8 @@ def get_notifications (db): # send_email (msg ) # else : # mylog('verbose', ['[Notification] Skip email']) - # if conf.REPORT_APPRISE and check_config('apprise'): - # updateState("Send: Apprise") - # mylog('minimal', ['[Notification] Sending report by Apprise']) - # send_apprise (msg) - # else : - # mylog('verbose', ['[Notification] Skip Apprise']) + # + # if conf.REPORT_WEBHOOK and check_config('webhook'): # updateState("Send: Webhook") # mylog('minimal', ['[Notification] Sending report by Webhook'])