From eb7b7b57abca32f5b8f378a455cf1f56767138be Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Fri, 6 Oct 2023 22:53:15 +1100 Subject: [PATCH] Notification rework v0.2 --- pialert/__main__.py | 23 ++++- pialert/api.py | 6 +- pialert/const.py | 1 + pialert/helper.py | 11 +-- pialert/notifications.py | 88 +++++++++++++++++ pialert/reporting.py | 209 ++++++++++++++------------------------- 6 files changed, 188 insertions(+), 150 deletions(-) create mode 100644 pialert/notifications.py diff --git a/pialert/__main__.py b/pialert/__main__.py index ae59a7d7..fe7d5221 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -24,12 +24,13 @@ import multiprocessing import conf from const import * from logger import mylog -from helper import filePermissions, timeNowTZ, updateState, get_setting_value +from helper import filePermissions, timeNowTZ, updateState, get_setting_value, noti_struc from api import update_api from networkscan import process_scan from initialise import importConfigs from database import DB, get_all_devices -from reporting import send_notifications +from reporting import get_notifications +from notifications import Notifications from plugin import run_plugin_scripts, check_and_run_user_event @@ -146,8 +147,24 @@ def main (): # run all plugins registered to be run when new devices are found pluginsState = run_plugin_scripts(db, 'on_new_device', pluginsState) + # Notification handling + # ---------------------------------------- + # send all configured notifications - send_notifications(db) + notiStructure = get_notifications(db) + + # Write the notifications into the DB + notification = Notifications(db) + + # mylog('debug', f"[MAIN] notiStructure.text: {notiStructure.text} ") + # mylog('debug', f"[MAIN] notiStructure.json: {notiStructure.json} ") + # mylog('debug', f"[MAIN] notiStructure.html: {notiStructure.html} ") + + hasNotifications = notification.create(notiStructure.json, notiStructure.text, notiStructure.html, "") + + if hasNotifications: + pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) + # Commit SQL db.commitDB() diff --git a/pialert/api.py b/pialert/api.py index 13f21da1..58721fab 100755 --- a/pialert/api.py +++ b/pialert/api.py @@ -3,8 +3,7 @@ import json # pialert modules import conf -from const import (apiPath, sql_devices_all, sql_events_pending_alert, - sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings) +from const import (apiPath, sql_devices_all, sql_events_pending_alert, sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings, sql_notifications_all) from logger import mylog from helper import write_file @@ -19,8 +18,6 @@ def update_api(db, isNotification = False, updateOnlyDataSources = []): folder = apiPath - # update notifications moved to reporting send_api() - # Save plugins write_file(folder + 'plugins.json' , json.dumps({"data" : conf.plugins})) @@ -33,6 +30,7 @@ def update_api(db, isNotification = False, updateOnlyDataSources = []): ["plugins_history", sql_plugins_history], ["plugins_objects", sql_plugins_objects], ["plugins_language_strings", sql_language_strings], + ["notifications", sql_notifications_all], ["custom_endpoint", conf.API_CUSTOM_SQL], ] diff --git a/pialert/const.py b/pialert/const.py index c5eb0771..cd6daef5 100755 --- a/pialert/const.py +++ b/pialert/const.py @@ -36,6 +36,7 @@ sql_events_pending_alert = "SELECT * FROM Events where eve_PendingAlertEmail is sql_settings = "SELECT * FROM Settings" sql_plugins_objects = "SELECT * FROM Plugins_Objects" sql_language_strings = "SELECT * FROM Plugins_Language_Strings" +sql_notifications_all = "SELECT * FROM Notifications" sql_plugins_events = "SELECT * FROM Plugins_Events" sql_plugins_history = "SELECT * FROM Plugins_History ORDER BY DateTimeChanged DESC" sql_new_devices = """SELECT * FROM ( diff --git a/pialert/helper.py b/pialert/helper.py index 676bc52b..712635b4 100755 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -620,17 +620,8 @@ class json_struc: #------------------------------------------------------------------------------- class noti_struc: - def __init__(self, json, text, html, notificationType): + def __init__(self, json, text, html): self.json = json self.text = text self.html = html - # jsonFile = apiPath + f'/notifications_{notificationType}.json' - - # mylog('debug', [f"[Notifications] Writing {jsonFile}"]) - - # if notificationType != '': - - # # Update .json file - # with open(jsonFile, 'w') as jsonFile: - # json.dump(self, jsonFile, cls=NotiStrucEncoder, indent=4) diff --git a/pialert/notifications.py b/pialert/notifications.py new file mode 100644 index 00000000..875d0591 --- /dev/null +++ b/pialert/notifications.py @@ -0,0 +1,88 @@ +import datetime +import json +import uuid + +# PiAlert modules +import conf +import const +from const import pialertPath, logPath, apiPath +from logger import logResult, mylog, print_log +from helper import timeNowTZ + +#------------------------------------------------------------------------------- +# Notification object handling +#------------------------------------------------------------------------------- +class Notifications: + def __init__(self, db): + self.db = db + + # Create Notifications table if missing + self.db.sql.execute("""CREATE TABLE IF NOT EXISTS "Notifications" ( + "Index" INTEGER, + "GUID" TEXT UNIQUE, + "DateTimeCreated" TEXT, + "DateTimePushed" TEXT, + "Status" TEXT, + "JSON" TEXT, + "Text" TEXT, + "HTML" TEXT, + "PublishedVia" TEXT, + "Extra" TEXT, + PRIMARY KEY("Index" AUTOINCREMENT) + ); + """) + + self.save() + + # Create a new DB entry if new notiifcations available, otherwise skip + def create(self, JSON, Text, HTML, Extra=""): + + # Check if empty JSON + # _json = json.loads(JSON) + # Check if nothing to report + if JSON["internet"] == [] and JSON["new_devices"] == [] and JSON["down_devices"] == [] and JSON["events"] == [] and JSON["plugins"] == []: + self.HasNotifications = False + # end if nothing to report + return self.HasNotifications + + # continue and save into DB if notifications available + self.HasNotifications = True + + self.GUID = str(uuid.uuid4()) + self.DateTimeCreated = timeNowTZ() + self.DateTimePushed = "" + self.Status = "new" + self.JSON = JSON + self.Text = Text + self.HTML = HTML + self.PublishedVia = "" + self.Extra = Extra + + self.upsert() + + return self.HasNotifications + + # Only updates the status + def updateStatus(self, newStatus): + self.Status = newStatus + self.upsert() + + # Updates the Published properties + def updatePublishedVia(self, newPublishedVia): + self.PublishedVia = newPublishedVia + self.DateTimePushed = timeNowTZ() + self.upsert() + + # TODO Index vs hash to minimize SQL calls, finish CRUD operations, expose via API, use API in plugins + + def upsert(self): + self.db.sql.execute(""" + INSERT OR REPLACE INTO Notifications (GUID, DateTimeCreated, DateTimePushed, Status, JSON, Text, HTML, PublishedVia, Extra) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (self.GUID, self.DateTimeCreated, self.DateTimePushed, self.Status, json.dumps(self.JSON), self.Text, self.HTML, self.PublishedVia, self.Extra)) + + self.save() + + def save(self): + # Commit changes + self.db.commitDB() \ No newline at end of file diff --git a/pialert/reporting.py b/pialert/reporting.py index 0129c425..7157535d 100755 --- a/pialert/reporting.py +++ b/pialert/reporting.py @@ -12,9 +12,7 @@ import datetime import json - import socket - import subprocess import requests from json2table import convert @@ -50,10 +48,10 @@ json_final = [] #------------------------------------------------------------------------------- -def construct_notifications(db, sqlQuery, tableTitle, skipText = False, suppliedJsonStruct = None, notificationType=''): +def construct_notifications(db, sqlQuery, tableTitle, skipText = False, suppliedJsonStruct = None): if suppliedJsonStruct is None and sqlQuery == "": - return noti_struc("", "", "", notificationType) + return noti_struc("", "", "") table_attributes = {"style" : "border-collapse: collapse; font-size: 12px; color:#70707", "width" : "100%", "cellspacing" : 0, "cellpadding" : "3px", "bordercolor" : "#C0C0C0", "border":"1"} headerProps = "width='120px' style='color:white; font-size: 16px;' bgcolor='#64a0d6' " @@ -97,7 +95,7 @@ def construct_notifications(db, sqlQuery, tableTitle, skipText = False, supplied for header in headers: html = format_table(html, header, thProps) - notiStruc = noti_struc(jsn, text, html, notificationType) + notiStruc = noti_struc(jsn, text, html) if not notiStruc.json['data'] and not notiStruc.text and not notiStruc.html: @@ -108,7 +106,7 @@ def construct_notifications(db, sqlQuery, tableTitle, skipText = False, supplied return notiStruc -def send_notifications (db): +def get_notifications (db): sql = db.sql #TO-DO global mail_text, mail_html, json_final, partial_html, partial_txt, partial_json @@ -184,13 +182,12 @@ def send_notifications (db): if 'new_devices' in conf.INCLUDED_SECTIONS : # Compose New Devices Section - sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments, dev_Vendor as "Device Vendor" - FROM Events_Devices + sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments FROM Events_Devices WHERE eve_PendingAlertEmail = 1 AND eve_EventType = 'New Device' ORDER BY eve_DateTime""" - notiStruc = construct_notifications(db, sqlQuery, "New devices", "new_devices") + notiStruc = construct_notifications(db, sqlQuery, "New devices") # collect "new_devices" for the webhook json json_new_devices = notiStruc.json["data"] @@ -201,13 +198,12 @@ def send_notifications (db): if 'down_devices' in conf.INCLUDED_SECTIONS : # Compose Devices Down Section - sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments, dev_Vendor as "Device Vendor" - FROM Events_Devices + sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments FROM Events_Devices WHERE eve_PendingAlertEmail = 1 AND eve_EventType = 'Device Down' ORDER BY eve_DateTime""" - notiStruc = construct_notifications(db, sqlQuery, "Down devices", "down_Devices") + notiStruc = construct_notifications(db, sqlQuery, "Down devices") # collect "down_devices" for the webhook json json_down_devices = notiStruc.json["data"] @@ -218,14 +214,13 @@ def send_notifications (db): if 'events' in conf.INCLUDED_SECTIONS : # Compose Events Section - sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments, dev_Vendor as "Device Vendor" - FROM Events_Devices + sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments FROM Events_Devices WHERE eve_PendingAlertEmail = 1 AND eve_EventType IN ('Connected','Disconnected', 'IP Changed') ORDER BY eve_DateTime""" - notiStruc = construct_notifications(db, sqlQuery, "Events", "events") + notiStruc = construct_notifications(db, sqlQuery, "Events") # collect "events" for the webhook json json_events = notiStruc.json["data"] @@ -250,94 +245,92 @@ def send_notifications (db): plugins_report = len(json_plugins) > 0 mylog('verbose', ['[Notification] Plugins sections done.']) - json_final = { + final_json = { "internet": json_internet, "new_devices": json_new_devices, "down_devices": json_down_devices, - "events": json_events, - "ports": json_ports, + "events": json_events, "plugins": json_plugins, } - mail_text = removeDuplicateNewLines(mail_text) + final_text = removeDuplicateNewLines(mail_text) # Create clickable MAC links - mail_html = generate_mac_links (mail_html, deviceUrl) + final_html = generate_mac_links (mail_html, deviceUrl) # Write output emails for debug - write_file (logPath + '/report_output.json', json.dumps(json_final)) - write_file (logPath + '/report_output.txt', mail_text) - write_file (logPath + '/report_output.html', mail_html) + write_file (logPath + '/report_output.json', json.dumps(final_json)) + write_file (logPath + '/report_output.txt', final_text) + write_file (logPath + '/report_output.html', final_html) - # Write the notifications into the DB - # TODO + return noti_struc(final_json, final_text, final_html) - # Notify is something to report - if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or json_ports != [] or plugins_report: + # # Notify is something to report + # if hasNotifications: - mylog('none', ['[Notification] Changes detected, sending reports']) + # mylog('none', ['[Notification] Changes detected, sending reports']) - msg = noti_struc(json_final, mail_text, mail_html, 'master') + # msg = noti_struc(json_final, mail_text, mail_html) - mylog('minimal', ['[Notification] Udating API files']) - send_api() + # mylog('minimal', ['[Notification] Udating API files']) + # send_api() - if conf.REPORT_MAIL and check_config('email'): - updateState("Send: Email") - mylog('minimal', ['[Notification] Sending report by Email']) - send_email (msg ) - else : - mylog('verbose', ['[Notification] Skip email']) - if conf.REPORT_APPRISE and check_config('apprise'): - updateState("Send: Apprise") - mylog('minimal', ['[Notification] Sending report by Apprise']) - send_apprise (msg) - else : - mylog('verbose', ['[Notification] Skip Apprise']) - if conf.REPORT_WEBHOOK and check_config('webhook'): - updateState("Send: Webhook") - mylog('minimal', ['[Notification] Sending report by Webhook']) - send_webhook (msg) - else : - mylog('verbose', ['[Notification] Skip webhook']) - if conf.REPORT_NTFY and check_config('ntfy'): - updateState("Send: NTFY") - mylog('minimal', ['[Notification] Sending report by NTFY']) - send_ntfy (msg) - else : - mylog('verbose', ['[Notification] Skip NTFY']) - if conf.REPORT_PUSHSAFER and check_config('pushsafer'): - updateState("Send: PUSHSAFER") - mylog('minimal', ['[Notification] Sending report by PUSHSAFER']) - send_pushsafer (msg) - else : - mylog('verbose', ['[Notification] Skip PUSHSAFER']) - # Update MQTT entities - if conf.REPORT_MQTT and check_config('mqtt'): - updateState("Send: MQTT") - mylog('minimal', ['[Notification] Establishing MQTT thread']) - mqtt_start(db) - else : - mylog('verbose', ['[Notification] Skip MQTT']) - else : - mylog('verbose', ['[Notification] No changes to report']) + # if conf.REPORT_MAIL and check_config('email'): + # updateState("Send: Email") + # mylog('minimal', ['[Notification] Sending report by Email']) + # send_email (msg ) + # else : + # mylog('verbose', ['[Notification] Skip email']) + # if conf.REPORT_APPRISE and check_config('apprise'): + # updateState("Send: Apprise") + # mylog('minimal', ['[Notification] Sending report by Apprise']) + # send_apprise (msg) + # else : + # mylog('verbose', ['[Notification] Skip Apprise']) + # if conf.REPORT_WEBHOOK and check_config('webhook'): + # updateState("Send: Webhook") + # mylog('minimal', ['[Notification] Sending report by Webhook']) + # send_webhook (msg) + # else : + # mylog('verbose', ['[Notification] Skip webhook']) + # if conf.REPORT_NTFY and check_config('ntfy'): + # updateState("Send: NTFY") + # mylog('minimal', ['[Notification] Sending report by NTFY']) + # send_ntfy (msg) + # else : + # mylog('verbose', ['[Notification] Skip NTFY']) + # if conf.REPORT_PUSHSAFER and check_config('pushsafer'): + # updateState("Send: PUSHSAFER") + # mylog('minimal', ['[Notification] Sending report by PUSHSAFER']) + # send_pushsafer (msg) + # else : + # mylog('verbose', ['[Notification] Skip PUSHSAFER']) + # # Update MQTT entities + # if conf.REPORT_MQTT and check_config('mqtt'): + # updateState("Send: MQTT") + # mylog('minimal', ['[Notification] Establishing MQTT thread']) + # mqtt_start(db) + # else : + # mylog('verbose', ['[Notification] Skip MQTT']) + # else : + # mylog('verbose', ['[Notification] No changes to report']) - # Clean Pending Alert Events - sql.execute ("""UPDATE Devices SET dev_LastNotification = ? - WHERE dev_MAC IN (SELECT eve_MAC FROM Events - WHERE eve_PendingAlertEmail = 1) - """, (datetime.datetime.now(conf.tz),) ) - sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1""") + # # Clean Pending Alert Events + # sql.execute ("""UPDATE Devices SET dev_LastNotification = ? + # WHERE dev_MAC IN (SELECT eve_MAC FROM Events + # WHERE eve_PendingAlertEmail = 1) + # """, (datetime.datetime.now(conf.tz),) ) + # sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 + # WHERE eve_PendingAlertEmail = 1""") - # clear plugin events - sql.execute ("DELETE FROM Plugins_Events") + # # clear plugin events + # sql.execute ("DELETE FROM Plugins_Events") - # DEBUG - print number of rows updated - mylog('minimal', ['[Notification] Notifications changes: ', sql.rowcount]) + # # DEBUG - print number of rows updated + # mylog('minimal', ['[Notification] Notifications changes: ', sql.rowcount]) - # Commit changes - db.commitDB() + # # Commit changes + # db.commitDB() #------------------------------------------------------------------------------- @@ -438,56 +431,6 @@ def skip_repeated_notifications (db): db.commitDB() -#------------------------------------------------------------------------------- -# Notification object handling -#------------------------------------------------------------------------------- -class Notifications: - def __init__(self, db): - - self.db = db - - # Create Notifications table if missing - self.db.sql.execute("""CREATE TABLE IF NOT EXISTS "Notifications" ( - "Index" INTEGER, - "DateTimeCreated" TEXT, - "DateTimePushed" TEXT, - "Status" TEXT, - "JSON" TEXT, - "Text" TEXT, - "HTML" TEXT, - "PublishedVia" TEXT, - "Extra" TEXT, - PRIMARY KEY("Index" AUTOINCREMENT) - ); - """) - - self.save() - - def create(self, JSON, Text, HTML, Extra): - self.JSON = JSON - self.Text = Text - self.HTML = HTML - self.Extra = Extra - self.Status = "new" - - # TODO Init values that can be auto initialized - # TODO Check for nulls - # TODO Index vs hash to minimize SQL calls, finish CRUD operations, expose via API, use API in plugins - - # current_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - # self.db.sql.execute(""" - # INSERT INTO Notifications (DateTimeCreated, DateTimePushed, Status, JSON, Text, HTML, PublishedVia, Extra) - # VALUES (?, ?, ?, ?, ?, ?, ?, ?) - # """, (current_time, DateTimePushed, Status, JSON, Text, HTML, PublishedVia, Extra)) - - - - def save(self): - - # Commit changes - self.db.commitDB() - -