From a71f16ee37a3fb6b68858974e55899ce370c4779 Mon Sep 17 00:00:00 2001 From: Data-Monkey Date: Sat, 27 May 2023 22:38:12 +1000 Subject: [PATCH] changed to __main__.py and scanners folder --- Dockerfile | 8 +- dockerfiles/start.sh | 22 +- pialert/{pialert.py => __main__.py} | 65 ++++-- pialert/api.py | 16 +- pialert/database.py | 325 ++++----------------------- pialert/device.py | 32 +-- pialert/helper.py | 8 +- pialert/initialise.py | 8 +- pialert/networkscan.py | 19 +- pialert/plugin.py | 4 +- pialert/reporting.py | 48 ++-- pialert/{ => scanners}/arpscan.py | 6 +- pialert/{ => scanners}/internet.py | 36 +-- pialert/{ => scanners}/nmapscan.py | 0 pialert/{ => scanners}/pholusscan.py | 18 +- pialert/{ => scanners}/pihole.py | 45 +++- 16 files changed, 230 insertions(+), 430 deletions(-) rename pialert/{pialert.py => __main__.py} (85%) rename pialert/{ => scanners}/arpscan.py (84%) rename pialert/{ => scanners}/internet.py (81%) rename pialert/{ => scanners}/nmapscan.py (100%) rename pialert/{ => scanners}/pholusscan.py (90%) rename pialert/{ => scanners}/pihole.py (61%) diff --git a/Dockerfile b/Dockerfile index 43259427..0004f2ec 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM debian:bullseye-slim # default UID and GID -ENV USER=pi USER_ID=1000 USER_GID=1000 TZ=Europe/London PORT=20211 +ENV USER=pi USER_ID=1000 USER_GID=1000 PORT=20211 +#TZ=Europe/London # Todo, figure out why using a workdir instead of full paths don't work # Todo, do we still need all these packages? I can already see sudo which isn't needed @@ -46,3 +47,8 @@ RUN rm /etc/nginx/sites-available/default \ ENTRYPOINT ["tini", "--"] CMD ["/home/pi/pialert/dockerfiles/start.sh"] + + + + +## command to build docker: DOCKER_BUILDKIT=1 docker build . --iidfile dockerID \ No newline at end of file diff --git a/dockerfiles/start.sh b/dockerfiles/start.sh index 4ff77756..72c09e9b 100755 --- a/dockerfiles/start.sh +++ b/dockerfiles/start.sh @@ -1,15 +1,15 @@ #!/bin/sh /home/pi/pialert/dockerfiles/user-mapping.sh -# if custom variables not set we do not need to do anything -if [ -n "${TZ}" ]; then - FILECONF=/home/pi/pialert/config/pialert.conf - if [ -f "$FILECONF" ]; then - sed -ie "s|Europe/Berlin|${TZ}|g" /home/pi/pialert/config/pialert.conf - else - sed -ie "s|Europe/Berlin|${TZ}|g" /home/pi/pialert/back/pialert.conf_bak - fi -fi +# # if custom variables not set we do not need to do anything +# if [ -n "${TZ}" ]; then +# FILECONF=/home/pi/pialert/config/pialert.conf +# if [ -f "$FILECONF" ]; then +# sed -ie "s|Europe/Berlin|${TZ}|g" /home/pi/pialert/config/pialert.conf +# else +# sed -ie "s|Europe/Berlin|${TZ}|g" /home/pi/pialert/back/pialert.conf_bak +# fi +# fi if [ -n "${PORT}" ]; then sed -ie 's/listen 20211/listen '${PORT}'/g' /etc/nginx/sites-available/default @@ -29,5 +29,5 @@ chmod -R a+rw /home/pi/pialert/config # cron -f #python /home/pi/pialert/back/pialert.py -echo "DATA MONKEY VERSION ..." -python /home/pi/pialert/pialert/pialert.py +echo "[DEBUG] DATA MONKEY VERSION ..." +python /home/pi/pialert/pialert/ diff --git a/pialert/pialert.py b/pialert/__main__.py similarity index 85% rename from pialert/pialert.py rename to pialert/__main__.py index 9afaea84..7780d114 100755 --- a/pialert/pialert.py +++ b/pialert/__main__.py @@ -30,29 +30,53 @@ from api import update_api from networkscan import scan_network from initialise import importConfigs from mac_vendor import update_devices_MAC_vendors -from database import DB, get_all_devices, upgradeDB, sql_new_devices +from database import DB, get_all_devices, sql_new_devices from reporting import check_and_run_event, send_notifications from plugin import run_plugin_scripts # different scanners -from pholusscan import performPholusScan -from nmapscan import performNmapScan -from internet import check_internet_IP - +from scanners.pholusscan import performPholusScan +from scanners.nmapscan import performNmapScan +from scanners.internet import check_internet_IP # Global variables changedPorts_json_struc = None - - #=============================================================================== #=============================================================================== # MAIN #=============================================================================== #=============================================================================== +""" +main structure of Pi Alert + + Initialise All + start Loop forever + initialise loop + (re)import config + (re)import plugin config + run plugins (once) + run frontend events + update API + run scans + run plugins (scheduled) + check internet IP + check vendor + run PHOLUS + run NMAP + run "scan_network()" + ARP Scan + PiHole copy db + PiHole DHCP leases + processing scan results + run plugins (after Scan) + reporting + cleanup + end loop +""" def main (): - + conf.time_started = datetime.datetime.now() conf.cycle = "" conf.check_report = [1, "internet_IP", "update_vendors_silent"] @@ -102,7 +126,7 @@ def main (): # update time started time_started = datetime.datetime.now() # not sure why we need this ... loop_start_time = timeNow() - mylog('debug', ['[ +++++++ ', timeNow(), '] [MAIN] Stating loop']) + mylog('debug', '[MAIN] Stating loop') # re-load user configuration and plugins importConfigs(db) @@ -153,8 +177,8 @@ def main (): if last_update_vendors + datetime.timedelta(days = 7) < time_started: last_update_vendors = time_started conf.cycle = 'update_vendors' - mylog('verbose', ['[', timeNow(), '] cycle:',conf.cycle]) - update_devices_MAC_vendors() + mylog('verbose', ['[MAIN] cycle:',conf.cycle]) + update_devices_MAC_vendors(db) # Execute scheduled or one-off Pholus scan if enabled and run conditions fulfilled if conf.PHOLUS_RUN == "schedule" or conf.PHOLUS_RUN == "once": @@ -183,7 +207,7 @@ def main (): run = False # run once after application starts - if conf.NMAP_RUN == "once" and conf.nmapSchedule.last_run == 0: + if conf.NMAP_RUN == "once" and nmapSchedule.last_run == 0: run = True # run if overdue scheduled time @@ -191,14 +215,14 @@ def main (): run = nmapSchedule.runScheduleCheck() if run: - conf.nmapSchedule.last_run = timeNow() + nmapSchedule.last_run = timeNow() performNmapScan(db, get_all_devices(db)) # Perform a network scan via arp-scan or pihole if last_network_scan + datetime.timedelta(minutes=conf.SCAN_CYCLE_MINUTES) < time_started: last_network_scan = time_started conf.cycle = 1 # network scan - mylog('verbose', ['[', timeNow(), '] cycle:',conf.cycle]) + mylog('verbose', ['[MAIN] cycle:',conf.cycle]) updateState(db,"Scan: Network") # scan_network() @@ -214,8 +238,7 @@ def main (): # If thread is still active if p.is_alive(): - print("DEBUG scan_network running too long - let\'s kill it") - mylog('info', [' DEBUG scan_network running too long - let\'s kill it']) + mylog('none', "[MAIN] scan_network running too long - let\'s kill it") # Terminate - may not work if process is stuck for good p.terminate() @@ -254,7 +277,7 @@ def main (): if last_cleanup + datetime.timedelta(hours = 24) < time_started: last_cleanup = time_started conf.cycle = 'cleanup' - mylog('verbose', ['[', timeNow(), '] cycle:',conf.cycle]) + mylog('verbose', ['[MAIN] cycle:',conf.cycle]) db.cleanup_database(startTime, conf.DAYS_TO_KEEP_EVENTS, conf.PHOLUS_DAYS_DATA) # Commit SQL @@ -265,17 +288,17 @@ def main (): action = str(conf.cycle) if action == "1": action = "network_scan" - mylog('verbose', ['[', timeNow(), '] Last action: ', action]) + mylog('verbose', ['[MAIN] Last action: ', action]) conf.cycle = "" - mylog('verbose', ['[', timeNow(), '] cycle:',conf.cycle]) + mylog('verbose', ['[MAIN] cycle:',conf.cycle]) # Footer updateState(db,"Process: Wait") - mylog('verbose', ['[', timeNow(), '] Process: Wait']) + mylog('verbose', ['[MAIN] Process: Wait']) else: # do something conf.cycle = "" - mylog('verbose', ['[', timeNow(), '] [MAIN] waiting to start next loop']) + mylog('verbose', ['[MAIN] waiting to start next loop']) #loop time.sleep(5) # wait for N seconds diff --git a/pialert/api.py b/pialert/api.py index bb87ffa4..2dd2e966 100644 --- a/pialert/api.py +++ b/pialert/api.py @@ -2,11 +2,11 @@ import json # pialert modules +import conf from const import pialertPath from logger import mylog from files import write_file from database import * -from conf import ENABLE_PLUGINS, API_CUSTOM_SQL apiEndpoints = [] @@ -14,7 +14,7 @@ apiEndpoints = [] # API #=============================================================================== def update_api(isNotification = False, updateOnlyDataSources = []): - mylog('verbose', [' [API] Update API not ding anything for now !']) + mylog('verbose', [' [API] Update API not doing anything for now !']) return folder = pialertPath + '/front/api/' @@ -28,7 +28,7 @@ def update_api(isNotification = False, updateOnlyDataSources = []): write_file(folder + 'notification_json_final.json' , json.dumps(json_final)) # Save plugins - if ENABLE_PLUGINS: + if conf.ENABLE_PLUGINS: write_file(folder + 'plugins.json' , json.dumps({"data" : plugins})) # prepare database tables we want to expose @@ -42,7 +42,7 @@ def update_api(isNotification = False, updateOnlyDataSources = []): ["plugins_history", sql_plugins_history], ["plugins_objects", sql_plugins_objects], ["language_strings", sql_language_strings], - ["custom_endpoint", API_CUSTOM_SQL], + ["custom_endpoint", conf.API_CUSTOM_SQL], ] # Save selected database tables @@ -57,12 +57,12 @@ def update_api(isNotification = False, updateOnlyDataSources = []): class api_endpoint_class: - def __init__(self, sql, path): + def __init__(self, db, path): global apiEndpoints - - self.sql = sql - self.jsonData = get_table_as_json(sql).json + self.db = db + self.sql = db.sql + self.jsonData = db.get_table_as_json( self.sql).json self.path = path self.fileName = path.split('/')[-1] self.hash = hash(json.dumps(self.jsonData)) diff --git a/pialert/database.py b/pialert/database.py index 06857214..71b5d64c 100644 --- a/pialert/database.py +++ b/pialert/database.py @@ -48,7 +48,7 @@ class DB(): mylog('debug','openDB: databse already open') return - mylog('none', 'Opening DB' ) + mylog('none', '[Database] Opening DB' ) # Open DB and Cursor self.sql_connection = sqlite3.connect (fullDbPath, isolation_level=None) self.sql_connection.execute('pragma journal_mode=wal') # @@ -60,12 +60,11 @@ class DB(): def commitDB (self): if self.sql_connection == None : mylog('debug','commitDB: databse is not open') - return - - # mylog('debug','commitDB: comiting DB changes') + return False # Commit changes to DB self.sql_connection.commit() + return True #------------------------------------------------------------------------------- def get_sql_array(self, query): @@ -75,7 +74,7 @@ class DB(): self.sql.execute(query) rows = self.sql.fetchall() - self.commitDB() + #self.commitDB() # convert result into list of lists arr = [] @@ -386,293 +385,48 @@ class DB(): self.commitDB() -#------------------------------------------------------------------------------- -def get_table_as_json(db, sqlQuery): + #------------------------------------------------------------------------------- + def get_table_as_json(self, sqlQuery): - db.sql.execute(sqlQuery) + self.sql.execute(sqlQuery) - columnNames = list(map(lambda x: x[0], db.sql.description)) + columnNames = list(map(lambda x: x[0], self.sql.description)) - rows = db.sql.fetchall() + rows = self.sql.fetchall() - result = {"data":[]} + result = {"data":[]} - for row in rows: - tmp = row_to_json(columnNames, row) - result["data"].append(tmp) - return json_struc(result, columnNames) + for row in rows: + tmp = row_to_json(columnNames, row) + result["data"].append(tmp) + return json_struc(result, columnNames) - - - - - - -#------------------------------------------------------------------------------- -def upgradeDB(db: DB()): - sql = db.sql #TO-DO - - # indicates, if Online_History table is available - onlineHistoryAvailable = db.sql.execute(""" - SELECT name FROM sqlite_master WHERE type='table' - AND name='Online_History'; - """).fetchall() != [] - - # Check if it is incompatible (Check if table has all required columns) - isIncompatible = False - - if onlineHistoryAvailable : - isIncompatible = sql.execute (""" - SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Online_History') WHERE name='Archived_Devices' - """).fetchone()[0] == 0 - - # Drop table if available, but incompatible - if onlineHistoryAvailable and isIncompatible: - mylog('none','[upgradeDB] Table is incompatible, Dropping the Online_History table') - sql.execute("DROP TABLE Online_History;") - onlineHistoryAvailable = False - - if onlineHistoryAvailable == False : - sql.execute(""" - CREATE TABLE "Online_History" ( - "Index" INTEGER, - "Scan_Date" TEXT, - "Online_Devices" INTEGER, - "Down_Devices" INTEGER, - "All_Devices" INTEGER, - "Archived_Devices" INTEGER, - PRIMARY KEY("Index" AUTOINCREMENT) - ); - """) - - # Alter Devices table - # dev_Network_Node_MAC_ADDR column - dev_Network_Node_MAC_ADDR_missing = sql.execute (""" - SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_MAC_ADDR' - """).fetchone()[0] == 0 - - if dev_Network_Node_MAC_ADDR_missing : - mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_MAC_ADDR to the Devices table"]) - sql.execute(""" - ALTER TABLE "Devices" ADD "dev_Network_Node_MAC_ADDR" TEXT - """) - - # dev_Network_Node_port column - dev_Network_Node_port_missing = sql.execute (""" - SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_port' - """).fetchone()[0] == 0 - - if dev_Network_Node_port_missing : - mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_port to the Devices table"]) - sql.execute(""" - ALTER TABLE "Devices" ADD "dev_Network_Node_port" INTEGER - """) - - # dev_Icon column - dev_Icon_missing = sql.execute (""" - SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Icon' - """).fetchone()[0] == 0 - - if dev_Icon_missing : - mylog('verbose', ["[upgradeDB] Adding dev_Icon to the Devices table"]) - sql.execute(""" - ALTER TABLE "Devices" ADD "dev_Icon" TEXT - """) - - # indicates, if Settings table is available - settingsMissing = sql.execute(""" - SELECT name FROM sqlite_master WHERE type='table' - AND name='Settings'; - """).fetchone() == None - - # Re-creating Settings table - mylog('verbose', ["[upgradeDB] Re-creating Settings table"]) - - if settingsMissing == False: - sql.execute("DROP TABLE Settings;") - - sql.execute(""" - CREATE TABLE "Settings" ( - "Code_Name" TEXT, - "Display_Name" TEXT, - "Description" TEXT, - "Type" TEXT, - "Options" TEXT, - "RegEx" TEXT, - "Value" TEXT, - "Group" TEXT, - "Events" TEXT - ); - """) - - # indicates, if Pholus_Scan table is available - pholusScanMissing = sql.execute(""" - SELECT name FROM sqlite_master WHERE type='table' - AND name='Pholus_Scan'; - """).fetchone() == None - - # if pholusScanMissing == False: - # # Re-creating Pholus_Scan table - # sql.execute("DROP TABLE Pholus_Scan;") - # pholusScanMissing = True - - if pholusScanMissing: - mylog('verbose', ["[upgradeDB] Re-creating Pholus_Scan table"]) - sql.execute(""" - CREATE TABLE "Pholus_Scan" ( - "Index" INTEGER, - "Info" TEXT, - "Time" TEXT, - "MAC" TEXT, - "IP_v4_or_v6" TEXT, - "Record_Type" TEXT, - "Value" TEXT, - "Extra" TEXT, - PRIMARY KEY("Index" AUTOINCREMENT) - ); - """) - - # indicates, if Nmap_Scan table is available - nmapScanMissing = sql.execute(""" - SELECT name FROM sqlite_master WHERE type='table' - AND name='Nmap_Scan'; - """).fetchone() == None - - # Re-creating Parameters table - mylog('verbose', ["[upgradeDB] Re-creating Parameters table"]) - sql.execute("DROP TABLE Parameters;") - - sql.execute(""" - CREATE TABLE "Parameters" ( - "par_ID" TEXT PRIMARY KEY, - "par_Value" TEXT - ); - """) - - # Initialize Parameters if unavailable - initOrSetParam(db, 'Back_App_State','Initializing') - - # if nmapScanMissing == False: - # # Re-creating Nmap_Scan table - # sql.execute("DROP TABLE Nmap_Scan;") - # nmapScanMissing = True - - if nmapScanMissing: - mylog('verbose', ["[upgradeDB] Re-creating Nmap_Scan table"]) - sql.execute(""" - CREATE TABLE "Nmap_Scan" ( - "Index" INTEGER, - "MAC" TEXT, - "Port" TEXT, - "Time" TEXT, - "State" TEXT, - "Service" TEXT, - "Extra" TEXT, - PRIMARY KEY("Index" AUTOINCREMENT) - ); - """) - - # Plugin state - sql_Plugins_Objects = """ CREATE TABLE IF NOT EXISTS Plugins_Objects( - "Index" INTEGER, - Plugin TEXT NOT NULL, - Object_PrimaryID TEXT NOT NULL, - Object_SecondaryID TEXT NOT NULL, - DateTimeCreated TEXT NOT NULL, - DateTimeChanged TEXT NOT NULL, - Watched_Value1 TEXT NOT NULL, - Watched_Value2 TEXT NOT NULL, - Watched_Value3 TEXT NOT NULL, - Watched_Value4 TEXT NOT NULL, - Status TEXT NOT NULL, - Extra TEXT NOT NULL, - UserData TEXT NOT NULL, - ForeignKey TEXT NOT NULL, - PRIMARY KEY("Index" AUTOINCREMENT) - ); """ - sql.execute(sql_Plugins_Objects) - - # Plugin execution results - sql_Plugins_Events = """ CREATE TABLE IF NOT EXISTS Plugins_Events( - "Index" INTEGER, - Plugin TEXT NOT NULL, - Object_PrimaryID TEXT NOT NULL, - Object_SecondaryID TEXT NOT NULL, - DateTimeCreated TEXT NOT NULL, - DateTimeChanged TEXT NOT NULL, - Watched_Value1 TEXT NOT NULL, - Watched_Value2 TEXT NOT NULL, - Watched_Value3 TEXT NOT NULL, - Watched_Value4 TEXT NOT NULL, - Status TEXT NOT NULL, - Extra TEXT NOT NULL, - UserData TEXT NOT NULL, - ForeignKey TEXT NOT NULL, - PRIMARY KEY("Index" AUTOINCREMENT) - ); """ - sql.execute(sql_Plugins_Events) - - # Plugin execution history - sql_Plugins_History = """ CREATE TABLE IF NOT EXISTS Plugins_History( - "Index" INTEGER, - Plugin TEXT NOT NULL, - Object_PrimaryID TEXT NOT NULL, - Object_SecondaryID TEXT NOT NULL, - DateTimeCreated TEXT NOT NULL, - DateTimeChanged TEXT NOT NULL, - Watched_Value1 TEXT NOT NULL, - Watched_Value2 TEXT NOT NULL, - Watched_Value3 TEXT NOT NULL, - Watched_Value4 TEXT NOT NULL, - Status TEXT NOT NULL, - Extra TEXT NOT NULL, - UserData TEXT NOT NULL, - ForeignKey TEXT NOT NULL, - PRIMARY KEY("Index" AUTOINCREMENT) - ); """ - sql.execute(sql_Plugins_History) - - # Dynamically generated language strings - # indicates, if Language_Strings table is available - languageStringsMissing = sql.execute(""" - SELECT name FROM sqlite_master WHERE type='table' - AND name='Plugins_Language_Strings'; - """).fetchone() == None - - if languageStringsMissing == False: - sql.execute("DROP TABLE Plugins_Language_Strings;") - - sql.execute(""" CREATE TABLE IF NOT EXISTS Plugins_Language_Strings( - "Index" INTEGER, - Language_Code TEXT NOT NULL, - String_Key TEXT NOT NULL, - String_Value TEXT NOT NULL, - Extra TEXT NOT NULL, - PRIMARY KEY("Index" AUTOINCREMENT) - ); """) - - db.commitDB() + #------------------------------------------------------------------------------- + # referece from here: https://codereview.stackexchange.com/questions/241043/interface-class-for-sqlite-databases + #------------------------------------------------------------------------------- + def read(self, query, *args): + """check the query and arguments are aligned and are read only""" + mylog('debug',[ '[Database] - SELECT Query: ', query, " params: ", args]) + try: + assert query.count('?') == len(args) + assert query.upper().strip().startswith('SELECT') + self.sql.execute(query, args) + rows = self.sql.fetchall() + return rows + except AssertionError: + mylog('none',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) + except sqlite3.Error as e: + mylog('none',[ '[Database] - SQL ERROR: ', e]) + return None #------------------------------------------------------------------------------- def get_device_stats(db): - sql = db.sql #TO-DO # columns = ["online","down","all","archived","new","unknown"] - sql.execute(sql_devices_stats) - - row = sql.fetchone() - db.commitDB() - - return row + return db.read(sql_devices_stats) #------------------------------------------------------------------------------- def get_all_devices(db): - sql = db.sql #TO-DO - sql.execute(sql_devices_all) - - row = sql.fetchall() - - db.commitDB() - return row + return db.read(sql_devices_all) #------------------------------------------------------------------------------- @@ -681,20 +435,17 @@ def insertOnlineHistory(db, cycle): sql = db.sql #TO-DO startTime = timeNow() # Add to History - sql.execute("SELECT * FROM Devices") - History_All = sql.fetchall() + + History_All = db.read("SELECT * FROM Devices") History_All_Devices = len(History_All) - sql.execute("SELECT * FROM Devices WHERE dev_Archived = 1") - History_Archived = sql.fetchall() + History_Archived = db.read("SELECT * FROM Devices WHERE dev_Archived = 1") History_Archived_Devices = len(History_Archived) - sql.execute("""SELECT * FROM CurrentScan WHERE cur_ScanCycle = ? """, (cycle,)) - History_Online = sql.fetchall() + History_Online = db.read("SELECT * FROM CurrentScan WHERE cur_ScanCycle = ? ", cycle) History_Online_Devices = len(History_Online) History_Offline_Devices = History_All_Devices - History_Archived_Devices - History_Online_Devices sql.execute ("INSERT INTO Online_History (Scan_Date, Online_Devices, Down_Devices, All_Devices, Archived_Devices) "+ "VALUES ( ?, ?, ?, ?, ?)", (startTime, History_Online_Devices, History_Offline_Devices, History_All_Devices, History_Archived_Devices ) ) - db.commitDB() - + db.commitDB() \ No newline at end of file diff --git a/pialert/device.py b/pialert/device.py index e22ab380..a29d9ffc 100644 --- a/pialert/device.py +++ b/pialert/device.py @@ -6,10 +6,10 @@ import subprocess import conf from helper import timeNow -from internet import check_IP_format, get_internet_IP +from scanners.internet import check_IP_format, get_internet_IP from logger import mylog, print_log from mac_vendor import query_MAC_vendor -from pholusscan import performPholusScan, resolve_device_name_dig, resolve_device_name_pholus +from scanners.pholusscan import performPholusScan, resolve_device_name_dig, resolve_device_name_pholus #------------------------------------------------------------------------------- @@ -64,7 +64,7 @@ def save_scanned_devices (db, p_arpscan_devices, p_cycle_interval): local_ip_cmd = ["ip -o route get 1 | sed 's/^.*src \\([^ ]*\\).*$/\\1/;q'"] local_ip = subprocess.Popen (local_ip_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip() - mylog('debug', [' Saving this IP into the CurrentScan table:', local_ip]) + mylog('debug', ['[Save Devices] Saving this IP into the CurrentScan table:', local_ip]) if check_IP_format(local_ip) == '': local_ip = '0.0.0.0' @@ -82,19 +82,19 @@ def print_scan_stats (db): sql.execute ("""SELECT COUNT(*) FROM CurrentScan WHERE cur_ScanCycle = ? """, (conf.cycle,)) - mylog('verbose', [' Devices Detected.......: ', str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] Devices Detected.......: ', str (sql.fetchone()[0]) ]) # Devices arp-scan sql.execute ("""SELECT COUNT(*) FROM CurrentScan WHERE cur_ScanMethod='arp-scan' AND cur_ScanCycle = ? """, (conf.cycle,)) - mylog('verbose', [' arp-scan detected..: ', str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] arp-scan detected..: ', str (sql.fetchone()[0]) ]) # Devices Pi-hole sql.execute ("""SELECT COUNT(*) FROM CurrentScan WHERE cur_ScanMethod='PiHole' AND cur_ScanCycle = ? """, (conf.cycle,)) - mylog('verbose', [' Pi-hole detected...: +' + str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] Pi-hole detected...: +' + str (sql.fetchone()[0]) ]) # New Devices sql.execute ("""SELECT COUNT(*) FROM CurrentScan @@ -102,7 +102,7 @@ def print_scan_stats (db): AND NOT EXISTS (SELECT 1 FROM Devices WHERE dev_MAC = cur_MAC) """, (conf.cycle,)) - mylog('verbose', [' New Devices........: ' + str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] New Devices........: ' + str (sql.fetchone()[0]) ]) # Devices in this ScanCycle sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan @@ -110,7 +110,7 @@ def print_scan_stats (db): AND dev_ScanCycle = ? """, (conf.cycle,)) - mylog('verbose', [' Devices in this cycle..: ' + str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] Devices in this cycle..: ' + str (sql.fetchone()[0]) ]) # Down Alerts sql.execute ("""SELECT COUNT(*) FROM Devices @@ -120,7 +120,7 @@ def print_scan_stats (db): WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle) """, (conf.cycle,)) - mylog('verbose', [' Down Alerts........: ' + str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] Down Alerts........: ' + str (sql.fetchone()[0]) ]) # New Down Alerts sql.execute ("""SELECT COUNT(*) FROM Devices @@ -131,7 +131,7 @@ def print_scan_stats (db): WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle) """, (conf.cycle,)) - mylog('verbose', [' New Down Alerts....: ' + str (sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] New Down Alerts....: ' + str (sql.fetchone()[0]) ]) # New Connections sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan @@ -139,7 +139,7 @@ def print_scan_stats (db): AND dev_PresentLastScan = 0 AND dev_ScanCycle = ? """, (conf.cycle,)) - mylog('verbose', [' New Connections....: ' + str ( sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] New Connections....: ' + str ( sql.fetchone()[0]) ]) # Disconnections sql.execute ("""SELECT COUNT(*) FROM Devices @@ -149,7 +149,7 @@ def print_scan_stats (db): WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle) """, (conf.cycle,)) - mylog('verbose', [' Disconnections.....: ' + str ( sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] Disconnections.....: ' + str ( sql.fetchone()[0]) ]) # IP Changes sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan @@ -157,7 +157,7 @@ def print_scan_stats (db): AND dev_ScanCycle = ? AND dev_LastIP <> cur_IP """, (conf.cycle,)) - mylog('verbose', [' IP Changes.........: ' + str ( sql.fetchone()[0]) ]) + mylog('verbose', ['[Scan Stats] IP Changes.........: ' + str ( sql.fetchone()[0]) ]) @@ -395,7 +395,7 @@ def update_devices_names (db): db.commitDB() # Number of entries from previous Pholus scans - mylog('verbose', '[Update Device Name] Pholus entries from prev scans: ', len(pholusResults)) + mylog('verbose', ['[Update Device Name] Pholus entries from prev scans: ', len(pholusResults)]) for device in unknownDevices: newName = -1 @@ -422,8 +422,8 @@ def update_devices_names (db): recordsToUpdate.append ([newName, device['dev_MAC']]) # Print log - mylog('verbose', '[Update Device Name] Names Found (DiG/Pholus): ', len(recordsToUpdate), " (",foundDig,"/",foundPholus ,")" ) - mylog('verbose', '[Update Device Name] Names Not Found : ', len(recordsNotFound) ) + mylog('verbose', ['[Update Device Name] Names Found (DiG/Pholus): ', len(recordsToUpdate), " (",foundDig,"/",foundPholus ,")"] ) + mylog('verbose', ['[Update Device Name] Names Not Found : ', len(recordsNotFound)] ) # update not found devices with (name not found) sql.executemany ("UPDATE Devices SET dev_Name = ? WHERE dev_MAC = ? ", recordsNotFound ) diff --git a/pialert/helper.py b/pialert/helper.py index 85906df7..d33c5c97 100644 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -32,14 +32,12 @@ def updateState(db, newState): #sql = db.sql - mylog('debug', ' [updateState] changing state to: "' + newState +'"') + mylog('debug', '[updateState] changing state to: "' + newState +'"') db.sql.execute ("UPDATE Parameters SET par_Value='"+ newState +"' WHERE par_ID='Back_App_State'") db.commitDB() #------------------------------------------------------------------------------- def updateSubnets(scan_subnets): - - # remove old list subnets = [] # multiple interfaces @@ -64,8 +62,8 @@ def checkPermissionsOK(): dbR_access = (os.access(fullDbPath, os.R_OK)) dbW_access = (os.access(fullDbPath, os.W_OK)) - - mylog('none', ['\n Permissions check (All should be True)']) + mylog('none', ['\n']) + mylog('none', ['Permissions check (All should be True)']) mylog('none', ['------------------------------------------------']) mylog('none', [ " " , confPath , " | " , " READ | " , confR_access]) mylog('none', [ " " , confPath , " | " , " WRITE | " , confW_access]) diff --git a/pialert/initialise.py b/pialert/initialise.py index d7417cef..9e858d35 100644 --- a/pialert/initialise.py +++ b/pialert/initialise.py @@ -175,13 +175,12 @@ def importConfigs (db): if conf.ENABLE_PLUGINS: conf.plugins = get_plugins_configs() - mylog('none', ['[', timeNow(), '] Plugins: Number of dynamically loaded plugins: ', len(conf.plugins)]) + mylog('none', ['[Config] Plugins: Number of dynamically loaded plugins: ', len(conf.plugins)]) # handle plugins for plugin in conf.plugins: + pref = plugin["unique_prefix"] print_plugin_info(plugin, ['display_name','description']) - - pref = plugin["unique_prefix"] # if plugin["enabled"] == 'true': @@ -232,5 +231,4 @@ def importConfigs (db): # update_api(False, ["settings"]) # TO DO this creates a circular reference between API and HELPER ! - mylog('info', ['[', timeNow(), '] Config: Imported new config']) - + mylog('info', '[Config] Imported new config') diff --git a/pialert/networkscan.py b/pialert/networkscan.py index ac4dfbac..423f88b7 100644 --- a/pialert/networkscan.py +++ b/pialert/networkscan.py @@ -1,12 +1,12 @@ import conf -from arpscan import execute_arpscan +from scanners.arpscan import execute_arpscan +from scanners.pihole import copy_pihole_network, read_DHCP_leases from database import insertOnlineHistory from device import create_new_devices, print_scan_stats, save_scanned_devices, update_devices_data_from_scan, update_devices_names from helper import timeNow -from logger import mylog, print_log -from pihole import copy_pihole_network, read_DHCP_leases +from logger import mylog from reporting import skip_repeated_notifications @@ -18,19 +18,20 @@ from reporting import skip_repeated_notifications def scan_network (db): sql = db.sql #TO-DO - reporting = False + # Header # moved updateState to main loop # updateState(db,"Scan: Network") - mylog('verbose', ['[', timeNow(), '] Scan Devices:' ]) + mylog('verbose', ['[Network Scan] Scan Devices:' ]) # Query ScanCycle properties scanCycle_data = query_ScanCycle_Data (db, True) if scanCycle_data is None: - mylog('none', ['\n*************** ERROR ***************']) - mylog('none', ['ScanCycle %s not found' % conf.cycle ]) - mylog('none', [' Exiting...\n']) + mylog('none', ['\n']) + mylog('none', ['[Network Scan]*************** ERROR ***************']) + mylog('none', ['[Network Scan] ScanCycle %s not found' % conf.cycle ]) + mylog('none', ['[Network Scan] Exiting...\n']) return False db.commitDB() @@ -57,6 +58,7 @@ def scan_network (db): read_DHCP_leases (db) db.commitDB() + # Load current scan data mylog('verbose','[Network Scan] Processing scan results') save_scanned_devices (db, arpscan_devices, cycle_interval) @@ -111,7 +113,6 @@ def scan_network (db): # if ENABLE_PLUGINS: # run_plugin_scripts(db,'always_after_scan') - return reporting #------------------------------------------------------------------------------- def query_ScanCycle_Data (db, pOpenCloseDB = False, cycle = 1): diff --git a/pialert/plugin.py b/pialert/plugin.py index ec0904aa..647464ae 100644 --- a/pialert/plugin.py +++ b/pialert/plugin.py @@ -21,7 +21,7 @@ def timeNow(): #------------------------------------------------------------------------------- -def run_plugin_scripts(db, runType): +def run_plugin_scripts(db, runType, plugins = conf.plugins): # global plugins, tz, mySchedules @@ -30,7 +30,7 @@ def run_plugin_scripts(db, runType): mylog('debug', [' [Plugins] Check if any plugins need to be executed on run type: ', runType]) - for plugin in conf.plugins: + for plugin in plugins: shouldRun = False diff --git a/pialert/reporting.py b/pialert/reporting.py index 8ee72af2..d12ec74c 100644 --- a/pialert/reporting.py +++ b/pialert/reporting.py @@ -13,7 +13,7 @@ from json2table import convert # pialert modules import conf from const import pialertPath, logPath -from database import get_table_as_json +#from database import get_table_as_json from files import get_file_content, write_file from helper import generate_mac_links, isNewVersion, removeDuplicateNewLines, timeNow, hide_email, json_struc, updateState from logger import logResult, mylog, print_log @@ -52,7 +52,7 @@ def construct_notifications(db, sqlQuery, tableTitle, skipText = False, supplied text_line = '{}\t{}\n' if suppliedJsonStruct is None: - json_struc = get_table_as_json(db, sqlQuery) + json_struc = db.get_table_as_json(sqlQuery) else: json_struc = suppliedJsonStruct @@ -96,7 +96,7 @@ def send_notifications (db): plugins_report = False # Reporting section - mylog('verbose', [' Check if something to report']) + mylog('verbose', ['[Notification] Check if something to report']) # prepare variables for JSON construction json_internet = [] @@ -254,47 +254,47 @@ def send_notifications (db): # update_api(True) # TO-DO - mylog('none', [' Changes detected, sending reports']) + mylog('none', ['[Notification] Changes detected, sending reports']) if conf.REPORT_MAIL and check_config('email'): updateState(db,"Send: Email") - mylog('info', [' Sending report by Email']) + mylog('info', ['[Notification] Sending report by Email']) send_email (mail_text, mail_html) else : - mylog('verbose', [' Skip email']) + mylog('verbose', ['[Notification] Skip email']) if conf.REPORT_APPRISE and check_config('apprise'): updateState(db,"Send: Apprise") - mylog('info', [' Sending report by Apprise']) + mylog('info', ['[Notification] Sending report by Apprise']) send_apprise (mail_html, mail_text) else : - mylog('verbose', [' Skip Apprise']) + mylog('verbose', ['[Notification] Skip Apprise']) if conf.REPORT_WEBHOOK and check_config('webhook'): updateState(db,"Send: Webhook") - mylog('info', [' Sending report by Webhook']) + mylog('info', ['[Notification] Sending report by Webhook']) send_webhook (json_final, mail_text) else : - mylog('verbose', [' Skip webhook']) + mylog('verbose', ['[Notification] Skip webhook']) if conf.REPORT_NTFY and check_config('ntfy'): updateState(db,"Send: NTFY") - mylog('info', [' Sending report by NTFY']) + mylog('info', ['[Notification] Sending report by NTFY']) send_ntfy (mail_text) else : - mylog('verbose', [' Skip NTFY']) + mylog('verbose', ['[Notification] Skip NTFY']) if conf.REPORT_PUSHSAFER and check_config('pushsafer'): updateState(db,"Send: PUSHSAFER") - mylog('info', [' Sending report by PUSHSAFER']) + mylog('info', ['[Notification] Sending report by PUSHSAFER']) send_pushsafer (mail_text) else : - mylog('verbose', [' Skip PUSHSAFER']) + mylog('verbose', ['[Notification] Skip PUSHSAFER']) # Update MQTT entities if conf.REPORT_MQTT and check_config('mqtt'): updateState(db,"Send: MQTT") - mylog('info', [' Establishing MQTT thread']) + mylog('info', ['[Notification] Establishing MQTT thread']) mqtt_start() else : - mylog('verbose', [' Skip MQTT']) + mylog('verbose', ['[Notification] Skip MQTT']) else : - mylog('verbose', [' No changes to report']) + mylog('verbose', ['[Notification] No changes to report']) # Clean Pending Alert Events sql.execute ("""UPDATE Devices SET dev_LastNotification = ? @@ -310,7 +310,7 @@ def send_notifications (db): changedPorts_json_struc = None # DEBUG - print number of rows updated - mylog('info', ['[', timeNow(), '] Notifications: ', sql.rowcount]) + mylog('info', ['[Notification] Notifications changes: ', sql.rowcount]) # Commit changes db.commitDB() @@ -321,42 +321,42 @@ def check_config(service): if service == 'email': if conf.SMTP_SERVER == '' or conf.REPORT_FROM == '' or conf.REPORT_TO == '': - mylog('none', [' Error: Email service not set up correctly. Check your pialert.conf SMTP_*, REPORT_FROM and REPORT_TO variables.']) + mylog('none', ['[Check Config] Error: Email service not set up correctly. Check your pialert.conf SMTP_*, REPORT_FROM and REPORT_TO variables.']) return False else: return True if service == 'apprise': if conf.APPRISE_URL == '' or conf.APPRISE_HOST == '': - mylog('none', [' Error: Apprise service not set up correctly. Check your pialert.conf APPRISE_* variables.']) + mylog('none', ['[Check Config] Error: Apprise service not set up correctly. Check your pialert.conf APPRISE_* variables.']) return False else: return True if service == 'webhook': if conf.WEBHOOK_URL == '': - mylog('none', [' Error: Webhook service not set up correctly. Check your pialert.conf WEBHOOK_* variables.']) + mylog('none', ['[Check Config] Error: Webhook service not set up correctly. Check your pialert.conf WEBHOOK_* variables.']) return False else: return True if service == 'ntfy': if conf.NTFY_HOST == '' or conf.NTFY_TOPIC == '': - mylog('none', [' Error: NTFY service not set up correctly. Check your pialert.conf NTFY_* variables.']) + mylog('none', ['[Check Config] Error: NTFY service not set up correctly. Check your pialert.conf NTFY_* variables.']) return False else: return True if service == 'pushsafer': if conf.PUSHSAFER_TOKEN == 'ApiKey': - mylog('none', [' Error: Pushsafer service not set up correctly. Check your pialert.conf PUSHSAFER_TOKEN variable.']) + mylog('none', ['[Check Config] Error: Pushsafer service not set up correctly. Check your pialert.conf PUSHSAFER_TOKEN variable.']) return False else: return True if service == 'mqtt': if conf.MQTT_BROKER == '' or conf.MQTT_PORT == '' or conf.MQTT_USER == '' or conf.MQTT_PASSWORD == '': - mylog('none', [' Error: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.']) + mylog('none', ['[Check Config] Error: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.']) return False else: return True diff --git a/pialert/arpscan.py b/pialert/scanners/arpscan.py similarity index 84% rename from pialert/arpscan.py rename to pialert/scanners/arpscan.py index 54af2213..e3077a24 100644 --- a/pialert/arpscan.py +++ b/pialert/scanners/arpscan.py @@ -3,7 +3,6 @@ import subprocess from logger import mylog - #------------------------------------------------------------------------------- def execute_arpscan (userSubnets): @@ -34,6 +33,7 @@ def execute_arpscan (userSubnets): unique_devices.append(device) # return list + mylog('debug', ['[ARP Scan] Completed found ', len(unique_devices) ,' devices ' ]) return unique_devices #------------------------------------------------------------------------------- @@ -41,6 +41,7 @@ def execute_arpscan_on_interface (interface): # Prepare command arguments subnets = interface.strip().split() # Retry is 6 to avoid false offline devices + mylog('debug', ['[ARP Scan] - arpscan command: sudo arp-scan --ignoredups --retry=6 ', str(subnets)]) arpscan_args = ['sudo', 'arp-scan', '--ignoredups', '--retry=6'] + subnets # Execute command @@ -49,7 +50,8 @@ def execute_arpscan_on_interface (interface): result = subprocess.check_output (arpscan_args, universal_newlines=True) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [e.output]) + mylog('none', ['[ARP Scan]', e.output]) result = "" + mylog('debug', ['[ARP Scan] on Interface Completed with results: ', result]) return result diff --git a/pialert/internet.py b/pialert/scanners/internet.py similarity index 81% rename from pialert/internet.py rename to pialert/scanners/internet.py index cd68f881..d309a5c7 100644 --- a/pialert/internet.py +++ b/pialert/scanners/internet.py @@ -23,53 +23,53 @@ def check_internet_IP ( db ): # Header updateState(db,"Scan: Internet IP") - mylog('verbose', ['[', timeNow(), '] Check Internet IP:']) + mylog('verbose', ['[Internet IP] Check Internet IP started']) # Get Internet IP - mylog('verbose', [' Retrieving Internet IP:']) + mylog('verbose', ['[Internet IP] - Retrieving Internet IP']) internet_IP = get_internet_IP(conf.DIG_GET_IP_ARG) # TESTING - Force IP # internet_IP = "1.2.3.4" # Check result = IP if internet_IP == "" : - mylog('none', [' Error retrieving Internet IP']) - mylog('none', [' Exiting...']) + mylog('none', ['[Internet IP] Error retrieving Internet IP']) + mylog('none', ['[Internet IP] Exiting...']) return False - mylog('verbose', [' ', internet_IP]) + mylog('verbose', ['[Internet IP] IP: ', internet_IP]) # Get previous stored IP - mylog('verbose', [' Retrieving previous IP:']) + mylog('verbose', ['[Internet IP] Retrieving previous IP:']) previous_IP = get_previous_internet_IP (db) - mylog('verbose', [' ', previous_IP]) + mylog('verbose', ['[Internet IP] ', previous_IP]) # Check IP Change if internet_IP != previous_IP : - mylog('info', [' New internet IP: ', internet_IP]) + mylog('info', ['[Internet IP] New internet IP: ', internet_IP]) save_new_internet_IP (db, internet_IP) else : - mylog('verbose', [' No changes to perform']) + mylog('verbose', ['[Internet IP] No changes to perform']) # Get Dynamic DNS IP if conf.DDNS_ACTIVE : - mylog('verbose', [' Retrieving Dynamic DNS IP']) + mylog('verbose', ['[DDNS] Retrieving Dynamic DNS IP']) dns_IP = get_dynamic_DNS_IP() # Check Dynamic DNS IP if dns_IP == "" or dns_IP == "0.0.0.0" : - mylog('info', [' Error retrieving Dynamic DNS IP']) - mylog('info', [' ', dns_IP]) + mylog('none', ['[DDNS] Error retrieving Dynamic DNS IP']) + mylog('none', ['[DDNS] ', dns_IP]) # Check DNS Change if dns_IP != internet_IP : - mylog('info', [' Updating Dynamic DNS IP']) + mylog('none', ['[DDNS] Updating Dynamic DNS IP']) message = set_dynamic_DNS_IP () - mylog('info', [' ', message]) + mylog('none', ['[DDNS] ', message]) else : - mylog('verbose', [' No changes to perform']) + mylog('verbose', ['[DDNS] No changes to perform']) else : - mylog('verbose', [' Skipping Dynamic DNS update']) + mylog('verbose', ['[DDNS] Skipping Dynamic DNS update']) @@ -164,7 +164,7 @@ def get_dynamic_DNS_IP (): dig_output = subprocess.check_output (dig_args, universal_newlines=True) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [e.output]) + mylog('none', ['[DDNS] ERROR - ', e.output]) dig_output = '' # probably no internet # Check result is an IP @@ -189,7 +189,7 @@ def set_dynamic_DNS_IP (): universal_newlines=True) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [e.output]) + mylog('none', ['[DDNS] ERROR - ',e.output]) curl_output = "" return curl_output diff --git a/pialert/nmapscan.py b/pialert/scanners/nmapscan.py similarity index 100% rename from pialert/nmapscan.py rename to pialert/scanners/nmapscan.py diff --git a/pialert/pholusscan.py b/pialert/scanners/pholusscan.py similarity index 90% rename from pialert/pholusscan.py rename to pialert/scanners/pholusscan.py index 6c6f79ec..fe4c9dfe 100644 --- a/pialert/pholusscan.py +++ b/pialert/scanners/pholusscan.py @@ -15,7 +15,7 @@ def performPholusScan (db, timeoutSec, userSubnets): temp = subnet.split("--interface=") if len(temp) != 2: - mylog('none', [" Skip scan (need subnet in format '192.168.1.0/24 --inteface=eth0'), got: ", subnet]) + mylog('none', ["[PholusScan] Skip scan (need subnet in format '192.168.1.0/24 --inteface=eth0'), got: ", subnet]) return mask = temp[0].strip() @@ -23,8 +23,8 @@ def performPholusScan (db, timeoutSec, userSubnets): # logging & updating app state updateState(db,"Scan: Pholus") - mylog('info', ['[', timeNow(), '] Scan: Pholus for ', str(timeoutSec), 's ('+ str(round(int(timeoutSec) / 60, 1)) +'min)']) - mylog('verbose', [" Pholus scan on [interface] ", interface, " [mask] " , mask]) + mylog('none', ['[PholusScan] Scan: Pholus for ', str(timeoutSec), 's ('+ str(round(int(timeoutSec) / 60, 1)) +'min)']) + mylog('verbose', ["[PholusScan] Pholus scan on [interface] ", interface, " [mask] " , mask]) # the scan always lasts 2x as long, so the desired user time from settings needs to be halved adjustedTimeout = str(round(int(timeoutSec) / 2, 0)) @@ -40,15 +40,15 @@ def performPholusScan (db, timeoutSec, userSubnets): output = subprocess.check_output (pholus_args, universal_newlines=True, stderr=subprocess.STDOUT, timeout=(timeoutSec + 30)) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [e.output]) - mylog('none', [" Error - Pholus Scan - check logs"]) + mylog('none', [[PholusScan], e.output]) + mylog('none', ["[PholusScan] Error - Pholus Scan - check logs"]) except subprocess.TimeoutExpired as timeErr: - mylog('none', [' Pholus TIMEOUT - the process forcefully terminated as timeout reached']) + mylog('none', ['[PholusScan] Pholus TIMEOUT - the process forcefully terminated as timeout reached']) if output == "": # check if the subprocess failed - mylog('none', ['[', timeNow(), '] Scan: Pholus FAIL - check logs']) + mylog('none', ['[PholusScan] Scan: Pholus FAIL - check logs']) else: - mylog('verbose', ['[', timeNow(), '] Scan: Pholus SUCCESS']) + mylog('verbose', ['[PholusScan] Scan: Pholus SUCCESS']) # check the last run output f = open(logPath + '/pialert_pholus_lastrun.log', 'r+') @@ -176,7 +176,7 @@ def resolve_device_name_dig (pMAC, pIP): newName = subprocess.check_output (dig_args, universal_newlines=True) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [e.output]) + mylog('none', ['[device_name_dig] ', e.output]) # newName = "Error - check logs" return -1 diff --git a/pialert/pihole.py b/pialert/scanners/pihole.py similarity index 61% rename from pialert/pihole.py rename to pialert/scanners/pihole.py index b85efd28..63ee3ba8 100644 --- a/pialert/pihole.py +++ b/pialert/scanners/pihole.py @@ -1,6 +1,10 @@ """ module to import db and leases from PiHole """ +import sqlite3 + +import conf from const import piholeDB, piholeDhcpleases +from logger import mylog #------------------------------------------------------------------------------- def copy_pihole_network (db): @@ -10,11 +14,20 @@ def copy_pihole_network (db): sql = db.sql # TO-DO # Open Pi-hole DB - sql.execute ("ATTACH DATABASE '"+ piholeDB +"' AS PH") + mylog('debug', '[PiHole Network] - attach PiHole DB') + + try: + sql.execute ("ATTACH DATABASE '"+ piholeDB +"' AS PH") + except sqlite3.Error as e: + mylog('none',[ '[PiHole Network] - SQL ERROR: ', e]) + # Copy Pi-hole Network table - sql.execute ("DELETE FROM PiHole_Network") - sql.execute ("""INSERT INTO PiHole_Network (PH_MAC, PH_Vendor, PH_LastQuery, + + try: + sql.execute ("DELETE FROM PiHole_Network") + + sql.execute ("""INSERT INTO PiHole_Network (PH_MAC, PH_Vendor, PH_LastQuery, PH_Name, PH_IP) SELECT hwaddr, macVendor, lastQuery, (SELECT name FROM PH.network_addresses @@ -24,24 +37,29 @@ def copy_pihole_network (db): FROM PH.network WHERE hwaddr NOT LIKE 'ip-%' AND hwaddr <> '00:00:00:00:00:00' """) - sql.execute ("""UPDATE PiHole_Network SET PH_Name = '(unknown)' + sql.execute ("""UPDATE PiHole_Network SET PH_Name = '(unknown)' WHERE PH_Name IS NULL OR PH_Name = '' """) - # Close Pi-hole DB - sql.execute ("DETACH PH") - db.commit() + # Close Pi-hole DB + sql.execute ("DETACH PH") + except sqlite3.Error as e: + mylog('none',[ '[PiHole Network] - SQL ERROR: ', e]) + + db.commitDB() + + mylog('debug',[ '[PiHole Network] - completed - found ',sql.rowcount, ' devices']) return str(sql.rowcount) != "0" #------------------------------------------------------------------------------- def read_DHCP_leases (db): """ read the PiHole DHCP file and insert all records into the DHCP_Leases table. - """ - - sql = db.sql # TO-DO + """ + mylog('debug', '[PiHole DHCP] - read DHCP_Leases file') # Read DHCP Leases # Bugfix #1 - dhcp.leases: lines with different number of columns (5 col) data = [] + reporting = False with open(piholeDhcpleases, 'r') as f: for line in f: reporting = True @@ -50,8 +68,11 @@ def read_DHCP_leases (db): data.append (row) # Insert into PiAlert table - sql.executemany ("""INSERT INTO DHCP_Leases (DHCP_DateTime, DHCP_MAC, + db.sql.executemany ("""INSERT INTO DHCP_Leases (DHCP_DateTime, DHCP_MAC, DHCP_IP, DHCP_Name, DHCP_MAC2) VALUES (?, ?, ?, ?, ?) """, data) - db.commit() + db.commitDB() + + mylog('debug', ['[PiHole DHCP] - completed - added ',len(data), ' devices.']) + return reporting \ No newline at end of file