diff --git a/pialert/__main__.py b/pialert/__main__.py index 7780d114..095b0cce 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -27,10 +27,10 @@ from const import * from logger import mylog from helper import filePermissions, isNewVersion, timeNow, timeNowTZ, updateState from api import update_api -from networkscan import scan_network +from networkscan import process_scan, scan_network from initialise import importConfigs from mac_vendor import update_devices_MAC_vendors -from database import DB, get_all_devices, sql_new_devices +from database import DB, get_all_devices from reporting import check_and_run_event, send_notifications from plugin import run_plugin_scripts @@ -145,7 +145,7 @@ def main (): check_and_run_event(db) # Update API endpoints - update_api() + update_api(db) # proceed if 1 minute passed if last_scan_run + datetime.timedelta(minutes=1) < loop_start_time : @@ -252,6 +252,10 @@ def main (): if conf.ENABLE_PLUGINS: run_plugin_scripts(db,'always_after_scan') + # -------------------------------------------------- + # process all the scanned data into new devices + mylog('debug', "[MAIN] start processig scan results") + process_scan (db, conf.arpscan_devices ) # Reporting if conf.cycle in conf.check_report: diff --git a/pialert/api.py b/pialert/api.py index 2dd2e966..91549bf7 100644 --- a/pialert/api.py +++ b/pialert/api.py @@ -3,33 +3,27 @@ import json # pialert modules import conf -from const import pialertPath +from const import (apiPath, sql_devices_all, sql_nmap_scan_all, sql_pholus_scan_all, sql_events_pending_alert, + sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings) from logger import mylog -from files import write_file -from database import * +from helper import write_file apiEndpoints = [] #=============================================================================== # API #=============================================================================== -def update_api(isNotification = False, updateOnlyDataSources = []): - mylog('verbose', [' [API] Update API not doing anything for now !']) - return +def update_api(db, isNotification = False, updateOnlyDataSources = []): + mylog('verbose', ['[API] Update API starting']) + # return - folder = pialertPath + '/front/api/' + folder = apiPath - if isNotification: - # Update last notification alert in all formats - mylog('verbose', [' [API] Updating notification_* files in /front/api']) - - write_file(folder + 'notification_text.txt' , mail_text) - write_file(folder + 'notification_text.html' , mail_html) - write_file(folder + 'notification_json_final.json' , json.dumps(json_final)) + # update notifications moved to reporting send_api() # Save plugins if conf.ENABLE_PLUGINS: - write_file(folder + 'plugins.json' , json.dumps({"data" : plugins})) + write_file(folder + 'plugins.json' , json.dumps({"data" : conf.plugins})) # prepare database tables we want to expose dataSourcesSQLs = [ @@ -50,19 +44,19 @@ def update_api(isNotification = False, updateOnlyDataSources = []): if updateOnlyDataSources == [] or dsSQL[0] in updateOnlyDataSources: - api_endpoint_class(dsSQL[1], folder + 'table_' + dsSQL[0] + '.json') + api_endpoint_class(db, dsSQL[1], folder + 'table_' + dsSQL[0] + '.json') #------------------------------------------------------------------------------- class api_endpoint_class: - def __init__(self, db, path): + def __init__(self, db, query, path): global apiEndpoints self.db = db - self.sql = db.sql - self.jsonData = db.get_table_as_json( self.sql).json + self.query = query + self.jsonData = db.get_table_as_json(self.query).json self.path = path self.fileName = path.split('/')[-1] self.hash = hash(json.dumps(self.jsonData)) @@ -76,7 +70,7 @@ class api_endpoint_class: # search previous endpoint states to check if API needs updating for endpoint in apiEndpoints: # match sql and API endpoint path - if endpoint.sql == self.sql and endpoint.path == self.path: + if endpoint.query == self.query and endpoint.path == self.path: found = True if endpoint.hash != self.hash: changed = True @@ -87,7 +81,7 @@ class api_endpoint_class: # cehck if API endpoints have changed or if it's a new one if not found or changed: - mylog('verbose', [f' [API] Updating {self.fileName} file in /front/api']) + mylog('verbose', [f'[API] Updating {self.fileName} file in /front/api']) write_file(self.path, json.dumps(self.jsonData)) @@ -98,5 +92,5 @@ class api_endpoint_class: # update hash apiEndpoints[changedIndex].hash = self.hash else: - mylog('info', [f' [API] ERROR Updating {self.fileName}']) + mylog('info', [f'[API] ERROR Updating {self.fileName}']) diff --git a/pialert/conf.py b/pialert/conf.py index 69157e1d..74991966 100644 --- a/pialert/conf.py +++ b/pialert/conf.py @@ -16,7 +16,7 @@ newVersionAvailable = False time_started = '' check_report = [] log_timestamp = 0 - +arpscan_devices = [] # ACTUAL CONFIGRATION ITEMS set to defaults diff --git a/pialert/const.py b/pialert/const.py index ab720c0a..f57aca39 100644 --- a/pialert/const.py +++ b/pialert/const.py @@ -12,10 +12,36 @@ dbPath = '/db/pialert.db' pluginsPath = pialertPath + '/front/plugins' logPath = pialertPath + '/front/log' +apiPath = pialertPath + '/front/api/' fullConfPath = pialertPath + confPath fullDbPath = pialertPath + dbPath fullPholusPath = pialertPath+'/pholus/pholus3.py' + vendorsDB = '/usr/share/arp-scan/ieee-oui.txt' piholeDB = '/etc/pihole/pihole-FTL.db' -piholeDhcpleases = '/etc/pihole/dhcp.leases' \ No newline at end of file +piholeDhcpleases = '/etc/pihole/dhcp.leases' + + +#=============================================================================== +# SQL queries +#=============================================================================== +sql_devices_all = "select dev_MAC, dev_Name, dev_DeviceType, dev_Vendor, dev_Group, dev_FirstConnection, dev_LastConnection, dev_LastIP, dev_StaticIP, dev_PresentLastScan, dev_LastNotification, dev_NewDevice, dev_Network_Node_MAC_ADDR, dev_Network_Node_port, dev_Icon from Devices" +sql_devices_stats = "SELECT Online_Devices as online, Down_Devices as down, All_Devices as 'all', Archived_Devices as archived, (select count(*) from Devices a where dev_NewDevice = 1 ) as new, (select count(*) from Devices a where dev_Name = '(unknown)' or dev_Name = '(name not found)' ) as unknown from Online_History order by Scan_Date desc limit 1" +sql_nmap_scan_all = "SELECT * FROM Nmap_Scan" +sql_pholus_scan_all = "SELECT * FROM Pholus_Scan" +sql_events_pending_alert = "SELECT * FROM Events where eve_PendingAlertEmail is not 0" +sql_settings = "SELECT * FROM Settings" +sql_plugins_objects = "SELECT * FROM Plugins_Objects" +sql_language_strings = "SELECT * FROM Plugins_Language_Strings" +sql_plugins_events = "SELECT * FROM Plugins_Events" +sql_plugins_history = "SELECT * FROM Plugins_History ORDER BY 'Index' DESC" +sql_new_devices = """SELECT * FROM ( SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC FROM Events_Devices + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType = 'New Device' + ORDER BY eve_DateTime ) t1 + LEFT JOIN + ( + SELECT dev_Name, dev_MAC as dev_MAC_t2 FROM Devices + ) t2 + ON t1.dev_MAC = t2.dev_MAC_t2""" \ No newline at end of file diff --git a/pialert/database.py b/pialert/database.py index 71b5d64c..c40a3732 100644 --- a/pialert/database.py +++ b/pialert/database.py @@ -3,7 +3,7 @@ import sqlite3 # pialert modules -from const import fullDbPath +from const import fullDbPath, sql_devices_stats, sql_devices_all from logger import mylog from helper import json_struc, initOrSetParam, row_to_json, timeNow #, updateState @@ -11,28 +11,7 @@ from helper import json_struc, initOrSetParam, row_to_json, timeNow #, updateSta -#=============================================================================== -# SQL queries -#=============================================================================== -sql_devices_all = "select dev_MAC, dev_Name, dev_DeviceType, dev_Vendor, dev_Group, dev_FirstConnection, dev_LastConnection, dev_LastIP, dev_StaticIP, dev_PresentLastScan, dev_LastNotification, dev_NewDevice, dev_Network_Node_MAC_ADDR, dev_Network_Node_port, dev_Icon from Devices" -sql_devices_stats = "SELECT Online_Devices as online, Down_Devices as down, All_Devices as 'all', Archived_Devices as archived, (select count(*) from Devices a where dev_NewDevice = 1 ) as new, (select count(*) from Devices a where dev_Name = '(unknown)' or dev_Name = '(name not found)' ) as unknown from Online_History order by Scan_Date desc limit 1" -sql_nmap_scan_all = "SELECT * FROM Nmap_Scan" -sql_pholus_scan_all = "SELECT * FROM Pholus_Scan" -sql_events_pending_alert = "SELECT * FROM Events where eve_PendingAlertEmail is not 0" -sql_settings = "SELECT * FROM Settings" -sql_plugins_objects = "SELECT * FROM Plugins_Objects" -sql_language_strings = "SELECT * FROM Plugins_Language_Strings" -sql_plugins_events = "SELECT * FROM Plugins_Events" -sql_plugins_history = "SELECT * FROM Plugins_History ORDER BY 'Index' DESC" -sql_new_devices = """SELECT * FROM ( SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC FROM Events_Devices - WHERE eve_PendingAlertEmail = 1 - AND eve_EventType = 'New Device' - ORDER BY eve_DateTime ) t1 - LEFT JOIN - ( - SELECT dev_Name, dev_MAC as dev_MAC_t2 FROM Devices - ) t2 - ON t1.dev_MAC = t2.dev_MAC_t2""" + class DB(): diff --git a/pialert/files.py b/pialert/files.py deleted file mode 100644 index 55d33faa..00000000 --- a/pialert/files.py +++ /dev/null @@ -1,37 +0,0 @@ -import io -import sys - - -#------------------------------------------------------------------------------- -def write_file (pPath, pText): - # Write the text depending using the correct python version - if sys.version_info < (3, 0): - file = io.open (pPath , mode='w', encoding='utf-8') - file.write ( pText.decode('unicode_escape') ) - file.close() - else: - file = open (pPath, 'w', encoding='utf-8') - if pText is None: - pText = "" - file.write (pText) - file.close() - -#------------------------------------------------------------------------------- -def get_file_content(path): - - f = open(path, 'r') - content = f.read() - f.close() - - return content - -#------------------------------------------------------------------------------- -def read_config_file(filename): - """ - retuns dict on the config file key:value pairs - """ - # load the variables from pialert.conf - code = compile(filename.read_text(), filename.name, "exec") - confDict = {} # config dictionary - exec(code, {"__builtins__": {}}, confDict) - return confDict \ No newline at end of file diff --git a/pialert/helper.py b/pialert/helper.py index d33c5c97..e1d5881f 100644 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -1,5 +1,7 @@ """ Colection of generic functions to support Pi.Alert """ +import io +import sys import datetime import os import re @@ -14,7 +16,6 @@ import requests import conf from const import * from logger import mylog, logResult -# from api import update_api # to avoid circular reference @@ -297,3 +298,25 @@ class json_struc: +#------------------------------------------------------------------------------- +def get_file_content(path): + + f = open(path, 'r') + content = f.read() + f.close() + + return content + +#------------------------------------------------------------------------------- +def write_file (pPath, pText): + # Write the text depending using the correct python version + if sys.version_info < (3, 0): + file = io.open (pPath , mode='w', encoding='utf-8') + file.write ( pText.decode('unicode_escape') ) + file.close() + else: + file = open (pPath, 'w', encoding='utf-8') + if pText is None: + pText = "" + file.write (pText) + file.close() \ No newline at end of file diff --git a/pialert/initialise.py b/pialert/initialise.py index 9e858d35..38a6dd5b 100644 --- a/pialert/initialise.py +++ b/pialert/initialise.py @@ -10,7 +10,7 @@ import conf from const import * from helper import collect_lang_strings, timeNow, updateSubnets, initOrSetParam from logger import mylog -from files import read_config_file +from api import update_api from scheduler import schedule_class from plugin import get_plugins_configs, print_plugin_info @@ -148,9 +148,6 @@ def importConfigs (db): # API conf.API_CUSTOM_SQL = ccd('API_CUSTOM_SQL', 'SELECT * FROM Devices WHERE dev_PresentLastScan = 0' , c_d, 'Custom endpoint', 'text', '', 'API') - # Prepare scheduler - #global tz, mySchedules, plugins - # Init timezone in case it changed conf.tz = timezone(conf.TIMEZONE) @@ -228,7 +225,20 @@ def importConfigs (db): db.commitDB() # update only the settings datasource - # update_api(False, ["settings"]) - # TO DO this creates a circular reference between API and HELPER ! + update_api(db, False, ["settings"]) + #TO DO this creates a circular reference between API and HELPER ! mylog('info', '[Config] Imported new config') + + + +#------------------------------------------------------------------------------- +def read_config_file(filename): + """ + retuns dict on the config file key:value pairs + """ + # load the variables from pialert.conf + code = compile(filename.read_text(), filename.name, "exec") + confDict = {} # config dictionary + exec(code, {"__builtins__": {}}, confDict) + return confDict \ No newline at end of file diff --git a/pialert/logger.py b/pialert/logger.py index 8b6cf060..3ae32dc2 100644 --- a/pialert/logger.py +++ b/pialert/logger.py @@ -9,11 +9,9 @@ from const import * #------------------------------------------------------------------------------- # duplication from helper to avoid circle #------------------------------------------------------------------------------- -def timeNowTZ(): - if conf.tz == '': +def timeNow(): return datetime.datetime.now().replace(microsecond=0) - else: - return datetime.datetime.now(conf.tz).replace(microsecond=0) + #------------------------------------------------------------------------------- debugLevels = [ @@ -38,7 +36,7 @@ def mylog(requestedDebugLevel, n): #------------------------------------------------------------------------------- def file_print (*args): - result = timeNowTZ().strftime ('%H:%M:%S') + ' ' + result = timeNow().strftime ('%H:%M:%S') + ' ' for arg in args: result += str(arg) diff --git a/pialert/networkscan.py b/pialert/networkscan.py index 423f88b7..7fcf719a 100644 --- a/pialert/networkscan.py +++ b/pialert/networkscan.py @@ -36,14 +36,13 @@ def scan_network (db): db.commitDB() - # ScanCycle data - cycle_interval = scanCycle_data['cic_EveryXmin'] + # arp-scan command - arpscan_devices = [] + conf.arpscan_devices = [] if conf.ENABLE_ARPSCAN: mylog('verbose','[Network Scan] arp-scan start') - arpscan_devices = execute_arpscan (conf.userSubnets) + conf.arpscan_devices = execute_arpscan (conf.userSubnets) mylog('verbose','[Network Scan] arp-scan ends') # Pi-hole method @@ -59,51 +58,69 @@ def scan_network (db): db.commitDB() + +def process_scan (db, arpscan_devices = conf.arpscan_devices ): + + + # Query ScanCycle properties + scanCycle_data = query_ScanCycle_Data (db, True) + if scanCycle_data is None: + mylog('none', ['\n']) + mylog('none', ['[Process Scan]*************** ERROR ***************']) + mylog('none', ['[Process Scan] ScanCycle %s not found' % conf.cycle ]) + mylog('none', ['[Process Scan] Exiting...\n']) + return False + + db.commitDB() + + # ScanCycle data + cycle_interval = scanCycle_data['cic_EveryXmin'] + # Load current scan data - mylog('verbose','[Network Scan] Processing scan results') + mylog('verbose','[Process Scan] Processing scan results') save_scanned_devices (db, arpscan_devices, cycle_interval) # Print stats - mylog('none','[Network Scan] Print Stats') + mylog('none','[Process Scan] Print Stats') print_scan_stats(db) - mylog('none','[Network Scan] Stats end') + mylog('none','[Process Scan] Stats end') # Create Events - mylog('verbose','[Network Scan] Updating DB Info') - mylog('verbose','[Network Scan] Sessions Events (connect / discconnect)') + mylog('verbose','[Process Scan] Updating DB Info') + mylog('verbose','[Process Scan] Sessions Events (connect / discconnect)') insert_events(db) # Create New Devices # after create events -> avoid 'connection' event - mylog('verbose','[Network Scan] Creating new devices') + mylog('verbose','[Process Scan] Creating new devices') create_new_devices (db) # Update devices info - mylog('verbose','[Network Scan] Updating Devices Info') + mylog('verbose','[Process Scan] Updating Devices Info') update_devices_data_from_scan (db) # Resolve devices names - mylog('verbose','[Network Scan] Resolve devices names') + mylog('verbose','[Process Scan] Resolve devices names') update_devices_names(db) # Void false connection - disconnections - mylog('verbose','[Network Scan] Voiding false (ghost) disconnections') + mylog('verbose','[Process Scan] Voiding false (ghost) disconnections') void_ghost_disconnections (db) # Pair session events (Connection / Disconnection) - mylog('verbose','[Network Scan] Pairing session events (connection / disconnection) ') + mylog('verbose','[Process Scan] Pairing session events (connection / disconnection) ') pair_sessions_events(db) # Sessions snapshot - mylog('verbose','[Network Scan] Creating sessions snapshot') + mylog('verbose','[Process Scan] Creating sessions snapshot') create_sessions_snapshot (db) # Sessions snapshot - mylog('verbose','[Network Scan] Inserting scan results into Online_History') + mylog('verbose','[Process Scan] Inserting scan results into Online_History') insertOnlineHistory(db,conf.cycle) # Skip repeated notifications - mylog('verbose','[Network Scan] Skipping repeated notifications') + mylog('verbose','[Process Scan] Skipping repeated notifications') skip_repeated_notifications (db) # Commit changes diff --git a/pialert/plugin.py b/pialert/plugin.py index 647464ae..83714a62 100644 --- a/pialert/plugin.py +++ b/pialert/plugin.py @@ -7,9 +7,9 @@ from collections import namedtuple # pialert modules import conf from const import pluginsPath, logPath -from files import get_file_content, write_file from logger import mylog -from helper import timeNowTZ, updateState +from helper import timeNowTZ, updateState, get_file_content, write_file +from api import update_api @@ -269,7 +269,7 @@ def execute_plugin(db, plugin): process_plugin_events(db, plugin) # update API endpoints - # update_api(False, ["plugins_events","plugins_objects"]) # TO-DO - remover circular reference + update_api(db, False, ["plugins_events","plugins_objects"]) #------------------------------------------------------------------------------- def custom_plugin_decoder(pluginDict): diff --git a/pialert/reporting.py b/pialert/reporting.py index d12ec74c..6a8186ea 100644 --- a/pialert/reporting.py +++ b/pialert/reporting.py @@ -12,10 +12,8 @@ from json2table import convert # pialert modules import conf -from const import pialertPath, logPath -#from database import get_table_as_json -from files import get_file_content, write_file -from helper import generate_mac_links, isNewVersion, removeDuplicateNewLines, timeNow, hide_email, json_struc, updateState +from const import pialertPath, logPath, apiPath +from helper import generate_mac_links, removeDuplicateNewLines, timeNow, hide_email, json_struc, updateState, get_file_content, write_file from logger import logResult, mylog, print_log from mqtt import mqtt_start @@ -250,12 +248,13 @@ def send_notifications (db): write_file (logPath + '/report_output.html', mail_html) # Send Mail - if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or json_ports != [] or conf.debug_force_notification or plugins_report: - - # update_api(True) # TO-DO + if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or json_ports != [] or conf.debug_force_notification or plugins_report: mylog('none', ['[Notification] Changes detected, sending reports']) + mylog('info', ['[Notification] Udateing API files']) + send_api() + if conf.REPORT_MAIL and check_config('email'): updateState(db,"Send: Email") mylog('info', ['[Notification] Sending report by Email']) @@ -613,6 +612,13 @@ def to_text(_json): return payloadData +#------------------------------------------------------------------------------- +def send_api(): + mylog('verbose', ['[Send API] Updating notification_* files in ', apiPath]) + + write_file(apiPath + 'notification_text.txt' , mail_text) + write_file(apiPath + 'notification_text.html' , mail_html) + write_file(apiPath + 'notification_json_final.json' , json.dumps(json_final)) #------------------------------------------------------------------------------- diff --git a/pialert/scanners/nmapscan.py b/pialert/scanners/nmapscan.py index 38a55105..b13cb5d8 100644 --- a/pialert/scanners/nmapscan.py +++ b/pialert/scanners/nmapscan.py @@ -2,8 +2,7 @@ import subprocess import conf -from const import logPath -from database import sql_nmap_scan_all +from const import logPath, sql_nmap_scan_all from helper import json_struc, timeNow, updateState from logger import append_line_to_file, mylog #------------------------------------------------------------------------------- diff --git a/pialert/scanners/pholusscan.py b/pialert/scanners/pholusscan.py index fe4c9dfe..0f9e9fc1 100644 --- a/pialert/scanners/pholusscan.py +++ b/pialert/scanners/pholusscan.py @@ -40,7 +40,7 @@ def performPholusScan (db, timeoutSec, userSubnets): output = subprocess.check_output (pholus_args, universal_newlines=True, stderr=subprocess.STDOUT, timeout=(timeoutSec + 30)) except subprocess.CalledProcessError as e: # An error occured, handle it - mylog('none', [[PholusScan], e.output]) + mylog('none', ['[PholusScan]', e.output]) mylog('none', ["[PholusScan] Error - Pholus Scan - check logs"]) except subprocess.TimeoutExpired as timeErr: mylog('none', ['[PholusScan] Pholus TIMEOUT - the process forcefully terminated as timeout reached']) diff --git a/pialert/scanners/pihole.py b/pialert/scanners/pihole.py index 63ee3ba8..31ce2bdc 100644 --- a/pialert/scanners/pihole.py +++ b/pialert/scanners/pihole.py @@ -50,6 +50,9 @@ def copy_pihole_network (db): mylog('debug',[ '[PiHole Network] - completed - found ',sql.rowcount, ' devices']) return str(sql.rowcount) != "0" + +#------------------------------------------------------------------------------- + #------------------------------------------------------------------------------- def read_DHCP_leases (db): """