Merge pull request #1 from jokob-sk/main

Sinc

Dockerfile

@@ -8,7 +8,7 @@ ENV USER=pi USER_ID=1000 USER_GID=1000 TZ=Europe/London PORT=20211
 
 RUN apt-get update \
     && apt-get install --no-install-recommends tini ca-certificates curl libwww-perl arp-scan perl apt-utils cron sudo nginx-light php php-cgi php-fpm php-sqlite3 php-curl sqlite3 dnsutils net-tools python3 iproute2 nmap python3-pip zip -y \
-    && pip3 install requests paho-mqtt scapy cron-converter pytz \    
+    && pip3 install requests paho-mqtt scapy cron-converter pytz json2table \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \
     && apt-get clean autoclean \
     && apt-get autoremove \

back/pialert.py

@@ -38,6 +38,7 @@ import threading
 from pathlib import Path
 from cron_converter import Cron
 from pytz import timezone
+from json2table import convert
 
 #===============================================================================
 # SQL queries
@@ -68,6 +69,7 @@ piholeDhcpleases       = '/etc/pihole/dhcp.leases'
 debug_force_notification = False
 
 userSubnets = []
+changedPorts = []
 time_started = datetime.datetime.now()
 cron_instance = Cron()
 log_timestamp = time_started
@@ -289,7 +291,7 @@ def importConfig ():
     # Nmap
     global NMAP_ACTIVE, NMAP_TIMEOUT, NMAP_RUN, NMAP_RUN_SCHD, NMAP_ARGS 
     # API
-    global ENABLE_API, API_RUN, API_RUN_SCHD, API_RUN_INTERVAL
+    global ENABLE_API, API_RUN, API_RUN_SCHD, API_RUN_INTERVAL, API_CUSTOM_SQL
     
     mySettings = [] # reset settings
     # get config file
@@ -312,7 +314,7 @@ def importConfig ():
     TIMEZONE = ccd('TIMEZONE', 'Europe/Berlin' , c_d, 'Time zone', 'text', '', 'General')
     PIALERT_WEB_PROTECTION = ccd('PIALERT_WEB_PROTECTION', False , c_d, 'Enable logon', 'boolean', '', 'General')
     PIALERT_WEB_PASSWORD = ccd('PIALERT_WEB_PASSWORD', '8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92' , c_d, 'Logon password', 'readonly', '', 'General')
-    INCLUDED_SECTIONS = ccd('INCLUDED_SECTIONS', ['internet', 'new_devices', 'down_devices', 'events']   , c_d, 'Notify on', 'multiselect', "['internet', 'new_devices', 'down_devices', 'events']", 'General')
+    INCLUDED_SECTIONS = ccd('INCLUDED_SECTIONS', ['internet', 'new_devices', 'down_devices', 'events', 'ports']   , c_d, 'Notify on', 'multiselect', "['internet', 'new_devices', 'down_devices', 'events', 'ports']", 'General')
     SCAN_CYCLE_MINUTES = ccd('SCAN_CYCLE_MINUTES', 5 , c_d, 'Scan cycle delay (m)', 'integer', '', 'General')
     DAYS_TO_KEEP_EVENTS = ccd('DAYS_TO_KEEP_EVENTS', 90 , c_d, 'Delete events days', 'integer', '', 'General')
     REPORT_DASHBOARD_URL = ccd('REPORT_DASHBOARD_URL', 'http://pi.alert/' , c_d, 'PiAlert URL', 'text', '', 'General')
@@ -393,6 +395,7 @@ def importConfig ():
     API_RUN = ccd('API_RUN', 'schedule' , c_d, 'API execution', 'selecttext', "['none', 'interval', 'schedule']", 'API')
     API_RUN_SCHD = ccd('API_RUN_SCHD', '*/3 * * * *' , c_d, 'API schedule', 'text', '', 'API')    
     API_RUN_INTERVAL = ccd('API_RUN_INTERVAL', 10 , c_d, 'API update interval', 'integer', '', 'API')   
+    API_CUSTOM_SQL = ccd('API_CUSTOM_SQL', 'SELECT * FROM Devices WHERE dev_PresentLastScan = 0' , c_d, 'Custom endpoint', 'text', '', 'API')
     
     # Insert settings into the DB    
     sql.execute ("DELETE FROM Settings")    
@@ -574,10 +577,16 @@ def main ():
             if cycle in check_report:
                 # Check if new devices need to be scanned with Nmap
                 if NMAP_ACTIVE:
-                    sql.execute ("""SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC FROM Events_Devices
+                    sql.execute ("""SELECT * FROM 
+                                    ( SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC FROM Events_Devices
                                                                 WHERE eve_PendingAlertEmail = 1
                                                                 AND eve_EventType = 'New Device'
-                            ORDER BY eve_DateTime""")
+                                    ORDER BY eve_DateTime ) t1
+                                    LEFT JOIN 
+                                    (
+                                        SELECT dev_Name, dev_MAC as dev_MAC_t2 FROM Devices 
+                                    ) t2 
+                                    ON t1.dev_MAC = t2.dev_MAC_t2""")
 
                     newDevices = sql.fetchall()
                     commitDB()
@@ -1615,6 +1624,8 @@ def update_devices_names ():
 #-------------------------------------------------------------------------------
 def performNmapScan(devicesToScan):
 
+    global changedPorts     
+
     if len(devicesToScan) > 0:
 
         timeoutSec = NMAP_TIMEOUT
@@ -1660,8 +1671,8 @@ def performNmapScan(devicesToScan):
             for line in newLines:
                 append_line_to_file (logPath + '/pialert_nmap.log', line +'\n')                
             
-            # collect ports
+            # collect ports / new Nmap Entries
-            params = []
+            newEntries = []
 
             index = 0
             startCollecting = False
@@ -1675,15 +1686,111 @@ def performNmapScan(devicesToScan):
                 elif 'PORT' in line and 'STATE' in line and 'SERVICE' in line:    
                     startCollecting = False # end reached
                 elif startCollecting and len(line.split()) == 3:                                    
-                    params.append((device["dev_MAC"], timeNow(), line.split()[0], line.split()[1], line.split()[2], ''))
+                    newEntries.append(nmap_entry(device["dev_MAC"], timeNow(), line.split()[0], line.split()[1], line.split()[2], device["dev_Name"]))
                 elif 'Nmap done' in line:
                     duration = line.split('scanned in ')[1]            
             index += 1
 
-            if len(params) > 0:                
+            # previous Nmap Entries
+            oldEntries = []
+            
+            if len(newEntries) > 0:   
+
+                #  get all current NMAP ports from the DB
+                sql.execute(sql_nmap_scan_all) 
+
+                rows = sql.fetchall()
+
+                for row in rows: 
+                    oldEntries.append(nmap_entry(row["MAC"], row["Port"], row["State"], row["Service"], device["dev_Name"], row["Extra"], row["Index"]))
+
+                indexesToRemove = []
+
+                # Remove all entries already available in the database
+                for newEntry in newEntries:
+                    #  Check if available in oldEntries
+                    if any(x.hash == newEntry.hash for x in oldEntries):
+                        newEntries.pop(index)
+
+                file_print('[', timeNow(), '] Scan: Nmap found ', len(newEntries), ' new or changed ports')
+
+                # collect new ports, find the corresponding old entry and return for notification purposes
+                # also update the DB with the new values after deleting the old ones
+                if len(newEntries) > 0:
+
+                    params = []
+                    indexesToDelete = ""
+
+                    # Find old entry matching the new entry hash
+                    for newEntry in newEntries:                   
+
+                        foundEntry = None
+
+                        for oldEntry in oldEntries:
+                            if oldEntry.hash == newEntry.hash:
+
+                                params.append(newEntry.mac, newEntry.time, newEntry.port, newEntry.state, newEntry.service, oldEntry.extra)
+
+                                indexesToDelete = indexesToDelete + str(oldEntry.index) + ','
+
+                                foundEntry = oldEntry
+                        
+                        if foundEntry is not None:
+                            changedPorts.append(
+                                    {
+                                        'new' : { 
+                                                    "Name"   : foundEntry.name, 
+                                                    "MAC"    : newEntry.mac, 
+                                                    "Port"   : newEntry.port, 
+                                                    "State"  : newEntry.state, 
+                                                    "Service": newEntry.service, 
+                                                    "Extra"  : foundEntry.extra
+                                            }, 
+                                        'old' : { 
+                                                    "Name"   : foundEntry.name, 
+                                                    "MAC"    : foundEntry.mac, 
+                                                    "Port"   : foundEntry.port, 
+                                                    "State"  : foundEntry.state, 
+                                                    "Service": foundEntry.service, 
+                                                    "Extra"  : foundEntry.extra
+                                            }
+                                    }
+                                )                            
+                        else:
+                            changedPorts.append(
+                                    {
+                                        'new' : { 
+                                                    "Name"   : "New device", 
+                                                    "MAC"    : newEntry.mac, 
+                                                    "Port"   : newEntry.port, 
+                                                    "State"  : newEntry.state, 
+                                                    "Service": newEntry.service, 
+                                                    "Extra"  : ""
+                                            }
+                                    }
+                                )
+
+                    #  Delete old entries if available
+                    if len(indexesToDelete) > 0:
+                        sql.execute ("DELETE FROM Nmap_Scan where Index in (" + indexesToDelete[:-1] +")")
+                        commitDB ()
+
+                    # Insert new values into the DB 
                     sql.executemany ("""INSERT INTO Nmap_Scan ("MAC", "Time", "Port", "State", "Service", "Extra") VALUES (?, ?, ?, ?, ?, ?)""", params) 
                     commitDB ()
 
+#-------------------------------------------------------------------------------
+class nmap_entry:
+    def __init__(self, mac, time, port, state, service, name = '', extra = '', index = 0):
+        self.mac = mac
+        self.time = time
+        self.port = port
+        self.state = state
+        self.service = service
+        self.name = name
+        self.extra = extra
+        self.index = index
+        self.hash = str(hash(str(mac) + str(port)+ str(state)+ str(service)))
 
 #-------------------------------------------------------------------------------
 def performPholusScan (timeoutSec):
@@ -2027,9 +2134,14 @@ def skip_repeated_notifications ():
 json_final = []
 
 def send_notifications ():
-    global mail_text, mail_html, json_final    
+    global mail_text, mail_html, json_final, changedPorts
 
     deviceUrl              = REPORT_DASHBOARD_URL + '/deviceDetails.php?mac='
+    table_attributes = {"style" : "border-collapse: collapse; font-size: 12px; color:#70707", "width" : "100%", "cellspacing" : 0, "cellpadding" : "3px", "bordercolor" : "#C0C0C0", "border":"1"}
+    headerProps = "width='120px' style='color:blue; font-size: 12px;' bgcolor='#909090' "
+    thProps = "width='120px' style='color:#F0F0F0' bgcolor='#909090' "
+
+    build_direction = "TOP_TO_BOTTOM"
 
     # Reporting section
     file_print('  Check if something to report')    
@@ -2039,6 +2151,8 @@ def send_notifications ():
     json_new_devices = []
     json_down_devices = []
     json_events = []
+    json_ports = []
+
 
     # Disable reporting on events for devices where reporting is disabled based on the MAC address
     sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
@@ -2073,153 +2187,180 @@ def send_notifications ():
     mail_text = mail_text.replace ('<SERVER_NAME>', socket.gethostname() )
     mail_html = mail_html.replace ('<SERVER_NAME>', socket.gethostname() )
 
-    
-
     if 'internet' in INCLUDED_SECTIONS:
         # Compose Internet Section    
-        mail_section_Internet = False
+        text = ""  
-        mail_text_Internet = ''
-        mail_html_Internet = ''
-        text_line_template = '{} \t{}\t{}\t{}\n'
-        html_line_template = '<tr>\n'+ \
-            '  <td> <a href="{}{}"> {} </a> </td>\n  <td> {} </td>\n'+ \
-            '  <td style="font-size: 24px; color:#D02020"> {} </td>\n'+ \
-            '  <td> {} </td>\n</tr>\n'
 
-        sql.execute ("""SELECT * FROM Events
+        json_string = get_table_as_json("""SELECT eve_MAC as MAC,  eve_IP as IP, eve_DateTime as Datetime, eve_EventType as "Event Type", eve_AdditionalInfo as "Additional info" FROM Events
                         WHERE eve_PendingAlertEmail = 1 AND eve_MAC = 'Internet'
                         ORDER BY eve_DateTime""")  
 
+        if json_string["data"] == []:
+            html = ""      
+        else:
+            html = convert(json_string, build_direction=build_direction, table_attributes=table_attributes)
+
+            html = format_table(html, "data", headerProps, "Internet IP change")
+
+            headers = ["MAC", "Datetime", "IP", "Event Type", "Additional info"]
+
+            # prepare text-only message
+            text_line = '{}\t{}\n'
+
+            for device in json_string["data"]:
+                for header in headers:
+                    text += text_line.format ( header + ': ', device[header])  
+
+            #  Format HTML table headers
+            for header in headers:
+                html = format_table(html, header, thProps)   
+        
+        mail_text = mail_text.replace ('<SECTION_INTERNET>', text + '\n')
+        mail_html = mail_html.replace ('<INTERNET_TABLE>', html)
         
-        for eventAlert in sql :
-            mail_section_Internet = 'internet' in INCLUDED_SECTIONS
         # collect "internet" (IP changes) for the webhook json          
-            json_internet = add_json_list (eventAlert, json_internet)
+        json_internet = json_string["data"] 
-
-            mail_text_Internet += text_line_template.format (
-                'Event:', eventAlert['eve_EventType'], 'Time:', eventAlert['eve_DateTime'],
-                'IP:', eventAlert['eve_IP'], 'More Info:', eventAlert['eve_AdditionalInfo'])
-            mail_html_Internet += html_line_template.format (
-                deviceUrl, eventAlert['eve_MAC'],
-                eventAlert['eve_EventType'], eventAlert['eve_DateTime'],
-                eventAlert['eve_IP'], eventAlert['eve_AdditionalInfo'])
-
-
-        format_report_section (mail_section_Internet, 'SECTION_INTERNET',
-            'TABLE_INTERNET', mail_text_Internet, mail_html_Internet)
 
     if 'new_devices' in INCLUDED_SECTIONS:
         # Compose New Devices Section 
-        mail_section_new_devices = False
+        text = ""
-        mail_text_new_devices = ''
-        mail_html_new_devices = ''
-        text_line_template = '{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\n'
-        html_line_template    = '<tr>\n'+ \
-            '  <td> <a href="{}{}"> {} </a> </td>\n  <td> {} </td>\n'+\
-            '  <td> {} </td>\n  <td> {} </td>\n  <td> {} </td>\n</tr>\n'
 
-        sql.execute ("""SELECT * FROM Events_Devices
+        json_string = get_table_as_json("""SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
                         WHERE eve_PendingAlertEmail = 1
                         AND eve_EventType = 'New Device'
                         ORDER BY eve_DateTime""")        
+        if json_string["data"] == []:
+            html = ""                  
+        else:
+            html = convert(json_string, build_direction=build_direction, table_attributes=table_attributes)
+
+            html = format_table(html, "data", headerProps, "New devices")
+
+            headers = ["MAC", "Datetime", "IP", "Event Type", "Device name", "Comments"]
+
+            # prepare text-only message
+            text_line = '{}\t{}\n'
+            text = ""
+            for device in json_string["data"]:
+                for header in headers:
+                    text += text_line.format ( header + ': ', device[header])  
+
+            #  Format HTML table headers
+            for header in headers:
+                html = format_table(html, header, thProps)
+
+        mail_text = mail_text.replace ('<SECTION_NEW_DEVICES>', text + '\n')
+        mail_html = mail_html.replace ('<NEW_DEVICES_TABLE>', html)
         
-        for eventAlert in sql :
-            mail_section_new_devices = 'new_devices' in INCLUDED_SECTIONS
         # collect "new_devices" for the webhook json          
-            json_new_devices = add_json_list (eventAlert, json_new_devices)
+        json_new_devices = json_string["data"]
-
-            mail_text_new_devices += text_line_template.format (
-                'Name: ', eventAlert['dev_Name'], 'MAC: ', eventAlert['eve_MAC'], 'IP: ', eventAlert['eve_IP'],
-                'Time: ', eventAlert['eve_DateTime'], 'More Info: ', eventAlert['eve_AdditionalInfo'])
-            mail_html_new_devices += html_line_template.format (
-                deviceUrl, eventAlert['eve_MAC'], eventAlert['eve_MAC'],
-                eventAlert['eve_DateTime'], eventAlert['eve_IP'],
-                eventAlert['dev_Name'], eventAlert['eve_AdditionalInfo'])
-    
-        format_report_section (mail_section_new_devices, 'SECTION_NEW_DEVICES',
-            'TABLE_NEW_DEVICES', mail_text_new_devices, mail_html_new_devices)
 
 
     if 'down_devices' in INCLUDED_SECTIONS:
         # Compose Devices Down Section   
-        mail_section_devices_down = False
+        text = ""
-        mail_text_devices_down = ''
-        mail_html_devices_down = ''
-        text_line_template = '{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\n'
-        html_line_template     = '<tr>\n'+ \
-            '  <td> <a href="{}{}"> {} </a>  </td>\n  <td> {} </td>\n'+ \
-            '  <td> {} </td>\n  <td> {} </td>\n</tr>\n'
 
-        sql.execute ("""SELECT * FROM Events_Devices
+        json_string = get_table_as_json("""SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
                         WHERE eve_PendingAlertEmail = 1
                         AND eve_EventType = 'Device Down'
                         ORDER BY eve_DateTime""")        
+        if json_string["data"] == []:
+            html = ""      
+        else:
+            html = convert(json_string, build_direction=build_direction, table_attributes=table_attributes)
+
+            html = format_table(html, "data", headerProps, "Down devices")
+
+            headers = ["MAC", "Datetime", "IP", "Event Type", "Device name", "Comments"]
+
+            # prepare text-only message
+            text_line = '{}\t{}\n'
+            text = ""
+            for device in json_string["data"]:
+                for header in headers:
+                    text += text_line.format ( header + ': ', device[header])  
+
+            #  Format HTML table headers
+            for header in headers:
+                html = format_table(html, header, thProps)   
+
+        mail_text = mail_text.replace ('<SECTION_DEVICES_DOWN>', text + '\n')
+        mail_html = mail_html.replace ('<DOWN_DEVICES_TABLE>', html)
         
-        for eventAlert in sql :
-            mail_section_devices_down = 'down_devices' in INCLUDED_SECTIONS
         # collect "down_devices" for the webhook json
-            json_down_devices = add_json_list (eventAlert, json_down_devices)
+        json_down_devices = json_string["data"] 
-
-            mail_text_devices_down += text_line_template.format (
-                'Name: ', eventAlert['dev_Name'], 'MAC: ', eventAlert['eve_MAC'],
-                'Time: ', eventAlert['eve_DateTime'],'IP: ', eventAlert['eve_IP'])
-            mail_html_devices_down += html_line_template.format (
-                deviceUrl, eventAlert['eve_MAC'], eventAlert['eve_MAC'],
-                eventAlert['eve_DateTime'], eventAlert['eve_IP'],
-                eventAlert['dev_Name'])
-
-        format_report_section (mail_section_devices_down, 'SECTION_DEVICES_DOWN',
-            'TABLE_DEVICES_DOWN', mail_text_devices_down, mail_html_devices_down)
-
 
     if 'events' in INCLUDED_SECTIONS:
         # Compose Events Section  
-        mail_section_events = False
+        text = ""
-        mail_text_events   = ''
-        mail_html_events   = ''
-        text_line_template = '{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\t{}\t{}\n\n'
-        html_line_template = '<tr>\n  <td>'+ \
-                ' <a href="{}{}"> {} </a> </td>\n  <td> {} </td>\n'+ \
-                '  <td> {} </td>\n  <td> {} </td>\n  <td> {} </td>\n'+ \
-                '  <td> {} </td>\n</tr>\n'
 
-        sql.execute ("""SELECT * FROM Events_Devices
+        json_string = get_table_as_json("""SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
                         WHERE eve_PendingAlertEmail = 1
                         AND eve_EventType IN ('Connected','Disconnected',
                             'IP Changed')
                         ORDER BY eve_DateTime""")        
+        if json_string["data"] == []:
+            html = ""      
+        else:
+            html = convert(json_string, build_direction=build_direction, table_attributes=table_attributes)
+
+            html = format_table(html, "data", headerProps, "Events")
+
+            headers = ["MAC", "Datetime", "IP", "Event Type", "Device name", "Comments"]
+
+            # prepare text-only message
+            text_line = '{}\t{}\n'
+            text = ""
+            for device in json_string["data"]:
+                for header in headers:
+                    text += text_line.format ( header + ': ', device[header])              
+
+            #  Format HTML table headers
+            for header in headers:
+                html = format_table(html, header, thProps)   
+
+        mail_text = mail_text.replace ('<SECTION_EVENTS>', text + '\n')
+        mail_html = mail_html.replace ('<EVENTS_TABLE>', html)
         
-        for eventAlert in sql :
-            mail_section_events = 'events' in INCLUDED_SECTIONS
         # collect "events" for the webhook json        
-            json_events = add_json_list (eventAlert, json_events)
+        json_events = json_string["data"]
     
-            mail_text_events += text_line_template.format (
+    if 'ports' in INCLUDED_SECTIONS:           
-                'Name: ', eventAlert['dev_Name'], 'MAC: ', eventAlert['eve_MAC'], 
+        json_ports =  changedPorts
-                'IP: ', eventAlert['eve_IP'],'Time: ', eventAlert['eve_DateTime'],
-                'Event: ', eventAlert['eve_EventType'],'More Info: ', eventAlert['eve_AdditionalInfo'])
-            mail_html_events += html_line_template.format (
-                deviceUrl, eventAlert['eve_MAC'], eventAlert['eve_MAC'],
-                eventAlert['eve_DateTime'], eventAlert['eve_IP'],
-                eventAlert['eve_EventType'], eventAlert['dev_Name'],
-                eventAlert['eve_AdditionalInfo'])
 
-        format_report_section (mail_section_events, 'SECTION_EVENTS',
+        json_string = { "data" : changedPorts }
-            'TABLE_EVENTS', mail_text_events, mail_html_events)
+        
+        if json_string["data"] == []:
+            html = ""      
+        else:
+            html = convert(json_string, build_direction=build_direction, table_attributes=table_attributes)
+
+            html = format_table(html, "data", headerProps, "Changed or new ports")
+
+            headers = ["Name", "MAC", "Port", "State", "Service", "Extra"]
+
+            for header in headers:
+                html = format_table(html, header, thProps)        
+
+        mail_html = mail_html.replace ('<PORTS_TABLE>', html)
 
     json_final = {
                     "internet": json_internet,                        
                     "new_devices": json_new_devices,
                     "down_devices": json_down_devices,                        
-                    "events": json_events
+                    "events": json_events,
+                    "ports": json_ports,
                     }    
     
-    #  Write output emails for testing    
+    # Create clickable MAC links 
+    mail_html = generate_mac_links (mail_html, deviceUrl)
+
+    #  Write output emails for debug    
     write_file (logPath + '/report_output.txt', mail_text) 
     write_file (logPath + '/report_output.html', mail_html) 
 
     # Send Mail
-    if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or debug_force_notification:        
+    if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or json_ports != [] or debug_force_notification:        
 
         update_api(True)
 
@@ -2273,6 +2414,8 @@ def send_notifications ():
     sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
                     WHERE eve_PendingAlertEmail = 1""")
     
+    changedPorts = []
+
     # DEBUG - print number of rows updated
     file_print('    Notifications: ', sql.rowcount)
 
@@ -2326,6 +2469,26 @@ def check_config(service):
 
 
    
+#-------------------------------------------------------------------------------
+def format_table (html, thValue, props, newThValue = ''):
+
+    if newThValue == '':
+        newThValue = thValue
+        
+    return html.replace("<th>"+thValue+"</th>", "<th "+props+" >"+newThValue+"</th>" )
+
+#-------------------------------------------------------------------------------
+def generate_mac_links (html, deviceUrl):
+
+    p = re.compile(r'(?:[0-9a-fA-F]:?){12}')
+
+    MACs = re.findall(p, html)
+
+    for mac in MACs:        
+        html = html.replace('<td>' + mac + '</td>','<td><a href="' + deviceUrl + mac + '">' + mac + '</a></td>')
+
+    return html
+
 #-------------------------------------------------------------------------------
 def format_report_section (pActive, pSection, pTable, pText, pHTML):
     global mail_text
@@ -2968,29 +3131,30 @@ def update_api(isNotification = False):
         ["devices", sql_devices_all],
         ["nmap_scan", sql_nmap_scan_all],
         ["pholus_scan", sql_pholus_scan_all],
-        ["events_pending_alert", sql_events_pending_alert]
+        ["events_pending_alert", sql_events_pending_alert],
+        ["custom_endpoint", API_CUSTOM_SQL]
     ]
 
     # Save selected database tables
     for dsSQL in dataSourcesSQLs:
 
-        sql.execute(dsSQL[1]) 
+        json_string = get_table_as_json(dsSQL[1])
+
+        write_file(folder + 'table_' + dsSQL[0] + '.json'  , json.dumps(json_string))     
+
+#-------------------------------------------------------------------------------
+def get_table_as_json(sqlQuery):
+
+    sql.execute(sqlQuery) 
 
     columnNames = list(map(lambda x: x[0], sql.description)) 
 
     rows = sql.fetchall()    
 
-        json_string = get_table_as_json(rows, columnNames)
-
-        write_file(folder + 'table_' + dsSQL[0] + '.json'  , json.dumps(json_string))     
-
-#-------------------------------------------------------------------------------
-def get_table_as_json(rows, names):
-
     result = {"data":[]}
 
     for row in rows: 
-        tmp = fill_row(names, row)
+        tmp = fill_row(columnNames, row)
     
         result["data"].append(tmp)
     return result
@@ -3234,6 +3398,8 @@ def isNewVersion():
         buildTimestamp = int(f.read().strip())
         f.close() 
 
+        data = ""
+
         try:
             url = requests.get("https://api.github.com/repos/jokob-sk/Pi.Alert/releases")
             text = url.text

back/report_sample_2.html

@@ -2,137 +2,166 @@
    #  Pi.Alert
    #  Open Source Network Guard / WIFI & LAN intrusion detector 
    #
-#  repot_sample.html - Back module. Sample email reporting in HTML format
+   #  repot_template.html - Back module. Template to email reporting in HTML format
    #-------------------------------------------------------------------------------
    #  Puche 2021        pi.alert.application@gmail.com        GNU GPLv3
    #--------------------------------------------------------------------------- -->
-
    <html>
-
+    <head></head>
-<head>
-</head>
-
     <body>
        <font face=sans-serif>
-    <table align=center width=80% border=1 bordercolor=#909090 cellpadding=0 cellspacing=0 style="border-collapse: collapse; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.5)">
+          <table align=center width=100% cellpadding=0 cellspacing=0 style="border-radius: 5px;">
              <tr>
-        <td bgcolor=#EFB956  align=center style="padding: 20px 10px 10px 10px; font-size: 36px; font-weight: bold; color:#7F6000; text-shadow: 4px 4px 6px #909090">
+                <td bgcolor=#EFB956  align=center style="padding: 20px 10px 10px 10px; font-size: 30px; font-weight: bold; color:#000000; border-top-right-radius: 5px; border-top-left-radius: 5px; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
                    Pi.Alert Report
                 </td>
              </tr>
-
+             <tr>
+                <td bgcolor=#2656f1 width=100% align=center style="padding: 20px 10px 10px 10px; font-size: 20px; font-weight: bold; color:#ffffff; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
+                   <a style="color:#ffffff;cursor:pointer;" href="https://github.com/jokob-sk/Pi.Alert/releases">🆕 New version available 🆕</a>
+                </td>
+             </tr>
              <tr>
                 <td>
-            <table width=100% border=0 bgcolor=#FFD966 cellpadding=5px cellspacing=0 style="border-collapse: collapse; font-size: 16px; text-align:center; color:#5F5000">
+                   <table width=100% border=0 bgcolor=#FFD966 cellpadding=5px cellspacing=0 style="border-collapse: collapse; font-size: 15px; text-align:center; color:#404040">
                       <tr>
-                <td width=33%> Report Date: <b>2021-01-01 08:00</b> </td>
+                         <td width=100%> Report Date: <b>2023-01-30 22:17</b> </td>
-                <td width=34%> Scan Cycle: <b>1</b> </td>
-                <td width=33%> Server: <b>pi4</b> </td>
                       </tr>
                    </table>
                 </td>
              </tr>
-
-
-
              <tr>
-        <td bgcolor=#F5F5F5 height=400 valign=top style="padding: 20px">
+                <td bgcolor=#F5F5F5 height=200 valign=top style="padding: 10px">
-            
+                   <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
-
-            
-            <p style="font-size: 14px; font-weight: bold; color:#C04040; text-shadow: 2px 2px 4px #A0A0A0"> New Devices: </p> 
-
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=140> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th> Vendor </th>
-                </tr>
-
                       <tr>
-  <td> f8:d0:27:00:00:00 </td>
+                         <th width='120px' style='color:blue; font-size: 12px;' bgcolor='#909090'  >New devices</th>
-  <td> 2021-01-01 08:00:00 </td>
-  <td> 192.168.1.20 </td>
-  <td> Seiko Epson Corporation </td>
                       </tr>
                       <tr>
-  <td> c8:6c:3d:00:00:00 </td>
+                         <td>
-  <td> 2021-01-01 08:00:00 </td>
+                            <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
-  <td> 192.168.1.181 </td>
-  <td> Amazon Technologies Inc. </td>
-</tr>
-
-            </table>
-
-            <br>
-            
-
-            
-            <p style="font-size: 14px; font-weight: bold; color:#C04040; text-shadow: 2px 2px 4px #A0A0A0"> Devices Down: </p> 
-
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=140> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th> Device Name </th>
-                </tr>
-
                                <tr>
-  <td> 0c:ee:99:00:00:00 </td>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >MAC</th>
-  <td> 2021-01-01 08:00:00 </td>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Datetime</th>
-  <td> 192.168.1.171 </td>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >IP</th>
-  <td> Alexa - Echo </td>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Event Type</th>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Device name</th>
+                                  <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Comments</th>
                                </tr>
-
-            </table>
-
-            <br>
-            
-
-            
-            <p style="font-size: 14px; font-weight: bold; color:#409040; text-shadow: 2px 2px 4px #A0A0A0"> Events: </p> 
-
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=140> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th width=100> Event Type </th>
-                    <th width=140> Device Name </th>
-                    <th> Additional Info </th>
-                </tr>
-
                                <tr>
-  <td> 94:0c:98:00:00:00 </td>
+                                  <td><a href="http://192.168.1.1:20211/deviceDetails.php?mac=00:00:00:ef:a5:6c">00:00:00:ef:a5:6c</a></td>
-  <td> 2021-01-01 08:00:00 </td>
+                                  <td>2023-01-30 22:15:09</td>
-  <td> 192.168.1.132 </td>
+                                  <td>192.168.1.1</td>
-  <td> Connected </td>
+                                  <td>New Device</td>
-  <td> Person 1 - iPhone 11 </td>
+                                  <td>(name not found)</td>
                                   <td></td>
                                </tr>
                                <tr>
-  <td> 5c:41:5a:00:00:00 </td>
+                                  <td><a href="http://192.168.1.1:20211/deviceDetails.php?mac=00:00:00:ef:a5:6c">00:00:00:ef:a5:6c</a></td>
-  <td> 2021-01-01 08:00:00 </td>
+                                  <td>2023-01-30 22:17:59</td>
-  <td> 192.168.1.170 </td>
+                                  <td>192.168.1.82</td>
-  <td> IP Changed </td>
+                                  <td>New Device</td>
-  <td> Alexa Dot </td>
+                                  <td>(name not found)</td>
-  <td> Previous IP: 192.168.1.243 </td>
+                                  <td></td>
+                               </tr>
+                            </table>
+                         </td>
+                      </tr>
+                   </table>
+                   <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                      <tr>
+                         <th width='120px' style='color:blue; font-size: 12px;' bgcolor='#909090'  >Events</th>
+                      </tr>
+                      <tr>
+                         <td>
+                            <ul>
+                               <li>
+                                  <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                                     <tr>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >MAC</th>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Datetime</th>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >IP</th>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Event Type</th>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Device name</th>
+                                        <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Comments</th>
+                                     </tr>
+                                     <tr>
+                                        <td><a href="http://192.168.1.1:20211/deviceDetails.php?mac=00:00:00:ef:a5:6c">00:00:00:ef:a5:6c</a></td>
+                                        <td>2023-01-30 22:15:09</td>
+                                        <td>192.168.1.92</td>
+                                        <td>Disconnected</td>
+                                        <td>(name not found)</td>
+                                        <td></td>
+                                     </tr>
+                                  </table>
+                               </li>
+                            </ul>
+                         </td>
+                      </tr>
+                   </table>
+                   <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                      <tr>
+                         <th width='120px' style='color:blue; font-size: 12px;' bgcolor='#909090'  >Changed or new ports</th>
+                      </tr>
+                      <tr>
+                         <td>
+                            <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                               <tr>
+                                  <th>new</th>
+                               </tr>                              
+                               <tr>
+                                  <td>
+                                     <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                                        <tr>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Name</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >MAC</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Port</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >State</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Service</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Extra</th>
+                                        </tr>
+                                        <tr>
+                                           <td>New device</td>
+                                           <td><a href="http://192.168.1.1:20211/deviceDetails.php?mac=00:00:00:ef:a5:6c">00:00:00:ef:a5:6c</a></td>
+                                           <td>3263/tcp</td>
+                                           <td>open</td>
+                                           <td>ecolor-imager</td>
+                                           <td></td>
+                                        </tr>
+                                     </table>
+                                  </td>
+                               </tr>
+                               <tr>
+                                  <td>
+                                     <table style="border-collapse: collapse; font-size: 12px; color:#70707" width="100%" cellspacing="0" cellpadding="3px" bordercolor="#C0C0C0" border="1">
+                                        <tr>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Name</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >MAC</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Port</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >State</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Service</th>
+                                           <th width='120px' style='color:#F0F0F0' bgcolor='#909090'  >Extra</th>
+                                        </tr>
+                                        <tr>
+                                           <td>New device</td>
+                                           <td><a href="http://192.168.1.1:20211/deviceDetails.php?mac=00:00:00:ef:a5:6c">00:00:00:ef:a5:6c</a></td>
+                                           <td>3264/tcp</td>
+                                           <td>open</td>
+                                           <td>ccmail</td>
+                                           <td></td>
                                         </tr>
-
                                      </table>
-            
                                   </td>
                                </tr>
                                
+                            </table>
+                         </td>
+                      </tr>
+                   </table>
              <tr>
                 <td>
-            <table width=100% border=0 bgcolor=#70AD47 cellpadding=5px cellspacing=0 style="border-collapse: collapse; font-size: 12px; font-weight: bold; color:#385723">
+                   <table width=100% bgcolor=#46802e cellpadding=5px cellspacing=0 style="font-size: 13px; font-weight: bold; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px;">
                       <tr>
-                    <td width=25% style="text-align:Left">   Puche 2021</td>
+                         <td width=50% style="text-align:center"> Pi.Alert - Synology-NAS</td>
-                    <td width=50% style="text-align:center"> Pi.Alert 2.50 &nbsp; / &nbsp; 2021-01-01  </td>
-                    <td width=25% style="text-align:right">  GNU GPLv3</td>
                       </tr>
                    </table>
                 </td>
@@ -141,3 +170,4 @@
        </font>
     </body>
  </html>
+ 

back/report_template.txt

@@ -1,19 +1,15 @@
 Report Date: <REPORT_DATE>
 Server:      <SERVER_NAME>
-<SECTION_NEW_DEVICES>
+
 New Devices
 ----------------------
-<TABLE_NEW_DEVICES>
+<SECTION_NEW_DEVICES>
-</SECTION_NEW_DEVICES><SECTION_DEVICES_DOWN>
 Devices Down
 ----------------------
-<TABLE_DEVICES_DOWN>
+<SECTION_DEVICES_DOWN>
-</SECTION_DEVICES_DOWN><SECTION_EVENTS>
 Events
 ----------------------
-<TABLE_EVENTS>
+<SECTION_EVENTS>
-</SECTION_EVENTS><SECTION_INTERNET>
 Internet
 ----------------------
-<TABLE_INTERNET>
+<SECTION_INTERNET>
-</SECTION_INTERNET>

back/report_template_new_version.html

@@ -21,7 +21,7 @@
         </td>
         </tr>
         <tr> 
-            <td bgcolor=#2656f1 width=100% align=center style="padding: 20px 10px 10px 10px; font-size: 20px; font-weight: bold; color:#ffffff; border-top-right-radius: 5px; border-top-left-radius: 5px; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
+            <td bgcolor=#2656f1 width=100% align=center style="padding: 20px 10px 10px 10px; font-size: 20px; font-weight: bold; color:#ffffff; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2)">
                 <a style="color:#ffffff;cursor:pointer;" href="https://github.com/jokob-sk/Pi.Alert/releases">🆕 New version available 🆕</a>
             </td>             
         </tr>
@@ -36,76 +36,16 @@
 
         <tr>
         <td bgcolor=#F5F5F5 height=200 valign=top style="padding: 10px">
-            <SECTION_INTERNET>
-            <p style="font-size: 24px; font-weight: bold; color:#C04040"> Internet: </p> 
 
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070">
+            <INTERNET_TABLE>
-                <tr bgcolor=#909090 style="color:#f0f0f0">
-                    <th width=140> Event Type </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th> Additional Info </th>
-                </tr>
 
-                <TABLE_INTERNET>
+            <NEW_DEVICES_TABLE>
-            </table>
         
-            <br>
+            <DOWN_DEVICES_TABLE>
-            </SECTION_INTERNET>
         
-            <SECTION_NEW_DEVICES>
+            <EVENTS_TABLE>
-            <p style="font-size: 14px; font-weight: bold; color:#C04040"> New Devices: </p> 
         
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070">
+            <PORTS_TABLE>
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=150> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th width=140> Device Name </th>
-                    <th> Vendor </th>
-                </tr>
-
-                <TABLE_NEW_DEVICES>
-            </table>
-
-            <br>
-            </SECTION_NEW_DEVICES>
-
-            <SECTION_DEVICES_DOWN>
-            <p style="font-size: 14px; font-weight: bold; color:#C04040"> Devices Down: </p> 
-
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070">
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=140> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th> Device Name </th>
-                </tr>
-
-                <TABLE_DEVICES_DOWN>
-            </table>
-
-            <br>
-            </SECTION_DEVICES_DOWN>
-
-            <SECTION_EVENTS>
-            <p style="font-size: 14px; font-weight: bold; color:#409040"> Events: </p> 
-
-            <table width=100% border=1 bordercolor=#C0C0C0 cellpadding=3px cellspacing=0 style="border-collapse: collapse; font-size: 12px; color:#707070">
-                <tr bgcolor=#909090 style="color:#F0F0F0">
-                    <th width=140> MAC </th>
-                    <th width=130> Datetime </th>
-                    <th width=100> IP </th>
-                    <th width=100> Event Type </th>
-                    <th width=140> Device Name </th>
-                    <th> Additional Info </th>
-                </tr>
-
-                <TABLE_EVENTS>
-            </table>
-            </SECTION_EVENTS>
-        </td>
-        </tr>
         
         <tr>
         <td>
@@ -118,5 +58,7 @@
         </tr>
     </table>
     </font>
+
+
 </body>
 </html>

back/webhook_json_sample.json

@@ -17,197 +17,68 @@
                     "title": "Pi.Alert Notifications",
                     "title_link": "",
                     "text": {
-                        "internet": [
+                      "internet": [],
-                            [
+                      "new_devices": [{
-                              "Internet",
+                        "MAC": "74:ac:74:ac:74:ac",
-                              "243.243.243.243",
+                        "Datetime": "2023-01-30 22:15:09",
-                              "2022-01-06 18:32:03",
+                        "IP": "192.168.1.1",
-                              "Internet IP Changed",
+                        "Event Type": "New Device",
-                              "Previous Internet IP: 0.0.0.0",
+                        "Device name": "(name not found)",
-                              1,
+                        "Comments": null
-                              null
+                      }],
-                            ],
+                      "down_devices": [],
-                            [
+                      "events": [{
-                              "Internet",
+                        "MAC": "74:ac:74:ac:74:ac",
-                              "243.243.243.243",
+                        "Datetime": "2023-01-30 22:15:09",
-                              "2022-01-06 18:32:03",
+                        "IP": "192.168.1.92",
-                              "New Device",
+                        "Event Type": "Disconnected",
-                              null,
+                        "Device name": "(name not found)",
-                              1,
+                        "Comments": null
-                              null
+                      }, {
-                            ]
+                        "MAC": "74:ac:74:ac:74:ac",
-                          ],
+                        "Datetime": "2023-01-30 22:15:09",
-                        "new_devices": [
+                        "IP": "192.168.1.150",
-                            [
+                        "Event Type": "Disconnected",
-                              "b8:b8:b8:b8:b8:b8",
+                        "Device name": "(name not found)",
-                              "192.168.1.19",
+                        "Comments": null
-                              "2023-01-06 18:32:03",
+                      }],
-                              "New Device",
+                      "ports": [{
-                              "Raspberry Pi Foundation",
+                        "new": {
-                              1,
+                          "Name": "New device",
-                              null,
+                          "MAC": "74:ac:74:ac:74:ac",
-                              "b8:b8:b8:b8:b8:b8",
+                          "Port": "22/tcp",
-                              "raspberrypi",
+                          "State": "open",
-                              "(unknown)",
+                          "Service": "ssh",
-                              null,
+                          "Extra": ""
-                              "Raspberry Pi Foundation",
+                        }
-                              0,
+                      }, {
-                              null,
+                        "new": {
-                              null,
+                          "Name": "New device",
-                              "2021-01-06 18:32:03",
+                          "MAC": "74:ac:74:ac:74:ac",
-                              "2021-01-06 18:32:03",
+                          "Port": "53/tcp",
-                              "192.168.1.19",
+                          "State": "open",
-                              0,
+                          "Service": "domain",
-                              1,
+                          "Extra": ""
-                              1,
+                        }
-                              1,
+                      }, {
-                              0,
+                        "new": {
-                              0,
+                          "Name": "New device",
-                              null,
+                          "MAC": "74:ac:74:ac:74:ac",
-                              1,
+                          "Port": "80/tcp",
-                              1,
+                          "State": "open",
-                              null,
+                          "Service": "http",
-                              0,
+                          "Extra": ""
-                              null,
+                        }
-                              null
+                      }, {
-                            ],
+                        "new": {
-                            [
+                          "Name": "New device",
-                              "b1:b8:b8:b8:b8:b8",
+                          "MAC": "74:ac:74:ac:74:ac",
-                              "192.168.1.45",
+                          "Port": "443/tcp",
-                              "2021-01-06 18:32:03",
+                          "State": "open",
-                              "New Device",
+                          "Service": "https",
-                              "EliteGroup Computer Systems Co., LTD",
+                          "Extra": ""
-                              1,
+                        }
-                              null,
+                      }]
-                              "b1:b8:b8:b8:b8:b8",
-                              "my-NUC",
-                              "(unknown)",
-                              null,
-                              "EliteGroup Computer Systems Co., LTD",
-                              0,
-                              null,
-                              null,
-                              "2023-01-06 18:32:03",
-                              "2023-01-06 18:32:03",
-                              "192.168.1.45",
-                              0,
-                              1,
-                              1,
-                              1,
-                              0,
-                              0,
-                              null,
-                              1,
-                              1,
-                              null,
-                              0,
-                              null,
-                              null
-                            ]
-                          ],
-                        "down_devices": [
-                            [
-                              "aa:77:aa:77:aa:77",
-                              "192.168.1.151",
-                              "2021-01-07 14:20:53",
-                              "Device Down",
-                              "",
-                              1,
-                              25,
-                              "aa:77:aa:77:aa:77",
-                              "ttgo_tdisplay_weather",
-                              "(unknown)",
-                              "",
-                              "Espressif Inc.",
-                              0,
-                              "",
-                              "",
-                              "2021-01-06 23:13:06",
-                              "2021-01-06 23:13:06",
-                              "192.168.1.151",
-                              0,
-                              1,
-                              1,
-                              0,
-                              1,
-                              0,
-                              "2021-01-06 23:34:37.067330",
-                              0,
-                              0,
-                              "",
-                              0,
-                              "",
-                              ""
-                            ]
-                          ],
-                        "events": [
-                            [
-                                "aa:77:aa:77:aa:77",
-                                "192.168.1.151",
-                                "2022-08-12 21:48:00",
-                                "Connected",
-                                "",
-                                1,
-                                null,
-                                "aa:77:aa:77:aa:77",
-                                "ESP32 - display",
-                                "House",
-                                "",
-                                "Espressif Inc.",
-                                0,
-                                "",
-                                "",
-                                "2022-07-21 20:35:00",
-                                "2022-08-12 21:48:00",
-                                "192.168.1.151",
-                                0,
-                                1,
-                                1,
-                                1,
-                                0,
-                                0,
-                                "2022-08-12 21:42:47.937413",
-                                1,
-                                0,
-                                "",
-                                0,
-                                "aa:77:aa:77:aa:77",
-                                ""
-                            ],
-                            [
-                                "aa:77:aa:77:aa:77",
-                                "192.168.1.149",
-                                "2022-08-12 21:48:00",
-                                "Connected",
-                                "",
-                                1,
-                                null,
-                                "aa:77:aa:77:aa:77",
-                                "ESP32 - 1",
-                                "House",
-                                "Singleboard Computer (SBC)",
-                                "Espressif Inc.",
-                                0,
-                                "",
-                                "",
-                                "2022-07-15 05:30:00",
-                                "2022-08-12 21:48:00",
-                                "192.168.1.149",
-                                0,
-                                1,
-                                1,
-                                1,
-                                0,
-                                0,
-                                "2022-08-12 21:42:47.937413",
-                                1,
-                                1,
-                                "",
-                                0,
-                                "aa:77:aa:77:aa:77",
-                                ""
-                            ]
-                        ]
                     }
                 }
             ]

docker-compose.yml

@@ -9,7 +9,7 @@ services:
     volumes:
       - ${APP_DATA_LOCATION}/pialert/config:/home/pi/pialert/config
       # - ${APP_DATA_LOCATION}/pialert/db/pialert.db:/home/pi/pialert/db/pialert.db
-      - ${APP_DATA_LOCATION}/pialert/db:/home/pi/pialert/db      
+      - ${APP_DATA_LOCATION}/pialert/db2:/home/pi/pialert/db      
       # (optional) useful for debugging if you have issues setting up the container
       - ${LOGS_LOCATION}:/home/pi/pialert/front/log      
       # DELETE START anyone trying to use this file: comment out / delete BELOW lines, they are only for development purposes

docs/API.md

@@ -29,6 +29,7 @@ You can access the following files:
   | `table_nmap_scan.json` | The current state of the discovered ports by the regular NMAP scans. |
   | `table_pholus_scan.json` | The latest state of the [pholus](https://github.com/jokob-sk/Pi.Alert/tree/main/pholus) (A multicast DNS and DNS Service Discovery Security Assessment Tool) scan results. |
   | `table_events_pending_alert.json` | The list of the unprocessed (pending) notification events. |
+  | `table_custom_endpoint.json` | A custom endpoint generated by the SQL query specified by the `API_CUSTOM_SQL` setting. |
   
   Current/latest state of the aforementioned files depends on your settings.
 

front/deviceDetails.php

@@ -1575,6 +1575,32 @@ function skipNotifications () {
   // Set cycle 0
   $('#txtScanCycle').val ('no');  
 }
+// -----------------------------------------------------------------------------
+function askDeleteDeviceEvents () {
+  // Check MAC
+  if (mac == '') {
+    return;
+  }
+
+  // Ask delete device Events 
+  showModalWarning ('<?php echo lang('DevDetail_button_DeleteEvents');?>', '<?php echo lang('DevDetail_button_DeleteEvents_Warning');?>',
+    '<?php echo lang('Gen_Cancel');?>', '<?php echo lang('Gen_Delete');?>', 'deleteDeviceEvents');
+}
+
+function deleteDeviceEvents () {
+  // Check MAC
+  if (mac == '') {
+    return;
+  }
+
+  // Delete device events
+  $.get('php/server/devices.php?action=deleteDeviceEvents&mac='+ mac, function(msg) {
+    showMessage (msg);
+  });
+
+  // Deactivate controls
+  $('#panDetails :input').attr('disabled', true);
+}
 
 // -----------------------------------------------------------------------------
 // Overwrite all devices of the same type with the currently selected icon

front/devices.php

@@ -204,14 +204,26 @@
 // -----------------------------------------------------------------------------
 function main () {
 
-  // get visible columns
+  // get from cookie if available (need to use decodeURI as saved as part of URI in PHP)
-  $.get('php/server/parameters.php?action=get&expireMinutes=525600&defaultValue='+columnsStr+'&parameter=Front_Devices_Columns_Visible&skipcache', function(data) {
+  cookieColumnsVisibleStr = decodeURI(getCookie("Front_Devices_Columns_Visible")).replaceAll('%2C',',')  
 
+  defaultValue = cookieColumnsVisibleStr == "" ? columnsStr : cookieColumnsVisibleStr;
+
+  // get visible columns
+  $.get('php/server/parameters.php?action=get&expireMinutes=525600&defaultValue='+defaultValue+'&parameter=Front_Devices_Columns_Visible&skipcache', function(data) {
+    
+    // save which columns are in the Devices page visible
     tableColumnVisible = numberArrayFromString(data);
 
-    // get the custom order specified by the user
+    // get from cookie if available (need to use decodeURI as saved as part of URI in PHP)
-    $.get('php/server/parameters.php?action=get&expireMinutes=525600&defaultValue='+columnsStr+'&parameter=Front_Devices_Columns_Order&skipcache', function(data) {
+    cookieColumnsOrderStr = decodeURI(getCookie("Front_Devices_Columns_Order")).replaceAll('%2C',',')
 
+    defaultValue = cookieColumnsOrderStr == "" ? columnsStr : cookieColumnsOrderStr;    
+
+    // get the custom order specified by the user
+    $.get('php/server/parameters.php?action=get&expireMinutes=525600&defaultValue='+defaultValue+'&parameter=Front_Devices_Columns_Order&skipcache', function(data) {
+    
+      // save the columns order in the Devices page 
       tableColumnOrder = numberArrayFromString(data);
 
       //initialize the table headers in the correct order
@@ -244,8 +256,6 @@ function main () {
 
       $('#tableDevices tr').html(html);   
 
-
-
       // get parameter value
       $.get('php/server/parameters.php?action=get&defaultValue=50&parameter='+ parTableRows, function(data) {
         var result = JSON.parse(data);
@@ -282,14 +292,13 @@ function main () {
 // -----------------------------------------------------------------------------
 var tableColumnHide = [];
 
+// mapping the default order to the user specified one
 function mapIndx(oldIndex)
 {
   for(i=0;i<tableColumnOrder.length;i++)
   {
     if(tableColumnOrder[i] == oldIndex)
     {
-      // console.log('newIndex')
-      // console.log(i)
       return i;
     }
   }

front/network.php

@@ -502,7 +502,8 @@
   
   // ---------------------------------------------------------------------------
   var leafNodesCount = 0;
-  var treeLoadedAlready = false;
+  var visibleNodesCount = 0;
+  var parentNodesCount = 0;
   var hiddenMacs = []; // hidden children
   var hiddenChildren = [];
   
@@ -520,17 +521,24 @@
       //... of the current node
       if(list[i].parentMac == node.mac && !hiddenMacs.includes(list[i].parentMac))
       {   
+        visibleNodesCount++
+
         // and process them 
         children.push(getChildren(list[i], list, path + ((path == "") ? "" : '|') + list[i].parentMac, hiddenMacs))
+
       }
     }
 
     // note the total number of leaf nodes to calculate the font scaling
-    if(!treeLoadedAlready && children.length == 0) 
+    if(children.length == 0) 
     { 
       leafNodesCount++ 
+    } else
+    {
+      parentNodesCount++
     }  
 
+    
     return { 
       name: node.name,      
       path: path,
@@ -602,10 +610,17 @@
   // --------------------------------------------------------------------------- 
   var myTree;
   var treeAreaHeight = 600;
+  var emSize;
+  var nodeHeight;
+
   function initTree(myHierarchy)
   {    
-    // to prevent font scaling everytime we collapse/expand a subtree
+    // calculate the font size of the leaf nodes to fit everything into the tree area
-    treeLoadedAlready = true;
+    leafNodesCount == 0 ? 1 : leafNodesCount;
+    emSize = ((600/(20*leafNodesCount)).toFixed(2));        
+    emSize = emSize > 1 ? 1 : emSize;      
+    
+    nodeHeight = ((emSize*100*0.30).toFixed(0))
 
     $("#networkTree").attr('style', "height:"+treeAreaHeight+"px; width:1070px")
 
@@ -613,17 +628,11 @@
       htmlId: "networkTree",
       
       renderNode:  nodeData =>  { 
-
+        var fontSize = "font-size:"+emSize+"em;";
-        // calculate the font size of the leaf nodes to fit everything into the tree area
-        leafNodesCount == 0 ? 1 : leafNodesCount;
-        emSize = ((600/(20*leafNodesCount)).toFixed(2));        
-        emSize = emSize > 1 ? 1 : emSize;
-        
-        var fontSize = (nodeData.data.hasChildren) ? "" : "font-size:"+emSize+"em;";
 
         deviceIcon = (!emptyArr.includes(nodeData.data.icon )) ?  "<div class='netIcon ' ><i class='fa fa-"+nodeData.data.icon +"'></i></div>"    : "";
         collapseExpandIcon = nodeData.data.hiddenChildren ?  "square-plus" :"square-minus";
-        collapseExpandHtml = (nodeData.data.hasChildren) ?  "<div class='netCollapse' data-mytreepath='"+nodeData.data.path+"' data-mytreemac='"+nodeData.data.mac+"'><i class='fa fa-"+ collapseExpandIcon +" pointer'></i></div>"    : "";
+        collapseExpandHtml = (nodeData.data.hasChildren) ?  "<div class='netCollapse' style='font-size:"+emSize*2.5+"em;' data-mytreepath='"+nodeData.data.path+"' data-mytreemac='"+nodeData.data.mac+"'><i class='fa fa-"+ collapseExpandIcon +" pointer'></i></div>"    : "";
         statusCss = " netStatus-" + nodeData.data.status;
 
         selectedNodeMac = $(".nav-tabs-custom .active a").attr('data-mytabmac')
@@ -638,7 +647,6 @@
                                  justify-content:center;\
                                  " + fontSize + "\
                                  align-items:center;\
-                                 background-color:" +nodeData.data.color+";\
                                  border-radius:5px;'\
                           >\
                           <div class='netNodeText '>\
@@ -655,7 +663,7 @@
       mainAxisNodeSpacing: 'auto',
       // mainAxisNodeSpacing: 3,
       secondaryAxisNodeSpacing: 0.3,
-      nodeHeight: '25',    
+      nodeHeight: nodeHeight.toString(),    
       marginTop: '5',
       hasZoom: false,
       hasPan: false,

front/php/templates/language/en_us.php

@@ -195,7 +195,7 @@ $lang['en_us'] = array(
 'DevDetail_EveandAl_ScanCycle' => 'Scan device',
 'DevDetail_EveandAl_AlertAllEvents' => 'Alert All Events',
 'DevDetail_EveandAl_AlertDown' => 'Alert Down',
-'DevDetail_EveandAl_Skip' => 'Skip repeated notifications during',
+'DevDetail_EveandAl_Skip' => 'Skip repeated notifications for',
 'DevDetail_EveandAl_NewDevice' => 'New Device',
 'DevDetail_EveandAl_Archived' => 'Archived',
 'DevDetail_EveandAl_RandomMAC' => 'Random MAC',
@@ -638,7 +638,7 @@ the arp-scan will take hours to complete instead of seconds.
 'NMAP_ACTIVE_name' => 'Cycle run',
 'NMAP_ACTIVE_description' => 'If enabled this will execute a scan on a newly found device. For a scheduled or one-off scan, check the <a href="#NMAP_RUN"><code>NMAP_RUN</code> setting</a>.',
 'NMAP_TIMEOUT_name' => 'Run timeout',
-'NMAP_TIMEOUT_description' => 'Maximum time to wait for an Nmap scan to finish on any device.',
+'NMAP_TIMEOUT_description' => 'Maximum time in seconds to wait for an Nmap scan to finish on any device.',
 'NMAP_RUN_name' => 'Scheduled run',
 'NMAP_RUN_description' => 'Enable a regular Nmap scan on your network on all devices. The scheduling settings can be found below. If you select <code>once</code> Nmap is run only once on start for the time specified in <a href="#NMAP_TIMEOUT"><code>NMAP_TIMEOUT</code> setting</a>.',
 'NMAP_RUN_SCHD_name' => 'Schedule',
@@ -646,16 +646,21 @@ the arp-scan will take hours to complete instead of seconds.
 'NMAP_ARGS_name' => 'Arguments',
 'NMAP_ARGS_description' => 'Arguments used to run the Nmap scan. Be careful to specify <a href="https://linux.die.net/man/1/nmap" target="_blank">the arguments</a> correctly. For example <code>-p -10000</code> scans ports from 1 to 10000.',
 
-// Nmap
+// API
 'API_settings_group' => '<i class="fa fa-arrow-down-up-across-line"></i> API',
 'ENABLE_API_name' => 'Enable API',
-'ENABLE_API_description' => 'If enabled the app will start publishing and updating <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/API.md" target="_blank">simple API endpoints</a> under the <code>/home/pi/pialert/front/api/</code> folder and thus on the <code>pialert_url/api/File_name`</code> url.',
+'ENABLE_API_description' => 'If enabled the app will start publishing and updating <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/API.md" target="_blank">simple API endpoints</a> under the <code>/home/pi/pialert/front/api/</code> folder and thus on the <code>pialert_url/api/File_name</code> url.',
 'API_RUN_name' => 'Scheduling updates',
 'API_RUN_description' => 'Scheduling settings to specify when the API endpoints should be updated. If set to <code>schedule</code> then endpoints will be updated on a specified cron-like schedule specified by the <code>API_RUN_SCHD</code> setting. Otherwise if set to <code>interval</code> endpoints will be updated every N seconds specified by the <code>API_RUN_INTERVAL</code> setting.',
 'API_RUN_SCHD_name' => 'Schedule',
 'API_RUN_SCHD_description' => 'Depends on the <code>API_RUN</code> settings to be set to <code>schedule</code>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a  href="https://crontab.guru/" target="_blank">crontab.guru</a>).',
-'API_RUN_INTERVAL_name' => 'API update interval',
+'API_RUN_INTERVAL_name' => 'Update interval',
 'API_RUN_INTERVAL_description' => 'Depends on the <code>API_RUN</code> settings to be set to <code>interval</code>. The minimum cycle is <code>5</code> seconds.',
+'API_CUSTOM_SQL_name' => 'Custom endpoint',
+'API_CUSTOM_SQL_description' => 'You can specify a custom SQL query which will generate a JSON file and then expose it via the <a href="/api/table_custom_endpoint.json" target="_blank"><code>table_custom_endpoint.json</code> file endpoint</a>.',
+
+
+
 );
 
 ?>

front/settings.php

@@ -285,7 +285,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 <script>
 
   // number of settings has to be equal to
-  var settingsNumber = 65;
+  var settingsNumber = 66;
 
   // Wrong number of settings processing
   if(<?php echo count($settings)?> != settingsNumber)