vendor_update plugin

2023-09-18 08:31:41 +10:00
parent a5b952f18c
commit 19f42e60ba
11 changed files with 586 additions and 133 deletions
--- a/front/plugins/db_cleanup/config.json
+++ b/front/plugins/db_cleanup/config.json
@@ -125,15 +125,15 @@
      }],
      "description": [{
          "language_code":"en_us",
-          "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
+          "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
      },
      {
          "language_code":"es_es",
-          "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
+          "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
      },
      {
          "language_code":"de_de",
-          "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
+          "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
      }]
  },    
    {
--- a/front/plugins/vendor_update/README.md
+++ b/front/plugins/vendor_update/README.md
@@ -0,0 +1,7 @@
+## Overview
+
+Plugin to run regular database cleanup tasks. It is strongly recommended to have an hourly or at least daily schedule running.
+
+### Usage
+
+- Check the Settings page for details.
--- a/front/plugins/vendor_update/config.json
+++ b/front/plugins/vendor_update/config.json
@@ -0,0 +1,408 @@
+{
+  "code_name": "vendor_update",
+  "unique_prefix": "VNDRPDT",
+  "enabled": true,
+  "data_source": "script",  
+  "show_ui": true,
+  "localized": ["display_name", "description", "icon"],
+
+  "display_name": [
+    {
+      "language_code": "en_us",
+      "string": "Vendor update"
+    }
+  ],
+  "icon": [
+    {
+      "language_code": "en_us",
+      "string": "<i class=\"fa-solid fa-landmark-flag\"></i>"
+    }
+  ],
+  "description": [
+    {
+      "language_code": "en_us",
+      "string": "A plugin to schedule vendor database updates for mac based vendor resolution."
+    }  
+  ],
+    "params" : [
+  ],  
+
+  "settings": [
+    {
+      "function": "RUN",            
+      "type": "text.select",            
+      "default_value":"schedule",
+      "options": ["disabled", "once", "schedule", "always_after_scan"],
+      "localized": ["name", "description"],
+      "name" :[{
+          "language_code":"en_us",
+          "string" : "When to run"
+      },
+      {
+          "language_code":"es_es",
+          "string" : "Cuándo ejecutar"
+      },
+      {
+          "language_code":"de_de",
+          "string" : "Wann laufen"
+      }],
+      "description": [{
+          "language_code":"en_us",
+          "string" : "When the cleanup should be performed. An overnight weekly <code>SCHEDULE</code> is recommended."
+      }]
+  },    
+    {
+      "function": "CMD",
+      "type": "readonly",
+      "default_value": "python3 /home/pi/pialert/front/plugins/vendor_update/script.py",
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Command"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Comando"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Befehl"
+        }	        
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "Command to run. This can not be changed"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Comando a ejecutar. Esto no se puede cambiar"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Befehl zum Ausführen. Dies kann nicht geändert werden"
+        }	        
+      ]
+    },
+    {
+      "function": "RUN_SCHD",
+      "type": "text",
+      "default_value":"0 4 * * *",
+      "options": [],
+      "localized": ["name", "description"],
+      "name" : [{
+          "language_code":"en_us",
+          "string" : "Schedule"
+      },
+      {
+          "language_code":"es_es",
+          "string" : "Schedule"
+      },
+      {
+          "language_code":"de_de",
+          "string" : "Schedule"
+      }],
+      "description": [{
+          "language_code":"en_us",
+          "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#VNDRPDT_RUN\"><code>VNDRPDT_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
+      },
+      {
+          "language_code":"es_es",
+          "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#VNDRPDT_RUN\"><code>VNDRPDT_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
+      },
+      {
+          "language_code":"de_de",
+          "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#VNDRPDT_RUN\"><code>VNDRPDT_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
+      }]
+  },    
+    {
+      "function": "RUN_TIMEOUT",
+      "type": "integer",
+      "default_value": 300,
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Run timeout"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Tiempo límite de ejecución"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Zeitüberschreitung"
+        }	        
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
+        },
+        {
+          "language_code": "es_es",
+          "string": "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
+        },
+        {
+          "language_code": "de_de",
+          "string": "Maximale Zeit in Sekunden, die auf den Abschluss des Skripts gewartet werden soll. Bei Überschreitung dieser Zeit wird das Skript abgebrochen."
+        }        
+      ]
+    }
+  ],
+
+  "database_column_definitions": 
+  [          
+    {
+      "column": "Index",            
+      "css_classes": "col-sm-2",
+      "show": false,
+      "type": "label",            
+      "default_value":"",
+      "options": [],
+      "localized": ["name"],
+      "name":[{
+          "language_code":"en_us",
+          "string" : "N/A"
+          },
+          {
+          "language_code":"es_es",
+          "string" : "N/A"
+          }]
+    } ,
+    {
+        "column": "Plugin",
+        "css_classes": "col-sm-2",
+        "show": false,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "N/A"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "N/A"
+            }]
+    },
+    {
+        "column": "Object_PrimaryID",
+        "mapped_to_column": "cur_MAC", 
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "device_mac",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "MAC address"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "Dirección MAC"
+            }]
+    },
+    {
+        "column": "Object_SecondaryID",
+        "mapped_to_column": "cur_IP", 
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "device_ip",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "IP"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "IP"
+            }]
+    } ,
+    {
+        "column": "DateTimeCreated",
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Created"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "Creado"
+            }]
+    },
+    {
+        "column": "DateTimeChanged",
+        "mapped_to_column": "cur_DateTime", 
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Changed"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "Cambiado"
+            }]
+    },
+    {
+      "column": "Dummy",
+      "mapped_to_column": "cur_ScanMethod", 
+      "mapped_to_column_data": {
+        "value": "VNDRPDT"                
+      }, 
+      "css_classes": "col-sm-2",
+      "show": true,
+      "type": "label",            
+      "default_value":"",
+      "options": [],
+      "localized": ["name"],
+      "name":[{
+          "language_code":"en_us",
+          "string" : "Scan method"
+          },
+          {
+          "language_code":"es_es",
+          "string" : "Método de escaneo"
+          }]
+    } ,
+    {
+        "column": "Watched_Value1",
+        "mapped_to_column": "cur_Vendor", 
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Vendor" 
+            }]
+    },
+    {
+        "column": "Watched_Value2",
+        "mapped_to_column": "cur_Name", 
+        "css_classes": "col-sm-2",
+        "show": true,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Hostname"
+            },
+        {
+            "language_code":"es_es",
+            "string" : "Nombre de host"
+            }]
+    },
+    {
+        "column": "Watched_Value3",
+        "css_classes": "col-sm-2",
+        "show": false,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "N/A"
+            }]
+    } ,
+    {
+        "column": "Watched_Value4",
+        "css_classes": "col-sm-2",
+        "show": false,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "N/A"
+            }]
+    } ,       
+    {
+        "column": "UserData",
+        "css_classes": "col-sm-2",
+        "show": false,
+        "type": "textbox_save",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Comments"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "Comentarios"
+            }]
+    },      
+    {
+        "column": "Extra",
+        "css_classes": "col-sm-3",
+        "show": false,
+        "type": "label",            
+        "default_value":"",
+        "options": [],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "N/A"
+            }]
+    },       
+    {
+        "column": "Status",
+        "css_classes": "col-sm-1",
+        "show": true,
+        "type": "replace",            
+        "default_value":"",
+        "options": [
+            {
+                "equals": "watched-not-changed",
+                "replacement": "<div style='text-align:center'><i class='fa-solid fa-square-check'></i><div></div>"
+            },
+            {
+                "equals": "watched-changed",
+                "replacement": "<div style='text-align:center'><i class='fa-solid fa-triangle-exclamation'></i></div>"
+            },
+            {
+                "equals": "new",
+                "replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"                
+            },
+            {
+                "equals": "missing-in-last-scan",
+                "replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
+            }
+        ],
+        "localized": ["name"],
+        "name":[{
+            "language_code":"en_us",
+            "string" : "Status"
+            },
+            {
+            "language_code":"es_es",
+            "string" : "Estado"
+            }]
+    }       
+  ]
+}
--- a/front/plugins/vendor_update/script.py
+++ b/front/plugins/vendor_update/script.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# test script by running:
+# /home/pi/pialert/front/plugins/db_cleanup/script.py pluginskeephistory=250 hourstokeepnewdevice=48 daystokeepevents=90
+
+import os
+import pathlib
+import argparse
+import sys
+import hashlib
+import csv
+import sqlite3
+from io import StringIO
+from datetime import datetime
+
+sys.path.append("/home/pi/pialert/front/plugins")
+sys.path.append('/home/pi/pialert/pialert') 
+
+from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
+from logger import mylog, append_line_to_file
+from helper import timeNowTZ
+from const import logPath, pialertPath
+from device import query_MAC_vendor
+
+
+CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
+LOG_FILE = os.path.join(CUR_PATH, 'script.log')
+RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
+
+def main():   
+
+    mylog('verbose', ['[VNDRPDT] In script']) 
+
+    # Get newest DB   
+    update_vendor_database()
+
+    # Resolve missing vendors
+    plugin_objects = Plugin_Objects(RESULT_FILE)
+
+    plugin_objects = update_vendors('/home/pi/pialert/db/pialert.db', plugin_objects)
+
+    plugin_objects.write_result_file()
+    
+    mylog('verbose', ['[VNDRPDT] Update complete'])   
+    
+    return 0
+
+#===============================================================================
+# Update device vendors database
+#===============================================================================
+def update_vendor_database():
+
+    # Update vendors DB (iab oui)
+    mylog('verbose', ['    Updating vendors DB (iab & oui)'])    
+    update_args = ['sh', pialertPath + '/back/update_vendors.sh']
+
+    # Execute command     
+    try:
+        # try runnning a subprocess safely
+        update_output = subprocess.check_output (update_args)
+    except subprocess.CalledProcessError as e:
+        # An error occured, handle it
+        mylog('none', ['    FAILED: Updating vendors DB, set LOG_LEVEL=debug for more info'])  
+        mylog('none', [e.output])        
+
+
+# resolve missing vendors
+def update_vendors (dbPath, plugin_objects): 
+   
+    # Connect to the PiAlert SQLite database
+    conn    = sqlite3.connect(dbPath)
+    cursor  = conn.cursor()
+
+    # Initialize variables
+    recordsToUpdate = []
+    ignored = 0
+    notFound = 0
+
+    # All devices loop
+    mylog('verbose', ['    Searching devices vendor'])    
+
+    for device in cursor.execute ("""SELECT * FROM Devices
+                                        WHERE   dev_Vendor      = '(unknown)' 
+                                                OR dev_Vendor   =''
+                                                OR dev_Vendor   IS NULL""") :
+        # Search vendor in HW Vendors DB
+        vendor = query_MAC_vendor (device['dev_MAC'])
+        if vendor == -1 :
+            notFound += 1
+        elif vendor == -2 :
+            ignored += 1
+        else :
+            plugin_objects.add_object(
+                primaryId   = device['dev_MAC'],        # MAC (Device Name)
+                secondaryId = device['dev_LastIP'],     # IP Address (always 0.0.0.0)
+                watched1    = vendor,  
+                watched2    = device['dev_Name'],
+                watched3    = "",
+                watched4    = "",
+                extra       = "",            
+                foreignKey  = device['dev_MAC']           
+            )            
+            
+    # Print log    
+    mylog('verbose', ["    Devices Ignored:  ", ignored])
+    mylog('verbose', ["    Vendors Not Found:", notFound])
+    mylog('verbose', ["    Vendors updated:  ", len(plugin_objects) ])
+
+    conn.commit()    
+    # Close the database connection
+    conn.close()
+
+    return plugin_objects
+
+    
+
+#===============================================================================
+# BEGIN
+#===============================================================================
+if __name__ == '__main__':
+    main()