Plugins 0.4 - ForeignKey support added

2023-03-11 13:57:25 +11:00
parent d090b29c55
commit 2d0a4b79d8
7 changed files with 145 additions and 85 deletions
--- a/back/pialert.py
+++ b/back/pialert.py
@@ -40,6 +40,7 @@ from pathlib import Path
 from cron_converter import Cron
 from pytz import timezone
 from json2table import convert
+import hashlib

 #===============================================================================
 # SQL queries
@@ -54,7 +55,7 @@ sql_settings = "SELECT  * FROM Settings"
 sql_plugins_objects = "SELECT  * FROM Plugins_Objects"
 sql_language_strings = "SELECT  * FROM Plugins_Language_Strings"
 sql_plugins_events = "SELECT  * FROM Plugins_Events"
-sql_plugins_history = "SELECT  * FROM Plugins_History ORDER BY 'Index' DESC LIMIT 50"
+sql_plugins_history = "SELECT  * FROM Plugins_History ORDER BY 'Index' DESC"
 sql_new_devices = """SELECT * FROM ( SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC FROM Events_Devices
                                                                WHERE eve_PendingAlertEmail = 1
                                                                AND eve_EventType = 'New Device'
@@ -159,7 +160,7 @@ def print_log (pText):
    # Print line + time + elapsed time + text
    file_print ('[LOG_LEVEL=debug] ',
        # log_timestamp2, ' ',
-        log_timestamp2.replace(microsecond=0) - log_timestamp.replace(microsecond=0), ' ',
+        log_timestamp2.strftime ('%H:%M:%S'), ' ',
        pText)

    # Save current time to calculate elapsed time until next log
@@ -465,10 +466,10 @@ def importConfigs ():
    # -----------------
    plugins = get_plugins_configs()

-    mylog('none', ['[', timeNow(), '] Plugins: Number of dynamically loaded plugins: ', len(plugins.list)])
+    mylog('none', ['[', timeNow(), '] Plugins: Number of dynamically loaded plugins: ', len(plugins)])

    #  handle plugins
-    for plugin in plugins.list:
+    for plugin in plugins:
        print_plugin_info(plugin, ['display_name','description'])
        
        pref = plugin["unique_prefix"]   
@@ -3157,6 +3158,7 @@ def upgradeDB ():
                        Status TEXT NOT NULL,  
                        Extra TEXT NOT NULL,
                        UserData TEXT NOT NULL,
+                        ForeignKey TEXT NOT NULL,
                        PRIMARY KEY("Index" AUTOINCREMENT)
                    ); """
    sql.execute(sql_Plugins_Objects)
@@ -3176,6 +3178,7 @@ def upgradeDB ():
                        Status TEXT NOT NULL,              
                        Extra TEXT NOT NULL,
                        UserData TEXT NOT NULL,
+                        ForeignKey TEXT NOT NULL,
                        PRIMARY KEY("Index" AUTOINCREMENT)
                    ); """
    sql.execute(sql_Plugins_Events)
@@ -3195,6 +3198,7 @@ def upgradeDB ():
                        Status TEXT NOT NULL,              
                        Extra TEXT NOT NULL,
                        UserData TEXT NOT NULL,
+                        ForeignKey TEXT NOT NULL,
                        PRIMARY KEY("Index" AUTOINCREMENT)
                    ); """                    
    sql.execute(sql_Plugins_History)
@@ -3273,7 +3277,7 @@ def update_api(isNotification = False, updateOnlyDataSources = []):
        write_file(folder + 'notification_json_final.json'  , json.dumps(json_final))  

    # Save plugins
-    write_file(folder + 'plugins.json'  , json.dumps({"data" : plugins.list}))  
+    write_file(folder + 'plugins.json'  , json.dumps({"data" : plugins}))  

    #  prepare databse tables we want to expose 
    dataSourcesSQLs = [
@@ -3666,19 +3670,13 @@ def isNewVersion():
 #-------------------------------------------------------------------------------
 def get_plugins_configs():
    
-    pluginsDict = []
    pluginsList = []

    for root, dirs, files in os.walk(pluginsPath):
        for d in dirs:            # Loop over directories, not files            
            pluginsList.append(json.loads(get_file_content(pluginsPath + "/" + d + '/config.json')))          

-    return plugins_class(pluginsDict, pluginsList)
-
-#-------------------------------------------------------------------------------
-class plugins_class:
-    def __init__(self, list):
-        self.list = list
+    return pluginsList

 #-------------------------------------------------------------------------------
 def collect_lang_strings(json, pref):
@@ -3710,7 +3708,7 @@ def run_plugin_scripts(runType):

    mylog('debug', ['     [Plugins] Check if any plugins need to be executed on run type: ', runType])

-    for plugin in plugins.list:
+    for plugin in plugins:

        shouldRun = False

@@ -3823,9 +3821,9 @@ def execute_plugin(plugin):

        for line in newLines:
            columns = line.split("|")
-            # There has to be always 8 columns
-            if len(columns) == 8:
-                sqlParams.append((plugin["unique_prefix"], columns[0], columns[1], 'null', columns[2], columns[3], columns[4], columns[5], columns[6], 0, columns[7], 'null'))
+            # There has to be always 9 columns
+            if len(columns) == 9:
+                sqlParams.append((plugin["unique_prefix"], columns[0], columns[1], 'null', columns[2], columns[3], columns[4], columns[5], columns[6], 0, columns[7], 'null', columns[8]))
            else:
                mylog('none', ['        [Plugins]: Skipped invalid line in the output: ', line])
    
@@ -3844,9 +3842,9 @@ def execute_plugin(plugin):
        arr = get_sql_array (q) 

        for row in arr:
-            # There has to be always 8 columns
-            if len(row) == 8:
-                sqlParams.append((plugin["unique_prefix"], row[0], row[1], 'null', row[2], row[3], row[4], row[5], row[6], 0, row[7], 'null'))
+            # There has to be always 9 columns
+            if len(row) == 9:
+                sqlParams.append((plugin["unique_prefix"], row[0], row[1], 'null', row[2], row[3], row[4], row[5], row[6], 0, row[7], 'null', row[8]))
            else:
                mylog('none', ['        [Plugins]: Skipped invalid sql result'])

@@ -3860,9 +3858,9 @@ def execute_plugin(plugin):

    # process results if any
    if len(sqlParams) > 0:                
-        sql.executemany ("""INSERT INTO Plugins_Events ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) 
+        sql.executemany ("""INSERT INTO Plugins_Events ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) 
        commitDB ()
-        sql.executemany ("""INSERT INTO Plugins_History ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) 
+        sql.executemany ("""INSERT INTO Plugins_History ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams) 
        commitDB ()

        process_plugin_events(plugin)
@@ -3948,9 +3946,9 @@ def process_plugin_events(plugin):
            
            createdTime = plugObj.changed

-            sql.execute ("INSERT INTO Plugins_Objects (Plugin, Object_PrimaryID, Object_SecondaryID, DateTimeCreated, DateTimeChanged, Watched_Value1, Watched_Value2, Watched_Value3, Watched_Value4, Status, Extra, UserData) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (plugObj.pluginPref, plugObj.primaryId , plugObj.secondaryId , createdTime, plugObj.changed , plugObj.watched1 , plugObj.watched2 , plugObj.watched3 , plugObj.watched4 , plugObj.status , plugObj.extra, plugObj.userData ))    
+            sql.execute ("INSERT INTO Plugins_Objects (Plugin, Object_PrimaryID, Object_SecondaryID, DateTimeCreated, DateTimeChanged, Watched_Value1, Watched_Value2, Watched_Value3, Watched_Value4, Status, Extra, UserData, ForeignKey) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (plugObj.pluginPref, plugObj.primaryId , plugObj.secondaryId , createdTime, plugObj.changed , plugObj.watched1 , plugObj.watched2 , plugObj.watched3 , plugObj.watched4 , plugObj.status , plugObj.extra, plugObj.userData, plugObj.foreignKey ))    
        else:
-            q = f"UPDATE Plugins_Objects set Plugin = '{plugObj.pluginPref}', DateTimeChanged = '{plugObj.changed}', Watched_Value1 = '{plugObj.watched1}', Watched_Value2 = '{plugObj.watched2}', Watched_Value3 = '{plugObj.watched3}', Watched_Value4 = '{plugObj.watched4}', Status = '{plugObj.status}', Extra = '{plugObj.extra}' WHERE 'Index' = {plugObj.index}"
+            q = f"UPDATE Plugins_Objects set Plugin = '{plugObj.pluginPref}', DateTimeChanged = '{plugObj.changed}', Watched_Value1 = '{plugObj.watched1}', Watched_Value2 = '{plugObj.watched2}', Watched_Value3 = '{plugObj.watched3}', Watched_Value4 = '{plugObj.watched4}', Status = '{plugObj.status}', Extra = '{plugObj.extra}', ForeignKey = '{plugObj.foreignKey}' WHERE 'Index' = {plugObj.index}"

            sql.execute (q)

@@ -3964,7 +3962,7 @@ def process_plugin_events(plugin):
        if plugObj.status == 'new':
            createdTime = plugObj.changed        

-        sql.execute ("INSERT INTO Plugins_Events (Plugin, Object_PrimaryID, Object_SecondaryID, DateTimeCreated, DateTimeChanged, Watched_Value1, Watched_Value2, Watched_Value3, Watched_Value4, Status,  Extra, UserData) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (plugObj.pluginPref, plugObj.primaryId , plugObj.secondaryId , createdTime, plugObj.changed , plugObj.watched1 , plugObj.watched2 , plugObj.watched3 , plugObj.watched4 , plugObj.status , plugObj.extra, plugObj.userData ))    
+        sql.execute ("INSERT INTO Plugins_Events (Plugin, Object_PrimaryID, Object_SecondaryID, DateTimeCreated, DateTimeChanged, Watched_Value1, Watched_Value2, Watched_Value3, Watched_Value4, Status,  Extra, UserData, ForeignKey) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (plugObj.pluginPref, plugObj.primaryId , plugObj.secondaryId , createdTime, plugObj.changed , plugObj.watched1 , plugObj.watched2 , plugObj.watched3 , plugObj.watched4 , plugObj.status , plugObj.extra, plugObj.userData, plugObj.foreignKey ))    

        commitDB()

@@ -3984,8 +3982,10 @@ class plugin_object_class:
        self.status       = objDbRow[10]
        self.extra        = objDbRow[11]
        self.userData     = objDbRow[12]
+        self.foreignKey   = objDbRow[13]

-        self.idsHash      = str(hash(str(self.primaryId) + str(self.secondaryId)))    
+        # self.idsHash      = str(hash(str(self.primaryId) + str(self.secondaryId)))    
+        self.idsHash      = str(self.primaryId) + str(self.secondaryId)

        self.watchedClmns = []
        self.watchedIndxs = []          
--- a/front/php/server/dbHelper.php
+++ b/front/php/server/dbHelper.php
@@ -40,8 +40,8 @@
    $expireMinutes = $_REQUEST['expireMinutes'];    
  }

-  if (isset ($_REQUEST['key'])) {
-    $key = $_REQUEST['key'];    
+  if (isset ($_REQUEST['columnName'])) {
+    $columnName = $_REQUEST['columnName'];    
  }

  if (isset ($_REQUEST['id'])) {
@@ -66,8 +66,8 @@
    switch ($action) {
      case 'create':    create($skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values ); break;
      // case 'read'  :    read($skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values);    break;
-      case 'update':    update($key, $id, $skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values);  break;
-      case 'delete':    delete($key, $id, $dbtable);  break;
+      case 'update':    update($columnName, $id, $skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values);  break;
+      case 'delete':    delete($columnName, $id, $dbtable);  break;
      default:     logServerConsole ('Action: '. $action);  break;
    }
  }
@@ -76,7 +76,7 @@
 //------------------------------------------------------------------------------
 //  update
 //------------------------------------------------------------------------------
-function update($key, $id, $skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values) {
+function update($columnName, $id, $skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $values) {

  global $db;  

@@ -111,7 +111,7 @@ function update($key, $id, $skipCache, $defaultValue, $expireMinutes, $dbtable,
 
  // Update value
  $sql = 'UPDATE '.$dbtable.' SET '. $columnValues .'
-          WHERE "'. $key .'"="'. $id.'"';
+          WHERE "'. $columnName .'"="'. $id.'"';
  $result = $db->query($sql);

  if (! $result == TRUE) {
@@ -155,26 +155,37 @@ function create($skipCache, $defaultValue, $expireMinutes, $dbtable, $columns, $
 //------------------------------------------------------------------------------
 //  delete
 //------------------------------------------------------------------------------
-function delete($key, $id, $dbtable)
+function delete($columnName, $id, $dbtable)
 {
  global $db;

-   // handle one or multiple ids
-   if(strpos($id, ',') !== false) 
-   {
-     $idsArr = explode(",", $id);
-   }else
-   {
-     $idsArr = array($id);
-   }
+  // handle one or multiple ids
+  if(strpos($id, ',') !== false) 
+  {
+    $idsArr = explode(",", $id);
+  }else
+  {
+    $idsArr = array($id);
+  }
+
+  $idsStr = "";
+  
+  foreach ($idsArr as $item) 
+  {
+    $idsStr = $idsStr . '"' .$item.'"';
+  }

  // Insert new value
-  $sql = 'DELETE FROM '.$dbtable.' WHERE "'.$key.'" IN ('. $id .')';
+  $sql = 'DELETE FROM '.$dbtable.' WHERE "'.$columnName.'" IN ('. $idsStr .')';
  $result = $db->query($sql);

  if (! $result == TRUE) {
    echo "Error deleting entry\n\n$sql \n\n". $db->lastErrorMsg();
    return;
+  } else
+  {
+    echo lang('Gen_DataUpdatedUITakesTime');
+    return;
  }
 }

--- a/front/php/templates/language/en_us.php
+++ b/front/php/templates/language/en_us.php
@@ -13,6 +13,7 @@ $lang['en_us'] = array(
 // General
 //////////////////////////////////////////////////////////////////
 'Gen_Delete' => 'Delete',
+'Gen_DeleteAll' => 'Delete all',
 'Gen_Cancel' => 'Cancel',
 'Gen_Okay' => 'Ok',
 'Gen_Save' => 'Save',
@@ -669,6 +670,8 @@ The arp-scan time itself depends on the number of IP addresses to check so set t
 'NMAP_ARGS_description' => 'Arguments used to run the Nmap scan. Be careful to specify <a href="https://linux.die.net/man/1/nmap" target="_blank">the arguments</a> correctly. For example <code>-p -10000</code> scans ports from 1 to 10000.',

 // API
+'API_display_name' => 'API',
+'API_icon' => '<i class="fa fa-arrow-down-up-across-line"></i>',
 'API_CUSTOM_SQL_name' => 'Custom endpoint',
 'API_CUSTOM_SQL_description' => 'You can specify a custom SQL query which will generate a JSON file and then expose it via the <a href="/api/table_custom_endpoint.json" target="_blank"><code>table_custom_endpoint.json</code> file endpoint</a>.',

--- a/front/plugins.php
+++ b/front/plugins.php
@@ -63,6 +63,9 @@ function getFormControl(dbColumnDef, value, index) {
        case 'url':
            result = `<span><a href="${value}" target="_blank">${value}</a><span>`;
            break;
+        case 'devicemac':
+            result = `<span><a href="/deviceDetails.php?mac=${value}" target="_blank">${value}</a><span>`;
+            break;
        case 'threshold': 
            $.each(dbColumnDef.options, function(index, obj) {
                if(Number(value) < obj.maximum && result == '')
@@ -94,7 +97,7 @@ function saveData (id) {
    index  = $(`#${id}`).attr('data-my-index')
    columnValue = $(`#${id}`).val()

-    $.get(`php/server/dbHelper.php?action=update&dbtable=Plugins_Objects&key=Index&id=${index}&columns=UserData&values=${columnValue}`, function(data) {
+    $.get(`php/server/dbHelper.php?action=update&dbtable=Plugins_Objects&columnName=Index&id=${index}&columns=UserData&values=${columnValue}`, function(data) {
    
        // var result = JSON.parse(data);
        console.log(data) 
@@ -242,18 +245,25 @@ function generateTabs()

        // Generate the history rows        
        var histCount = 0
+        var histCountDisplayed = 0
+        
        for(i=0;i<pluginHistory.length;i++)
        {
            if(pluginHistory[i].Plugin == obj.unique_prefix)
            {
-                clm = ""
-
-                for(j=0;j<colDefinitions.length;j++) 
+                if(histCount < 50) // only display 50 entries to optimize performance
                {       
-                    clm += '<td>'+ pluginHistory[i][colDefinitions[j].column] +'</td>'
+                    clm = ""
+
+                    for(j=0;j<colDefinitions.length;j++) 
+                    {   
+                        clm += '<td>'+ pluginHistory[i][colDefinitions[j].column] +'</td>'
+                    }            
+                    hiRows += `<tr data-my-index="${pluginHistory[i]["Index"]}" >${clm}</tr>`
+
+                    histCountDisplayed++;
                }
-                hiRows += `<tr data-my-index="${pluginHistory[i]["Index"]}" >${clm}</tr>`
-                histCount++;
+                histCount++; // count and display the total
            }            
        }        

@@ -299,7 +309,7 @@ function generateTabs()
                        <li >
                            <a href="#historyTarget_${obj.unique_prefix}" data-toggle="tab" >
                                
-                                <i class="fa fa-clock"></i> <?= lang('Plugins_History');?> (${histCount})
+                                <i class="fa fa-clock"></i> <?= lang('Plugins_History');?> (${histCountDisplayed} out of ${histCount} )
                                
                            </a>
                        </li>
@@ -320,7 +330,7 @@ function generateTabs()
                            </tbody>
                        </table>
                        <div class="plugin-obj-purge">                                 
-                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_Objects' )"><?= lang('Gen_Purge');?></button> 
+                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_Objects' )"><?= lang('Gen_DeleteAll');?></button> 
                        </div> 
                    </div>
                    <div id="eventsTarget_${obj.unique_prefix}" class="tab-pane">
@@ -334,7 +344,7 @@ function generateTabs()
                            </tbody>
                        </table>
                        <div class="plugin-obj-purge">                                 
-                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_Events' )"><?= lang('Gen_Purge');?></button> 
+                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_Events' )"><?= lang('Gen_DeleteAll');?></button> 
                        </div> 
                    </div>    
                    <div id="historyTarget_${obj.unique_prefix}" class="tab-pane">
@@ -348,7 +358,7 @@ function generateTabs()
                            </tbody>
                        </table>
                        <div class="plugin-obj-purge">                                 
-                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_History' )"><?= lang('Gen_Purge');?></button> 
+                            <button class="btn btn-primary" onclick="purgeAll('${obj.unique_prefix}', 'Plugins_History' )"><?= lang('Gen_DeleteAll');?></button> 
                        </div> 
                    </div>                   

@@ -373,8 +383,6 @@ function generateTabs()

 // --------------------------------------------------------
 // handle first tab (objectsTarget_) display 
-var lastPrefix = ''
-
 function initTabs()
 {
    // events on tab change
@@ -384,26 +392,28 @@ function initTabs()
        // save the last prefix
        if(target.includes('_') == false )
        {
-            lastPrefix = target.split('#')[1]
+            pref = target.split('#')[1]
+        } else
+        {
+            pref = target.split('_')[1]
        } 

        everythingHidden = false;

-        if($('#objectsTarget_'+ lastPrefix) && $('#historyTarget_'+ lastPrefix) && $('#eventsTarget_'+ lastPrefix))
+        if($('#objectsTarget_'+ pref) != undefined && $('#historyTarget_'+ pref) != undefined && $('#eventsTarget_'+ pref) != undefined)
        {
-            everythingHidden = $('#objectsTarget_'+ lastPrefix).attr('class').includes('active') == false && $('#historyTarget_'+ lastPrefix).attr('class').includes('active') == false && $('#eventsTarget_'+ lastPrefix).attr('class').includes('active') == false;
+            everythingHidden = $('#objectsTarget_'+ pref).attr('class').includes('active') == false && $('#historyTarget_'+ pref).attr('class').includes('active') == false && $('#eventsTarget_'+ pref).attr('class').includes('active') == false;
        }

        // show the objectsTarget if no specific pane selected or if selected is hidden        
-        if((target == '#'+lastPrefix ) && everythingHidden) //|| target == '#objectsTarget_'+ lastPrefix
+        if((target == '#'+pref ) && everythingHidden) 
        {
-            var classTmp = $('#objectsTarget_'+ lastPrefix).attr('class');
+            var classTmp = $('#objectsTarget_'+ pref).attr('class');

-            if($('#objectsTarget_'+ lastPrefix).attr('class').includes('active') == false)
+            if($('#objectsTarget_'+ pref).attr('class').includes('active') == false)
            {   
-                console.log('show')
                classTmp += ' active';            
-                $('#objectsTarget_'+ lastPrefix).attr('class', classTmp)
+                $('#objectsTarget_'+ pref).attr('class', classTmp)
            }
        } 
    });
@@ -422,20 +432,27 @@ function purgeAll(callback) {
 }

 // --------------------------------------------------------
-dbIndexes = ''
-
 function purgeAllExecute() {
+    $.ajax({
+        method: "POST",
+        url: "php/server/dbHelper.php",
+        data: { action: "delete", dbtable: dbTable, columnName: 'Plugin', id:plugPrefix },
+        success: function(data, textStatus) {
+            showModalOk ('Result', data );
+        }
+    })

-    // Execute
-    // console.log("targetLogFile:" + targetLogFile)
-    // console.log("logFileAction:" + logFileAction)
+}

-    idArr = $('#NMAPSRV table[data-my-dbtable="Plugins_Objects"] tr[data-my-index]').map(function(){return $(this).attr("data-my-index");}).get();
+// --------------------------------------------------------
+function purgeVisible() {
+
+    idArr = $(`#${plugPrefix} table[data-my-dbtable="${dbTable}"] tr[data-my-index]`).map(function(){return $(this).attr("data-my-index");}).get();

    $.ajax({
        method: "POST",
        url: "php/server/dbHelper.php",
-        data: { action: "delete", dbtable: 'Plugins_Objects', key: 'Index', id:idArr.toString() },
+        data: { action: "delete", dbtable: dbTable, columnName: 'Index', id:idArr.toString() },
        success: function(data, textStatus) {
            showModalOk ('Result', data );
        }
--- a/front/plugins/README.md
+++ b/front/plugins/README.md
@@ -1,12 +1,22 @@
 # ⚠ Disclaimer

-Highly experimental feature. Follow the below very carefully and check example plugin(s). Plugin UI is not my priority right now, happy to approve PRs if you are interested in extending/improvintg the UI experience. 
+Highly experimental feature. Follow the below very carefully and check example plugin(s). Plugin UI is not my priority right now, happy to approve PRs if you are interested in extending/improvintg the UI experience (e.g. making the tables sortable/filterable). 
+
+## ❗ Known issues:
+
+These issues will be hopefully fixed with time, so please don't report them. Instead, if you know how, feel free to investigate and submit a PR to fix the below. Keep the PRs small as it's easier to approve them:
+
+* Existing plugin objects sometimes not interpreted correctly and a new object is created instead, resulting in dupliucat entries.
+* Occasional (experienced twice) hanging of processing plugin script file.
+* UI displaying outdated values until the API endpoints get refreshed. 

 ## Overview

-PiAlert comes with a simple plugin system to feed events from third-party scripts into the UI and then send notifications if desired.
+PiAlert comes with a plugin system to feed events from third-party scripts into the UI and then send notifications if desired.

-If you wish to develop a plugin, please check the existing plugin structure. Once the settings are saved by the user they need to be removed from the `pialert.conf` file manually if you want to re-initialize them from the `config.json` of teh plugin.
+If you wish to develop a plugin, please check the existing plugin structure. Once the settings are saved by the user they need to be removed from the `pialert.conf` file manually if you want to re-initialize them from the `config.json` of the plugin. 
+
+Again, please read the below carefully if you'd like to contribute with a plugin yourself. This documentation file might be outdated, so double check the sample plugins as well. 

 ## Plugin file structure overview 

@@ -35,6 +45,8 @@ You need to set the `data_source` to either `pialert-db-query` or `python-script
 ```json
 "data_source":  "pialert-db-query"
 ```
+Any of the above datasources have to return a "table" of the exact structure as outlined below.
+
 ### Column order and values

  | Order | Represented Column | Required | Description | 
@@ -47,6 +59,7 @@ You need to set the `data_source` to either `pialert-db-query` or `python-script
  | 5 | `Watched_Value3` | no | As above  |
  | 6 | `Watched_Value4` | no | As above  |
  | 7 | `Extra` | no | Any other data you want to pass and display in PiAlert and the notifications |
+  | 8 | `ForeignKey` | no | A foreign key that can be used to link to the parent object (usually a MAC address) |

 ### "data_source":  "python-script"

@@ -65,8 +78,8 @@ Valid CSV:

 ```csv

-https://www.google.com|null|2023-01-02 15:56:30|200|0.7898|null|null|null
-https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine
+https://www.google.com|null|2023-01-02 15:56:30|200|0.7898|null|null|null|null
+https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine|ff:ee:ff:11:ff:11

 ```

@@ -74,9 +87,9 @@ Invalid CSV with different errors on each line:

 ```csv

-https://www.google.com|null|2023-01-02 15:56:30|200|0.7898||null|null
+https://www.google.com|null|2023-01-02 15:56:30|200|0.7898||null|null|null
 https://www.duckduckgo.com|null|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine|
-|https://www.duckduckgo.com|null|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine
+|https://www.duckduckgo.com|null|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine|null
 null|192.168.1.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine
 https://www.duckduckgo.com|192.168.1.1|2023-01-02 15:56:30|null|0.9898|null|null|Best search engine
 https://www.google.com|null|2023-01-02 15:56:30|200|0.7898|||
@@ -100,7 +113,8 @@ SELECT  dv.dev_Name as Object_PrimaryID,
    ns.State as Watched_Value2,
    'null' as Watched_Value3,
    'null' as Watched_Value4,
-    ns.Extra as Extra
+    ns.Extra as Extra,
+    dv.dev_MAC as ForeignKey 
 FROM 
    (SELECT * FROM Nmap_Scan) ns 
 LEFT JOIN 
@@ -221,7 +235,7 @@ Example:
 ```
 ##### UI settings in database_column_definitions

-The UI will adjust how columns are displayed in the UI based on teh definition of the `database_column_definitions` object.
+The UI will adjust how columns are displayed in the UI based on the definition of the `database_column_definitions` object. Thease are the supported form controls and related functionality:

 - Only columns with `"show": true` and also with at least an english translation will be shown in the UI.
 - Supported types: `label`, `text`, `threshold`, `replace`
@@ -231,6 +245,8 @@ The UI will adjust how columns are displayed in the UI based on teh definition o
 - The `options` property is used in conjunction with these types:
  - `threshold` - The `options` array contains objects from lowest `maximum` to highest with corresponding `hexColor` used for the value background color if it's less than the specified `maximum`, but more than the previous one in the `options` array
  - `replace` - The `options` array contains objects with an `equals` property, that is compared to the "value" and if the values are the same, the string in `replacement` is displayed in the UI instead of the actual "value"
+  - `devicemac` - The value is considered to be a mac adress and a link pointing to the device with teh given mac address is generated.
+  - `url` - The value is considered to be a url so a link is generated.


 ```json
--- a/front/plugins/nmap_services/config.json
+++ b/front/plugins/nmap_services/config.json
@@ -150,6 +150,18 @@
                "string" : "Extra"
                }]
        },       
+        {
+            "column": "ForeignKey",
+            "show": true,
+            "type": "devicemac",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "MAC"
+                }]
+        },        
        {
            "column": "Status",
            "show": true,
@@ -195,7 +207,7 @@
        {
            "function": "CMD",
            "type": "text",
-            "default_value":"SELECT  dv.dev_Name as Object_PrimaryID, cast('http://' || dv.dev_LastIP as VARCHAR(100)) || ':' || cast( SUBSTR(ns.Port ,0, INSTR(ns.Port , '/')) as VARCHAR(100)) as Object_SecondaryID,  datetime() as DateTime,  ns.Service as Watched_Value1,        ns.State as Watched_Value2,        'null' as Watched_Value3,        'null' as Watched_Value4,        ns.Extra as Extra        FROM (SELECT * FROM Nmap_Scan) ns LEFT JOIN (SELECT dev_Name, dev_MAC, dev_LastIP FROM Devices) dv   ON ns.MAC = dv.dev_MAC",
+            "default_value":"SELECT  dv.dev_Name as Object_PrimaryID, cast('http://' || dv.dev_LastIP as VARCHAR(100)) || ':' || cast( SUBSTR(ns.Port ,0, INSTR(ns.Port , '/')) as VARCHAR(100)) as Object_SecondaryID,  datetime() as DateTime,  ns.Service as Watched_Value1,        ns.State as Watched_Value2,        'null' as Watched_Value3,        'null' as Watched_Value4,        ns.Extra as Extra, dv.dev_MAC as ForeignKey        FROM (SELECT * FROM Nmap_Scan) ns LEFT JOIN (SELECT dev_Name, dev_MAC, dev_LastIP FROM Devices) dv   ON ns.MAC = dv.dev_MAC",
            "options": [],
            "localized": ["name", "description"],
            "name" : [{
--- a/front/plugins/website_monitor/script.py
+++ b/front/plugins/website_monitor/script.py
@@ -51,8 +51,8 @@ def service_monitoring_log(site, status, latency):
                                                )
                             )
    with open(last_run, 'a') as last_run_logfile:
-        # https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine
-        last_run_logfile.write("{}|{}|{}|{}|{}|{}|{}|{}\n".format(
+        # https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine|null
+        last_run_logfile.write("{}|{}|{}|{}|{}|{}|{}|{}|{}\n".format(
                                                site,
                                                'null',
                                                strftime("%Y-%m-%d %H:%M:%S"),                                                
@@ -61,6 +61,7 @@ def service_monitoring_log(site, status, latency):
                                                'null',
                                                'null',
                                                'null',
+                                                'null',
                                                )
                             )