From 788f23b9557e77c1be1d3edb4c540799ce0f556d Mon Sep 17 00:00:00 2001 From: Ulrich Wisser Date: Tue, 23 Apr 2024 15:28:53 +0200 Subject: [PATCH 1/3] cleanDeviceName rewritten to remove all _ labels and remove search list and local domain --- server/helper.py | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/server/helper.py b/server/helper.py index f4554c66..b28ef79b 100755 --- a/server/helper.py +++ b/server/helper.py @@ -570,26 +570,43 @@ def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = Fals #------------------------------------------------------------------------------- +import dns.resolver + def cleanDeviceName(str, match_IP): - # alternative str.split('.')[0] - str = str.replace("._airplay", "") - str = str.replace("._tcp", "") - str = str.replace(".localdomain", "") - str = str.replace(".local", "") - str = str.replace("._esphomelib", "") - str = str.replace("._googlecast", "") - str = str.replace(".lan", "") - str = str.replace(".home", "") - str = re.sub(r'-[a-fA-F0-9]{32}', '', str) # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 - str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#' - # remove trailing dots + + print("START cleanDeviceName(" + str + ")") + + # replace all labels starting with underscore + str = re.sub(r'^_[^\.]*\.', '', str) # leading label + str = re.sub(r'\._[^\.]*\.', '.', str) # nested label + + # get a stub resolver for access to resolv.conf configuration + resolv = dns.resolver.Resolver() + + # replace the local domain name + str = re.sub(r'\.' + resolv.domain.to_text() + r'$', '', str) + + # replace dns search list + for name in resolv.search: + str = re.sub(r'\.' + name.to_text() + r'$', '', str) + + # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 + str = re.sub(r'-[a-fA-F0-9]{32}', '', str) + + # Remove everything after '#' including the '#' + str = re.sub(r'#.*', '', str) + + # remove trailing dot if str.endswith('.'): str = str[:-1] - + # add matching info if match_IP: str = str + " (IP match)" + print("END cleanDeviceName = " + str) + + # done return str From 8bb93282017dd04fee6145fa6dfd6877ef23ee01 Mon Sep 17 00:00:00 2001 From: Ulrich Wisser Date: Wed, 24 Apr 2024 18:02:33 +0200 Subject: [PATCH 2/3] make new cleanDeviceName configurable, fix debug logging --- front/plugins/newdev_template/config.json | 24 ++++++++++++++++--- server/helper.py | 29 +++++++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/front/plugins/newdev_template/config.json b/front/plugins/newdev_template/config.json index 6ac9b477..b59caa0c 100755 --- a/front/plugins/newdev_template/config.json +++ b/front/plugins/newdev_template/config.json @@ -605,6 +605,25 @@ "string": "The icon associated with the device. Check the documentation on icons for more details." } ] + }, + { + "function": "dev_CleanDeviceName", + "type": "integer.checkbox", + "default_value": 0, + "options": [], + "localized": ["name", "description"], + "name": [ + { + "language_code": "en_us", + "string": "Use new algorithm to cleanup device names" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Check to start using the new code for cleaning device names. Removes all labels starting with underscore and removes network domain and search list." + } + ] } ], "required": [ @@ -627,10 +646,9 @@ "dev_Archived", "dev_Network_Node_MAC_ADDR", "dev_Network_Node_port", - "dev_Icon" + "dev_Icon", + "dev_CleanDeviceName" ], "additionalProperties": false } - - \ No newline at end of file diff --git a/server/helper.py b/server/helper.py index b28ef79b..2ed5e9bc 100755 --- a/server/helper.py +++ b/server/helper.py @@ -573,8 +573,33 @@ def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = Fals import dns.resolver def cleanDeviceName(str, match_IP): + if get_setting_value('NEWDEV_dev_CleanDeviceName'): + return NEW_cleanDeviceName(str, match_IP) - print("START cleanDeviceName(" + str + ")") + # alternative str.split('.')[0] + str = str.replace("._airplay", "") + str = str.replace("._tcp", "") + str = str.replace(".localdomain", "") + str = str.replace(".local", "") + str = str.replace("._esphomelib", "") + str = str.replace("._googlecast", "") + str = str.replace(".lan", "") + str = str.replace(".home", "") + str = re.sub(r'-[a-fA-F0-9]{32}', '', str) # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 + str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#' + # remove trailing dots + if str.endswith('.'): + str = str[:-1] + + + if match_IP: + str = str + " (IP match)" + + return str + +def NEW_cleanDeviceName(str, match_IP): + + mylog('debug', ["START cleanDeviceName(" + str + ")"]) # replace all labels starting with underscore str = re.sub(r'^_[^\.]*\.', '', str) # leading label @@ -604,7 +629,7 @@ def cleanDeviceName(str, match_IP): if match_IP: str = str + " (IP match)" - print("END cleanDeviceName = " + str) + mylog('debug', ["END cleanDeviceName = " + str]) # done return str From aa28fe2b0e4a0fa307665c3f1d2c826345840949 Mon Sep 17 00:00:00 2001 From: Ulrich Wisser Date: Tue, 30 Apr 2024 14:44:53 +0200 Subject: [PATCH 3/3] - update dependencies for docker and debian install - follow project naming guidelines --- Dockerfile | 2 +- Dockerfile.debian | 2 +- front/plugins/newdev_template/config.json | 6 +- install/install_dependencies.debian.sh | 2 +- server/helper.py | 81 +++++++++++------------ 5 files changed, 46 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a857935..28c17d04 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV PATH="/opt/venv/bin:$PATH" COPY . ${INSTALL_DIR}/ -RUN pip install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet \ +RUN pip install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet dnspython\ && bash -c "find ${INSTALL_DIR} -type d -exec chmod 750 {} \;" \ && bash -c "find ${INSTALL_DIR} -type f -exec chmod 640 {} \;" \ && bash -c "find ${INSTALL_DIR} -type f \( -name '*.sh' -o -name '*.py' -o -name 'pialert-cli' -o -name 'speedtest-cli' \) -exec chmod 750 {} \;" diff --git a/Dockerfile.debian b/Dockerfile.debian index c2301beb..eccf213c 100755 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -41,7 +41,7 @@ RUN phpenmod -v 8.2 sqlite3 # Setup virtual python environment and use pip3 to install packages RUN apt-get install -y python3-venv RUN python3 -m venv myenv -RUN /bin/bash -c "source myenv/bin/activate && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && pip3 install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet" +RUN /bin/bash -c "source myenv/bin/activate && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && pip3 install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet dnspython" # Create a buildtimestamp.txt to later check if a new version was released RUN date +%s > ${INSTALL_DIR}/front/buildtimestamp.txt diff --git a/front/plugins/newdev_template/config.json b/front/plugins/newdev_template/config.json index b59caa0c..89ea701f 100755 --- a/front/plugins/newdev_template/config.json +++ b/front/plugins/newdev_template/config.json @@ -607,7 +607,7 @@ ] }, { - "function": "dev_CleanDeviceName", + "function": "LESS_NAME_CLEANUP", "type": "integer.checkbox", "default_value": 0, "options": [], @@ -615,7 +615,7 @@ "name": [ { "language_code": "en_us", - "string": "Use new algorithm to cleanup device names" + "string": "Less Name Cleanup" } ], "description": [ @@ -647,7 +647,7 @@ "dev_Network_Node_MAC_ADDR", "dev_Network_Node_port", "dev_Icon", - "dev_CleanDeviceName" + "LESS_NAME_CLEANUP" ], "additionalProperties": false } diff --git a/install/install_dependencies.debian.sh b/install/install_dependencies.debian.sh index 9caf6bfc..1f55b6f5 100755 --- a/install/install_dependencies.debian.sh +++ b/install/install_dependencies.debian.sh @@ -30,4 +30,4 @@ source myenv/bin/activate update-alternatives --install /usr/bin/python python /usr/bin/python3 10 # install packages thru pip3 -pip3 install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet +pip3 install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet dnspython diff --git a/server/helper.py b/server/helper.py index 2ed5e9bc..13c53602 100755 --- a/server/helper.py +++ b/server/helper.py @@ -573,8 +573,45 @@ def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = Fals import dns.resolver def cleanDeviceName(str, match_IP): - if get_setting_value('NEWDEV_dev_CleanDeviceName'): - return NEW_cleanDeviceName(str, match_IP) + if get_setting_value('NEWDEV_LESS_NAME_CLEANUP'): + mylog('debug', ["Using new cleanDeviceName(" + str + ")"]) + + # replace all labels starting with underscore + str = re.sub(r'^_[^\.]*\.', '', str) # leading label + str = re.sub(r'\._[^\.]*\.', '.', str) # nested label + + # get a stub resolver for access to resolv.conf configuration + resolv = dns.resolver.Resolver() + + # replace the local domain name + str = re.sub(r'\.' + resolv.domain.to_text() + r'$', '', str) + + # replace dns search list + for name in resolv.search: + str = re.sub(r'\.' + name.to_text() + r'$', '', str) + + # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 + str = re.sub(r'-[a-fA-F0-9]{32}', '', str) + + # Remove everything after '#' including the '#' + str = re.sub(r'#.*', '', str) + + # remove trailing dot + if str.endswith('.'): + str = str[:-1] + + # add matching info + if match_IP: + str = str + " (IP match)" + + # done + mylog('debug', ["cleanDeviceName = " + str]) + return str + + ################################ + # + # OLD cleanDeviceName + mylog('debug', ["Using old cleanDeviceName(" + str + ")"]) # alternative str.split('.')[0] str = str.replace("._airplay", "") @@ -591,50 +628,12 @@ def cleanDeviceName(str, match_IP): if str.endswith('.'): str = str[:-1] - if match_IP: str = str + " (IP match)" + mylog('debug', ["cleanDeviceName = " + str]) return str -def NEW_cleanDeviceName(str, match_IP): - - mylog('debug', ["START cleanDeviceName(" + str + ")"]) - - # replace all labels starting with underscore - str = re.sub(r'^_[^\.]*\.', '', str) # leading label - str = re.sub(r'\._[^\.]*\.', '.', str) # nested label - - # get a stub resolver for access to resolv.conf configuration - resolv = dns.resolver.Resolver() - - # replace the local domain name - str = re.sub(r'\.' + resolv.domain.to_text() + r'$', '', str) - - # replace dns search list - for name in resolv.search: - str = re.sub(r'\.' + name.to_text() + r'$', '', str) - - # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 - str = re.sub(r'-[a-fA-F0-9]{32}', '', str) - - # Remove everything after '#' including the '#' - str = re.sub(r'#.*', '', str) - - # remove trailing dot - if str.endswith('.'): - str = str[:-1] - - # add matching info - if match_IP: - str = str + " (IP match)" - - mylog('debug', ["END cleanDeviceName = " + str]) - - # done - return str - - #------------------------------------------------------------------------------- # String manipulation methods #-------------------------------------------------------------------------------