Update robots.pm

diff --git a/wwwroot/cgi-bin/lib/robots.pm b/wwwroot/cgi-bin/lib/robots.pm
index 9052f4fd192d1735fe362d87a0ee5d21ac2610b5..0e151c24db2be45bca6b905acd00542056a6f530 100644 (file)
--- a/wwwroot/cgi-bin/lib/robots.pm
+++ b/wwwroot/cgi-bin/lib/robots.pm
@@ -121,7 +121,7 @@
 # 2005-12-15\r
 #              added FAST Enteprise Crawler/6 (www dot fastsearch dot com). Note spelling Enteprise not Enterprise.\r
 #              added findlinks http://wortschatz.uni-leipzig.de/findlinks/\r
-#              added IBM Almaden Research Center WebFountain\99 http://www.almaden.ibm.com/cs/crawler [hc3]\r
+#              added IBM Almaden Research Center WebFountain™ http://www.almaden.ibm.com/cs/crawler [hc3]\r
 #              added INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)\r
 #              added lmspider (lmspider@scansoft.com) http://www.nuance.com/\r
 #              added noxtrumbot http://www.noxtrum.com/\r
@@ -401,6 +401,7 @@
 # RJC\r
 \r
 # Removed Baidu catchall because its picking up baidu.sogo.uc.UCBrowser which is a phone browser\r
+# Added baiduspider- catchall instead\r
 \r
 # Newly added from 2021-05-05\r
 # Adsbot\r
@@ -408,10 +409,10 @@
 # Bytespider\r
 # CheckMarkNetwork/\r
 # DuckDuckBot\r
-# Edg/\r
-# Foregenix Web Scan\r
+# # Foregenix Web Scan\r
 # IonCrawl\r
 # Linguee Bot\r
+# Neevabot\r
 # PetalBot\r
 # TkBot\r
 # vuhuvBot\r
@@ -444,6 +445,7 @@
 'Yahoo![\x20]Slurp',\r
 'Baiduspider/',\r
 'Baiduspider\-image',\r
+'Baiduspider-',\r
 'YandexBot/',\r
 'YandexImages/',\r
 'YandexImageResizer',\r
@@ -530,7 +532,6 @@
 'DuckDuckBot-Https',\r
 'DuckDuckBot',\r
 'DuckDuckGo\-Favicons\-Bot/',\r
-'Edg/',\r
 'ELinks/',\r
 'ELinks[\x20]\(',\r
 'EmailMarketingRobot/',\r
@@ -633,6 +634,7 @@
 'My[\x20]Nutch[\x20]Spider/',\r
 'myse/Nutch',\r
 'Naaraa',\r
+'Neevabot',\r
 'NerdyBot',\r
 'netEstate[\x20]NE[\x20]Crawler',\r
 'NetResearchServer/',\r
@@ -1625,6 +1627,7 @@
 'Yahoo![\x20]Slurp','Yahoo! Slurp',\r
 'Baiduspider/','Baiduspider',\r
 'Baiduspider\-image','Baiduspider-image',\r
+'Baiduspider-','Baiduspider ( catchall )',\r
 'YandexBot/','YandexBot',\r
 'YandexImages/','YandexImages',\r
 'YandexImageResizer','YandexImageResizer',\r
@@ -1711,7 +1714,6 @@
 'DuckDuckBot-Https','DuckDuckBot-Https',\r
 'DuckDuckBot','DuckDuckBot',\r
 'DuckDuckGo\-Favicons\-Bot/','DuckDuckGo-Favicons-Bot',\r
-'Edg/','Edg',\r
 'ELinks/','ELinks',\r
 'ELinks[\x20]\(','ELinks (',\r
 'EmailMarketingRobot/','EmailMarketingRobot',\r
@@ -1814,6 +1816,7 @@
 'My[\x20]Nutch[\x20]Spider/','My Nutch Spider',\r
 'myse/Nutch','myse/Nutch',\r
 'Naaraa','Naaraa',\r
+'Neevabot','Neevabot',\r
 'NerdyBot','NerdyBot',\r
 'netEstate[\x20]NE[\x20]Crawler','netEstate NE Crawler',\r
 'NetResearchServer/','NetResearchServer',\r