Update robots.pm

diff --git a/wwwroot/cgi-bin/lib/robots.pm b/wwwroot/cgi-bin/lib/robots.pm
index 31e6a2a3aeac6365802d3b9c63c6f86918f02dd8..d8609c9f81e89fefc5acb8ef602e46e8d48229ce 100644 (file)
--- a/wwwroot/cgi-bin/lib/robots.pm
+++ b/wwwroot/cgi-bin/lib/robots.pm
@@ -43,6 +43,10 @@
 #\r
 #-------------------------------------------------------\r
 \r
+# 2023-07-04 RobC \r
+#              Removed Dalvik as native Android UI Browser User Agent\r
+#              Removed CFNetwork as native iOS and OSX Browser User Agent\r
+\r
 # 2021-05--05 RobC\r
 \r
 # Removed Baidu catchall because its picking up baidu.sogo.uc.UCBrowser which is a phone browser\r
@@ -514,7 +518,6 @@
 'CUBOT',\r
 'Curl/PHP',\r
 'cyencebot',\r
-'Dalvik/',\r
 'DataCrawler/',\r
 'daumoa',\r
 'daum',\r
@@ -780,7 +783,6 @@
 'ng/1\.',\r
 'ng/2\.',\r
 'libwww\-perl',\r
-'CFNetwork',\r
 'urllib',\r
 'javabee',\r
 'projectwf\-java\-test\-crawler',\r
@@ -1696,7 +1698,6 @@
 'CUBOT','CUBOT',\r
 'Curl/PHP','Curl/PHP',\r
 'cyencebot','cyencebot',\r
-'Dalvik/','Dalvik',\r
 'DataCrawler/','DataCrawler',\r
 'daumoa','daumoa',\r
 'daum','daum',\r
@@ -1962,7 +1963,6 @@
 'ng/1\.','ng/1.',\r
 'ng/2\.','ng/2.',\r
 'libwww\-perl','libwww-perl',\r
-'CFNetwork','CFNetwork',\r
 'urllib','urllib',\r
 'javabee','javabee',\r
 'projectwf\-java\-test\-crawler','projectwf-java-test-crawler',\r