From: Dominique Lederer <dominique.lederer@return1.at>
Date: Tue, 4 Jul 2023 13:12:16 +0000 (+0200)
Subject: Update robots.pm
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=73b79849e29d6938c6116cc276a32c3af8ac0af0;p=thirdparty%2FAWStats.git

Update robots.pm
---

diff --git a/wwwroot/cgi-bin/lib/robots.pm b/wwwroot/cgi-bin/lib/robots.pm
index 31e6a2a3..d8609c9f 100644
--- a/wwwroot/cgi-bin/lib/robots.pm
+++ b/wwwroot/cgi-bin/lib/robots.pm
@@ -43,6 +43,10 @@
 #
 #-------------------------------------------------------
 
+# 2023-07-04 RobC 
+#              Removed Dalvik as native Android UI Browser User Agent
+#              Removed CFNetwork as native iOS and OSX Browser User Agent
+
 # 2021-05--05 RobC
 
 # Removed Baidu catchall because its picking up baidu.sogo.uc.UCBrowser which is a phone browser
@@ -514,7 +518,6 @@
 'CUBOT',
 'Curl/PHP',
 'cyencebot',
-'Dalvik/',
 'DataCrawler/',
 'daumoa',
 'daum',
@@ -780,7 +783,6 @@
 'ng/1\.',
 'ng/2\.',
 'libwww\-perl',
-'CFNetwork',
 'urllib',
 'javabee',
 'projectwf\-java\-test\-crawler',
@@ -1696,7 +1698,6 @@
 'CUBOT','CUBOT',
 'Curl/PHP','Curl/PHP',
 'cyencebot','cyencebot',
-'Dalvik/','Dalvik',
 'DataCrawler/','DataCrawler',
 'daumoa','daumoa',
 'daum','daum',
@@ -1962,7 +1963,6 @@
 'ng/1\.','ng/1.',
 'ng/2\.','ng/2.',
 'libwww\-perl','libwww-perl',
-'CFNetwork','CFNetwork',
 'urllib','urllib',
 'javabee','javabee',
 'projectwf\-java\-test\-crawler','projectwf-java-test-crawler',