Add files via upload

diff --git a/wwwroot/cgi-bin/lib/browsers_phone.pm b/wwwroot/cgi-bin/lib/browsers_phone.pm
index 384536b56c83bac6939f3633e353972e18d637a9..22ad5e1f0c7f57497450155a4ea7545c033f4364 100644 (file)
--- a/wwwroot/cgi-bin/lib/browsers_phone.pm
+++ b/wwwroot/cgi-bin/lib/browsers_phone.pm
@@ -24,6 +24,9 @@
                
 #package AWSUA;
 
+# RJC added from 2021-05-05
+# baidu\sogo\.uc.UCBrowser
+
 # Relocated from main file for easier editing
 %BrowsersFamily = (
     'msie'      => 1,
@@ -124,6 +127,7 @@
 'webmirror',
 'webvcr',
 'qnx\svoyager',
+'baidu\.sogo\.uc\.UCBrowser',
 # Site grabbers
 'teleport',
 'webcapture',
@@ -734,6 +738,7 @@
 'webmirror','WebMirror',
 'webvcr','WebVCR',
 'qnx\svoyager','QNX Voyager',
+'baidu\.sogo\.uc\.UCBrowser','baidu.sogo.uc.UCBrowser',
 # Site grabbers
 'teleport','TelePort Pro',
 'webcapture','Acrobat Webcapture',

diff --git a/wwwroot/cgi-bin/lib/robots.pm b/wwwroot/cgi-bin/lib/robots.pm
index 19a95905d5c69c7acbd138e77578dcaefd092440..9052f4fd192d1735fe362d87a0ee5d21ac2610b5 100644 (file)
--- a/wwwroot/cgi-bin/lib/robots.pm
+++ b/wwwroot/cgi-bin/lib/robots.pm
@@ -398,6 +398,23 @@
 # Note: Robots IDs are in lower case, '_', ' ' and '+' are changed into '[_+\s]' and are quoted.\r
 #-------------------------------------------------------\r
 \r
+# RJC\r
+\r
+# Removed Baidu catchall because its picking up baidu.sogo.uc.UCBrowser which is a phone browser\r
+\r
+# Newly added from 2021-05-05\r
+# Adsbot\r
+# BW/\r
+# Bytespider\r
+# CheckMarkNetwork/\r
+# DuckDuckBot\r
+# Edg/\r
+# Foregenix Web Scan\r
+# IonCrawl\r
+# Linguee Bot\r
+# PetalBot\r
+# TkBot\r
+# vuhuvBot\r
 \r
 @RobotsSearchIDOrder_list1 = (\r
 # Common robots (In robot file)\r
@@ -418,6 +435,7 @@
 'Googlebot\-Video/',\r
 'AdsBot\-Google[\x20]\(',\r
 'AdsBot\-Google\-Mobile\-Apps',\r
+'Adsbot',\r
 'Mediapartners-Google',\r
 'Feedfetcher\-Google',\r
 'Google\-Adwords\-Instant',\r
@@ -426,7 +444,6 @@
 'Yahoo![\x20]Slurp',\r
 'Baiduspider/',\r
 'Baiduspider\-image',\r
-'baidu',\r
 'YandexBot/',\r
 'YandexImages/',\r
 'YandexImageResizer',\r
@@ -471,9 +488,12 @@
 'boitho\.com\-dc/',\r
 'BoogleBot',\r
 'BusinessBot:',\r
+'BW/',\r
+'Bytespider',\r
 'CatchBot/',\r
 'CB/Nutch',\r
 'CCBot/',\r
+'CheckMarkNetwork/',\r
 'Cliqzbot/',\r
 'CMS[\x20]Crawler',\r
 'Companybook\-Crawler',\r
@@ -508,7 +528,9 @@
 'DomainStatsBot/',\r
 'DotBot/',\r
 'DuckDuckBot-Https',\r
+'DuckDuckBot',\r
 'DuckDuckGo\-Favicons\-Bot/',\r
+'Edg/',\r
 'ELinks/',\r
 'ELinks[\x20]\(',\r
 'EmailMarketingRobot/',\r
@@ -531,6 +553,7 @@
 'flamingo',\r
 'FlippyBearBot/',\r
 '^foo$',\r
+'Foregenix[\x20]Web[\x20]Scan',\r
 'FreeWebMonitoring[\x20]SiteChecker/',\r
 'fujilabol',\r
 'FurlBot/',\r
@@ -566,6 +589,7 @@
 'Incutio[\x20]XML',\r
 'IndeedBot',\r
 'InfluenceBot',\r
+'IonCrawl',\r
 'IRLbot/',\r
 'IssueCrawler',\r
 'istellabot/',\r
@@ -579,6 +603,7 @@
 'Lightspeed',\r
 'linkapediabot',\r
 'metager\-linkchecker',\r
+'Linguee[\x20]Bot',\r
 'linkchecker',\r
 'LinkCheck',\r
 'linkdexbot/',\r
@@ -629,6 +654,7 @@
 'Pcore',\r
 'pdffillerbot/',\r
 'peopleman',\r
+'PetalBot',\r
 'PhantomJS',\r
 'PHP/5\.2\.8',\r
 'Pinterestbot',\r
@@ -696,6 +722,7 @@
 'Test[\x20]Spider',\r
 'TestCrawler',\r
 'The[\x20]Knowledge[\x20]AI',\r
+'TkBot',\r
 'tracemyfile',\r
 'trendiction',\r
 'TurnitinBot/',\r
@@ -717,6 +744,7 @@
 'VoilaBot',\r
 'VORTEX/',\r
 'voyager/',\r
+'vuhuvBot',\r
 'W3C_Validator/',\r
 'W3C\-checklink/',\r
 'WBSearchBot/',\r
@@ -1588,6 +1616,7 @@
 'Googlebot\-Video/','Googlebot-Video',\r
 'AdsBot\-Google[\x20]\(','AdsBot-Google',\r
 'AdsBot\-Google\-Mobile\-Apps','AdsBot-Google-Mobile-Apps',\r
+'Adsbot','Adsbot',\r
 'Mediapartners\-Google','Mediapartners-Google',\r
 'Feedfetcher\-Google','Feedfetcher-Google',\r
 'Google\-Adwords\-Instant','Google-Adwords-Instant',\r
@@ -1596,7 +1625,6 @@
 'Yahoo![\x20]Slurp','Yahoo! Slurp',\r
 'Baiduspider/','Baiduspider',\r
 'Baiduspider\-image','Baiduspider-image',\r
-'baidu','Baidu ( catchall )',\r
 'YandexBot/','YandexBot',\r
 'YandexImages/','YandexImages',\r
 'YandexImageResizer','YandexImageResizer',\r
@@ -1641,9 +1669,12 @@
 'boitho\.com\-dc/','boitho.com-dc',\r
 'BoogleBot','BoogleBot',\r
 'BusinessBot:','BusinessBot:',\r
+'BW/','BW',\r
+'Bytespider','Bytespider',\r
 'CatchBot/','CatchBot',\r
 'CB/Nutch','CB/Nutch',\r
 'CCBot/','CCBot',\r
+'CheckMarkNetwork/','CheckMarkNetwork',\r
 'Cliqzbot/','Cliqzbot',\r
 'CMS[\x20]Crawler','CMS Crawler',\r
 'Companybook\-Crawler','Companybook-Crawler',\r
@@ -1678,7 +1709,9 @@
 'DomainStatsBot/','DomainStatsBot',\r
 'DotBot/','DotBot',\r
 'DuckDuckBot-Https','DuckDuckBot-Https',\r
+'DuckDuckBot','DuckDuckBot',\r
 'DuckDuckGo\-Favicons\-Bot/','DuckDuckGo-Favicons-Bot',\r
+'Edg/','Edg',\r
 'ELinks/','ELinks',\r
 'ELinks[\x20]\(','ELinks (',\r
 'EmailMarketingRobot/','EmailMarketingRobot',\r
@@ -1701,6 +1734,7 @@
 'flamingo','Flamingo_SearchEngine',\r
 'FlippyBearBot/','FlippyBearBot',\r
 '^foo$','foo',\r
+'Foregenix[\x20]Web[\x20]Scan','Foregenix Web Scan',\r
 'FreeWebMonitoring[\x20]SiteChecker/','FreeWebMonitoring SiteChecker',\r
 'fujilabol','fujilabol',\r
 'FurlBot/','FurlBot',\r
@@ -1736,6 +1770,7 @@
 'Incutio[\x20]XML','Incutio XML',\r
 'IndeedBot','IndeedBot',\r
 'InfluenceBot','InfluenceBot',\r
+'IonCrawl','IonCrawl',\r
 'IRLbot/','IRLbot',\r
 'IssueCrawler','IssueCrawler',\r
 'istellabot/','istellabot',\r
@@ -1748,6 +1783,7 @@
 'Lightspeed','Lightspeed',\r
 'linkapediabot','linkapediabot',\r
 'metager\-linkchecker','metager-linkchecker',\r
+'Linguee[\x20]Bot','Linguee Bot',\r
 'linkchecker','linkchecker',\r
 'LinkCheck','LinkCheck',\r
 'linkdexbot/','linkdexbot',\r
@@ -1799,6 +1835,7 @@
 'Pcore','Pcore',\r
 'pdffillerbot/','pdffillerbot',\r
 'peopleman','peopleman',\r
+'PetalBot','PetalBot',\r
 'PhantomJS','PhantomJS',\r
 'PHP/5\.2\.8','PHP/5.2.8',\r
 'Pinterestbot','Pinterestbot',\r
@@ -1866,6 +1903,7 @@
 'Test[\x20]Spider','Test Spider',\r
 'TestCrawler','TestCrawler',\r
 'The[\x20]Knowledge[\x20]AI', 'The Knowledge AI',\r
+'TkBot','TkBot',\r
 'tracemyfile','tracemyfile',\r
 'trendiction','trendiction',\r
 'TurnitinBot/','TurnitinBot',\r
@@ -1887,6 +1925,7 @@
 'VoilaBot','VoilaBot',\r
 'VORTEX/','VORTEX',\r
 'voyager/','voyager',\r
+'vuhuvBot','vuhuvBot',\r
 'W3C_Validator/','W3C_Validator',\r
 'W3C\-checklink/','W3C-checklink',\r
 'WBSearchBot/','WBSearchBot',\r