Updated database

diff --git a/wwwroot/cgi-bin/lib/robots.pm b/wwwroot/cgi-bin/lib/robots.pm
index eda1e6bcbe4bb47e371d3912442ebb9ef4d9ac6d..2c304d1698204f073339379bda3f9cbb127157c3 100644 (file)
--- a/wwwroot/cgi-bin/lib/robots.pm
+++ b/wwwroot/cgi-bin/lib/robots.pm
@@ -314,6 +314,7 @@
 "ezresult",
 "gigabot",
 "gnodspider",
+"internetseer",
 "justview",
 "linkbot",
 "linkchecker",
@@ -330,7 +331,8 @@
 );
 @RobotsSearchIDOrder_list3 = (
 # Generic robot
-"robot"
+"robot",
+"crawl"
 );
 
 
@@ -618,6 +620,7 @@
 "gigabot","GigaBot",
 "gnodspider","GNOD Spider",
 "ia_archiver", "IA Archiver",
+"internetseer", "InternetSeer",
 "jennybot", "JennyBot",
 "justview", "JustView",
 "linkbot","LinkBot",
@@ -639,7 +642,8 @@
 "yandex", "Yandex bot",
 
 # Generic root ID
-"robot", "Unknown robot"
+"robot", "Unknown robot (identified by 'robot')",
+"crawl", "Unknown robot (identified by 'crawl')"
 );
 
 

diff --git a/wwwroot/cgi-bin/lib/search_engines.pm b/wwwroot/cgi-bin/lib/search_engines.pm
index 3aaeafa78a717292c181065e5012bb22965df6c3..8b8b4821ff5f1b327b1b1d228f96f65eb9ddea22 100644 (file)
--- a/wwwroot/cgi-bin/lib/search_engines.pm
+++ b/wwwroot/cgi-bin/lib/search_engines.pm
@@ -197,7 +197,7 @@
 );
 
 # If no rules are known, this will be used to clean URL of not keyword parameters.
-@WordsToCleanSearchUrl= ("act=","annuaire=","btng=","categoria=","cfg=","cof=","cou=","cp=","dd=","domain=","dt=","dw=","exec=","geo=","hc=","height=","hl=","hq=","hs=","id=","kl=","lang=","loc=","lr=","matchmode=","medor=","message=","meta=","mode=","order=","page=","par=","pays=","pg=","pos=","prg=","qc=","refer=","sa=","safe=","sc=","sort=","src=","start=","style=","stype=","sum=","tag=","temp=","theme=","url=","user=","width=","what=","\\.x=","\\.y=","y=","look=");
+@WordsToCleanSearchUrl= ("act=","annuaire=","btng=","categoria=","cfg=","cof=","cou=","cp=","dd=","domain=","dt=","dw=","enc=","exec=","geo=","hc=","height=","hl=","hq=","hs=","id=","kl=","lang=","loc=","lr=","matchmode=","medor=","message=","meta=","mode=","order=","page=","par=","pays=","pg=","pos=","prg=","qc=","refer=","sa=","safe=","sc=","sort=","src=","start=","style=","stype=","sum=","tag=","temp=","theme=","url=","user=","width=","what=","\\.x=","\\.y=","y=","look=");
 # Never put the following exclusion ("ask=","claus=","general=","kw=","keyword=","keywords=","MT","p=","q=","qr=","qt=","query=","s=","search=","searchText=","string=","su=","w=") because they are strings that contain keywords we're looking for.