User-agent: * Disallow: /cgi-bin/ Disallow: /msdnaa/ Disallow: /dept/calendar/ Disallow: /dept/phpicalendar/ Disallow: /dept/contest/ # twiki disallows Disallow: /bin/attach Disallow: /bin/changes Disallow: /bin/edit Disallow: /bin/geturl Disallow: /bin/installpasswd Disallow: /bin/mailnotify Disallow: /bin/manage Disallow: /bin/oops Disallow: /bin/passwd Disallow: /bin/preview Disallow: /bin/rdiff Disallow: /bin/rdiffauth Disallow: /bin/register Disallow: /bin/rename Disallow: /bin/save Disallow: /bin/savemulti Disallow: /bin/search Disallow: /bin/setlib.cfg Disallow: /bin/statistics Disallow: /bin/testenv Disallow: /bin/configure Disallow: /bin/upload Disallow: /bin/viewauth Disallow: /bin/viewfile Disallow: /list/ # page does not exist Disallow: /wcms/research/cisd Disallow: /research/cisd/projects/trips/lexicon/cgi/browseontology # bingbot is hammering us. throttle their crawls # 1 slow, 5 very slow, 10 extremely slow #User-agent: msnbot #Crawl-delay: 10 #User-agent: bingbot #Crawl-delay: 10 User-agent: * Crawl-delay: 10 # gildea wordnet indexing is confusing search results for BRAID and other site keyword searches Disallow: /~gildea/cgi-bin/ # mailman archives Disallow: /pipermail/ # google search console reports 'coverage error' indexing non-exisitent urls # report claims linked at /dept/seminars/view/nlprg/ but that does not seem to be the case Disallow: /dept/seminars/view/view/nlprg/2017/5 Disallow: /dept/seminars/view/view/nlprg/2016/12 Disallow: /dept/seminars/view/view/nlprg/2017/2 Disallow: /dept/seminars/view/view/nlprg/2017/1