# # robots.txt for https://www.w3.org/ # # $Id: robots.txt,v 1.89 2024/03/13 18:11:44 gerald Exp $ # # For use by search.w3.org User-agent: W3C-gsa Disallow: /Out-Of-Date User-agent: W3T_SE Disallow: /Out-Of-Date User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) Disallow: / # W3C Link checker User-agent: W3C-checklink Disallow: # Applebot continues to make hundreds of thousands of reqs/day for this area # even though it has been returning permanent redirects for years User-agent: Applebot Disallow: /People/domain/ # the following settings apply to all bots User-agent: * # Blogs - WordPress # https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization Disallow: /*/wp-admin/ Disallow: /*/wp-includes/ Disallow: /*/wp-content/plugins/ Disallow: /*/wp-content/cache/ Disallow: /*/wp-content/themes/ Disallow: /blog/*/trackback/ Disallow: /blog/*/feed/ Disallow: /blog/*/comments/ Disallow: /blog/*/category/*/* Disallow: /blog/*/*/trackback/ Disallow: /blog/*/*/feed/ Disallow: /blog/*/*/comments/ Disallow: /blog/*/*? Disallow: /community/trackback/ Disallow: /community/feed/ Disallow: /community/comments/ Disallow: /community/category/*/* Disallow: /community/*/trackback/ Disallow: /community/*/feed/ Disallow: /community/*/comments/ Disallow: /community/*/category/*/* Disallow: /community/*? Disallow: /Consortium/Offices/trackback/ Disallow: /Consortium/Offices/feed/ Disallow: /Consortium/Offices/comments/ Disallow: /Consortium/Offices/category/*/* Disallow: /Consortium/Offices/*/trackback/ Disallow: /Consortium/Offices/*/feed/ Disallow: /Consortium/Offices/*/comments/ Disallow: /Consortium/Offices/*? # Wikis - Mediawiki # https://www.mediawiki.org/wiki/Manual:Robots.txt Disallow: /wiki/index.php? Disallow: /wiki/index.php/Help Disallow: /wiki/index.php/MediaWiki Disallow: /wiki/index.php/Special: Disallow: /wiki/index.php/Template Disallow: /wiki/skins/ Disallow: /*/wiki/index.php? Disallow: /*/wiki/index.php/Help Disallow: /*/wiki/index.php/MediaWiki Disallow: /*/wiki/index.php/Special: Disallow: /*/wiki/index.php/Template Disallow: /*/wiki/Special: # various other access-controlled or expensive areas Disallow: /2004/ontaria/basic Disallow: /Team/ Disallow: /Project Disallow: /Web Disallow: /Systems Disallow: /Out-Of-Date Disallow: /2005/06/blog/ Disallow: /2004/08/W3CTalks Disallow: /2007/11/Talks/search Disallow: /People/all/ Disallow: /RDF/Validator/ARPServlet Disallow: /RDF/Validator/rdfval Disallow: /2003/03/Translations/byLanguage Disallow: /2003/03/Translations/byTechnology Disallow: /2005/11/Translations/Query Disallow: /2000/06/webdata/xslt Disallow: /2000/09/webdata/xslt Disallow: /2005/08/online_xslt/xslt Disallow: /Search/Mail/Public/ Disallow: /2006/02/chartergen Disallow: /2004/01/pp-impl Disallow: /Consortium/supporters Disallow: /2012/pyRdfa/extract Disallow: /WAI/PF/comments/ Disallow: /WAI/ER/tests/ Disallow: /participate/conferences.xml Disallow: /scripts/ Disallow: /2005/01/yacker/ Disallow: /2005/01/yacker? Disallow: /2005/07/pubrules? Disallow: /ns/hydra/console/? Disallow: /2007/08/grddl/? Disallow: /2009/07/webidl-check? Disallow: /RDF/Validator/ARPServlet? Disallow: /2000/06/webdata/xsv? Disallow: /2000/09/webdata/xsv? Disallow: /Style/CSS/members.be/ Disallow: /services Disallow: /2021/05/view-gallery/ Disallow: /*,* # WAI indexing Disallow: /WAI/beta/ Disallow: /WAI/ut1/ Disallow: /WAI/ut2/ Disallow: /WAI/ut3/ Disallow: /WAI/ut4/ # Disallow: /WAI/EO/Drafts/ Disallow: /WAI/drafts/