User-Agent: * Disallow: /api.php Disallow: /cors/ Disallow: /geoip$ Disallow: /rest_v1/ Disallow: /w/Property: Disallow: /w/Property%3A Disallow: /w/property: Disallow: /*?title=Property: Disallow: /*?title=Property%3A Disallow: /*?*&title=Property: Disallow: /*?*&title=Property%3A Disallow: /w/Special: Disallow: /w/Special%3A Disallow: /w/special: Disallow: /*?title=Special: Disallow: /*?title=Special%3A Disallow: /*?*&title=Special: Disallow: /*?*&title=Special%3A Disallow: /*?action= Disallow: /*?*&action= Disallow: /*?feed= Disallow: /*?*&feed= Disallow: /*?from= Disallow: /*?*&from= Disallow: /*?mobileaction= Disallow: /*?*&mobileaction= Disallow: /*?oldid= Disallow: /*?*&oldid= Disallow: /*?printable= Disallow: /*?*&printable= Disallow: /*?redirect= Disallow: /*?*&redirect= Disallow: /*?uselang= Disallow: /*?*&uselang= Disallow: /*?useskin= Disallow: /*?*&useskin= Disallow: /*?veaction= Disallow: /*?*&veaction= Disallow: /*?filefrom= Disallow: /*?*&filefrom= Disallow: /*?fileuntil= Disallow: /*?*&fileuntil= Disallow: /*?navbox= Disallow: /*?*&navbox= Disallow: /*?pageuntil= Disallow: /*?*&pageuntil= Disallow: /*?pagefrom= Disallow: /*?*&pagefrom= Disallow: /*?diff= Disallow: /*?*&diff= Disallow: /*?curid= Disallow: /*?*&curid= Disallow: /*?search= Disallow: /*?*&search= Disallow: /*?section= Disallow: /*?*§ion= Disallow: /srv/ Disallow: /w/ Disallow: /includes/ Disallow: /maintenance/ Disallow: /cache/ Disallow: /images/ Disallow: /extensions/ Disallow: /skins/ Allow: /var/www/html/w/sitemaps_WF/OJ_EN/sitemap-index-100OrangeJuice_wiki.xml Allow: /var/www/html/w/sitemaps_WF/MG/sitemap-index-MetalGear_wiki.xml Allow: /var/www/html/w/sitemaps_WF/TWEWY/sitemap-index-TWEWY_wiki.xml Sitemap: https://orangejuice.wiki/w/sitemaps_WF/OJ_EN/sitemap-index-100OrangeJuice_wiki.xml Sitemap: https://metalgear.wiki/w/sitemaps_WF/MG/sitemap-index-MetalGear_wiki.xml Sitemap: https://twewy.wiki/w/sitemaps_WF/TWEWY/sitemap-index-TWEWY_wiki.xml # SEO bot User-agent: googlebot # all Google services Crawl-delay: 1000 Disallow: /private/ # disallow this directory Disallow: /srv/ # disallow this directory Allow: /w/load.php Allow: /var/www/html/robots.txt User-agent: DuckDuckBot # all duck duck go Crawl-delay: 1000 Disallow: /private/ # disallow this directory Disallow: /srv/ # disallow this directory Allow: /w/load.php Allow: /var/www/html/robots.txt user-agent: Baiduspider Disallow: / Allow: /var/www/html/robots.txt # Bytedance's web crawler. Wastes bandwidth. Ignores Disallow and gets caught in loops accessing special pages User-agent: Bytespider Disallow: / Allow: /var/www/html/robots.txt # Huawei's web crawler. Wastes bandwidth. Ignores Disallow and gets caught in loops accessing special pages. Produces the majority of uncachable requests. User-agent: PetalBot Disallow: / Allow: /var/www/html/robots.txt # Bing bot User-agent: Bingbot Disallow: / Allow: /var/www/html/robots.txt User-agent: MJBot Disallow: / Allow: /var/www/html/robots.txt User-agent: netEstate NE Crawler Disallow: / Allow: /var/www/html/robots.txt User-agent: babbar Disallow: / Allow: /var/www/html/robots.txt User-agent: SentiBot Disallow: / Allow: /var/www/html/robots.txt User-agent: DataForSeoBot Disallow: / Allow: /var/www/html/robots.txt User-agent: naver.me Disallow: / Allow: /var/www/html/robots.txt User-agent: Turnitin Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: AhrefsBot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: DotBot Disallow: / Allow: /var/www/html/robots.txt # Unknown User-agent: Mauibot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: MJ12bot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: SemrushBot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: DataForSeoBot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: Neevabot Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: MegaIndex Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: Seekport Disallow: / Allow: /var/www/html/robots.txt # SEO bot User-agent: Barkrowler Disallow: / Allow: /var/www/html/robots.txt User-agent: ImagesiftBot Disallow: / Allow: /var/www/html/robots.txt User-agent: BLEXBot Disallow: / Allow: /var/www/html/robots.txt User-agent: ZoominfoBot Disallow: / Allow: /var/www/html/robots.txt User-agent: 80legs Disallow: / Allow: /var/www/html/robots.txt # Something to do with advertising User-agent: AdsBot-Google Disallow: / Allow: /var/www/html/robots.txt User-agent: Mediapartners-Google Disallow: / Allow: /var/www/html/robots.txt ############### ### AI bots ### ############### # Block PiplBot, which is for a people search engine [https://pipl.com/bot] User-agent: PiplBot Disallow: / Allow: /var/www/html/robots.txt User-agent: CCBot Disallow: / Allow: /var/www/html/robots.txt User-agent: Bytespider Disallow: / Allow: /var/www/html/robots.txt User-agent: Amazonbot Disallow: / Allow: /var/www/html/robots.txt User-agent: GPTBot Disallow: / Allow: /var/www/html/robots.txt User-agent: GoogleOther Disallow: / Allow: /var/www/html/robots.txt User-agent: Google-Extended Disallow: / Allow: /var/www/html/robots.txt User-agent: ClaudeBot Disallow: / Allow: /var/www/html/robots.txt User-Agent: PerplexityBot Disallow: / Allow: /var/www/html/robots.txt User-agent: ChatGPT-User Disallow: / Allow: /var/www/html/robots.txt User-agent: Applebot-Extended Disallow: / Allow: /var/www/html/robots.txt User-agent: anthropic-ai Disallow: / Allow: /var/www/html/robots.txt User-agent: Omgilibot Disallow: / Allow: /var/www/html/robots.txt User-agent: Omgili Disallow: / Allow: /var/www/html/robots.txt User-agent: FacebookBot Disallow: / Allow: /var/www/html/robots.txt User-agent: Diffbot Disallow: / Allow: /var/www/html/robots.txt User-agent: ImagesiftBot Disallow: / Allow: /var/www/html/robots.txt User-agent: cohere-ai Disallow: / Allow: /var/www/html/robots.txt User-agent: PetalBot Disallow: / Allow: /var/www/html/robots.txt User-agent: Internet Archive Disallow: / Allow: /var/www/html/robots.txt User-agent: nsa Disallow: / Allow: /var/www/html/robots.txt