# Veridian robots.txt file #=========================================================# # Block all web crawlers from accessing the site #=========================================================# User-agent: * Disallow: / # Applebot needs to be specifically told to stop; otherwise, it follows Googlebot User-agent: Applebot Disallow: / # Explicitly state that it is not for Google’s machine learning purposes User-agent: Google-Extended Disallow: / #=========================================================# # Only allow the following bots #=========================================================# User-agent: archive.org_bot Sitemap: https://www.archive.thecall.org/sitemap-workerscircle-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: bingbot Sitemap: https://www.archive.thecall.org/sitemap-workerscircle-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: Googlebot Sitemap: https://www.archive.thecall.org/sitemap-workerscircle-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: Yahoo Sitemap: https://www.archive.thecall.org/sitemap-workerscircle-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests