# Default crawl delay for unknown bots
User-agent: *
Disallow: /wp-admin/
# https://developers.cloudflare.com/fundamentals/get-started/reference/cdn-cgi-endpoint/
Disallow: /cdn-cgi/
Allow: /wp-admin/admin-ajax.php
Sitemap: https://truthout.org/sitemap_index.xml
Crawl-delay: 10

# Googlebot crawl delay unset
User-agent: googlebot
Disallow: /wp-admin/
Disallow: /cdn-cgi/
Allow: /wp-admin/admin-ajax.php
Sitemap: https://truthout.org/sitemap_index.xml
Crawl-delay: 0

# Googlebot-news crawl delay unset
User-agent: googlebot-news
Disallow: /wp-admin/
Disallow: /cdn-cgi/
Allow: /wp-admin/admin-ajax.php
Sitemap: https://truthout.org/news-sitemap.xml
Crawl-delay: 0

# Bingbot crawl delay unset
User-Agent: bingbot
Disallow: /wp-admin/
Disallow: /cdn-cgi/
Allow: /wp-admin/admin-ajax.php
Sitemap: https://truthout.org/news-sitemap.xml
Crawl-delay: 0