User-agent: * Crawl-delay: 10 Allow: /content/wp-admin/admin-ajax.php User-agent: * Disallow: /content/wp-content/uploads/wpo/wpo-plugins-tables-list.json Allow: /content/wp-content/uploads/ Disallow: /images/ Disallow: /content/ Disallow: /content/wp-admin/ Disallow: /content/wp-includes/ Disallow: /content/wp-content/plugins/ Disallow: /content/wp-content/themes/ Disallow: /content/wp-content/backups/ # Block all bots from crawling URLs with query parameters, except Facebook and Twitter User-agent: * Disallow: /*?* # Allow Facebook and Twitter bots to access URLs with query parameters User-agent: Facebookbot Allow: /*?* User-agent: facebookexternalhit Allow: /*?* User-agent: Twitterbot Allow: /*?* # Allow sitemap crawling Sitemap: /sitemap_index.xml