{ "schemaVersion": 1, "studyId": "robots-sitemap-live-benchmark", "collectedAt": "2026-09-13T23:17:34.398Z", "source": "Public robots.txt files and up to three declared sitemap URLs per fixed-cohort origin; conventional /sitemap.xml used only when none is declared.", "cohortSize": 49, "rows": [ { "origin": "https://ahrefs.com", "robotsStatus": 200, "robotsBytes": 908, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://ahrefs.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://authoritas.com", "robotsStatus": 200, "robotsBytes": 150, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.authoritas.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/rss+xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://botify.com", "robotsStatus": 200, "robotsBytes": 425, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.botify.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/rss+xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://brightedge.com", "robotsStatus": 200, "robotsBytes": 2512, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.brightedge.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml; charset=utf-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://brightlocal.com", "robotsStatus": 403, "robotsBytes": 103, "declarationState": "unknown", "declaredSitemapCount": null, "sitemapChecks": [] }, { "origin": "https://catchpoint.com", "robotsStatus": 200, "robotsBytes": 526, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.catchpoint.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/rss+xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://clearscope.io", "robotsStatus": 200, "robotsBytes": 247, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://clearscope.io/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/html; charset=utf-8", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://cognitiveseo.com", "robotsStatus": 200, "robotsBytes": 0, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://cognitiveseo.com/sitemap.xml", "probeKind": "fallback", "status": 404, "contentType": "text/html", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://conductor.com", "robotsStatus": 200, "robotsBytes": 162, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://www.conductor.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://www.conductor.com/docs/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://debugbear.com", "robotsStatus": 200, "robotsBytes": 182, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.debugbear.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://frase.io", "robotsStatus": 200, "robotsBytes": 1144, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.frase.io/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://gtmetrix.com", "robotsStatus": 200, "robotsBytes": 908, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://gtmetrix.com/sitemap.xml", "probeKind": "declared", "status": 403, "contentType": "text/html; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": false }, { "url": "https://gtmetrix.com/blog/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://jetoctopus.com", "robotsStatus": 200, "robotsBytes": 191, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://jetoctopus.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://jetoctopus.com/sitemap.rss", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://littlewarden.com", "robotsStatus": 200, "robotsBytes": 99, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://littlewarden.com/sitemap.xml", "probeKind": "fallback", "status": 404, "contentType": "application/xml; charset=utf-8", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://localfalcon.com", "robotsStatus": 404, "robotsBytes": 369845, "declarationState": "unknown", "declaredSitemapCount": null, "sitemapChecks": [] }, { "origin": "https://localviking.com", "robotsStatus": 200, "robotsBytes": 195, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://localviking.com/sitemaps/sitemap.xml.gz", "probeKind": "declared", "status": 200, "contentType": "application/octet-stream", "contentEncoding": null, "inspection": "gzip-decompressed", "looksLikeSitemap": true } ] }, { "origin": "https://lumar.io", "robotsStatus": 200, "robotsBytes": 540, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.lumar.io/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://majestic.com", "robotsStatus": 200, "robotsBytes": 517, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://majestic.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "application/xml", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://mangools.com", "robotsStatus": 200, "robotsBytes": 483, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://mangools.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://marketmuse.com", "robotsStatus": 200, "robotsBytes": 134, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.marketmuse.com/wp-sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://moz.com", "robotsStatus": 200, "robotsBytes": 1207, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://moz.com/sitemaps-1-sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://moz.com/community/q/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml; charset=utf-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://oncrawl.com", "robotsStatus": 200, "robotsBytes": 743, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://www.oncrawl.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://fr.oncrawl.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://patrickstox.com", "robotsStatus": 200, "robotsBytes": 2186, "declarationState": "observed", "declaredSitemapCount": 2, "sitemapChecks": [ { "url": "https://patrickstox.com/sitemap-index.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://patrickstox.com/media-sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://pi-datametrics.com", "robotsStatus": 200, "robotsBytes": 651, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://pi-datametrics.com/wp-sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://pingdom.com", "robotsStatus": 200, "robotsBytes": 233, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://pingdom.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://ryte.com", "robotsStatus": 200, "robotsBytes": 213903, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://ryte.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/html; charset=utf-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://screamingfrog.co.uk", "robotsStatus": 200, "robotsBytes": 1883, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://screamingfrog.co.uk/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://searchatlas.com", "robotsStatus": 200, "robotsBytes": 134, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://searchatlas.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://semrush.com", "robotsStatus": 200, "robotsBytes": 3645, "declarationState": "observed", "declaredSitemapCount": 14, "sitemapChecks": [ { "url": "https://www.semrush.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://pt.semrush.com/blog/sitemap/", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://es.semrush.com/blog/sitemap/", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seobility.net", "robotsStatus": 200, "robotsBytes": 4765, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.seobility.net/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seoclarity.net", "robotsStatus": 200, "robotsBytes": 431, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://seoclarity.net/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "application/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seomonitor.com", "robotsStatus": 200, "robotsBytes": 319, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.seomonitor.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seoptimer.com", "robotsStatus": 200, "robotsBytes": 201, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.seoptimer.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml;charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seositecheckup.com", "robotsStatus": 200, "robotsBytes": 183, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://seositecheckup.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://seranking.com", "robotsStatus": 200, "robotsBytes": 2385, "declarationState": "observed", "declaredSitemapCount": 19, "sitemapChecks": [ { "url": "https://seranking.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://seranking.com/blog/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://seranking.com/de/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://serpstat.com", "robotsStatus": 200, "robotsBytes": 568, "declarationState": "observed", "declaredSitemapCount": 4, "sitemapChecks": [ { "url": "https://serpstat.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml;charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://serpstat.com/ru/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml;charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://serpstat.com/de/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml;charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://similarweb.com", "robotsStatus": 200, "robotsBytes": 1026, "declarationState": "observed", "declaredSitemapCount": 3, "sitemapChecks": [ { "url": "https://www.similarweb.com/corp/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://www.similarweb.com/blog/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://www.similarweb.com/sitemaps/sitemap_index.xml.gz", "probeKind": "declared", "status": 200, "contentType": "binary/octet-stream", "contentEncoding": null, "inspection": "gzip-decompressed", "looksLikeSitemap": true } ] }, { "origin": "https://sistrix.com", "robotsStatus": 200, "robotsBytes": 440, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.sistrix.com/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://sitebulb.com", "robotsStatus": 200, "robotsBytes": 237, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://sitebulb.com/sitemap", "probeKind": "declared", "status": 200, "contentType": "application/xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://sitechecker.pro", "robotsStatus": 200, "robotsBytes": 1786, "declarationState": "observed", "declaredSitemapCount": 13, "sitemapChecks": [ { "url": "https://sitechecker.pro/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://sitechecker.pro/it/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://sitechecker.pro/de/sitemap_index.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=UTF-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://siteimprove.com", "robotsStatus": 200, "robotsBytes": 328, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.siteimprove.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://speedcurve.com", "robotsStatus": 200, "robotsBytes": 23, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://speedcurve.com/sitemap.xml", "probeKind": "fallback", "status": 404, "contentType": "text/html; charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://spyfu.com", "robotsStatus": 200, "robotsBytes": 22246, "declarationState": "observed", "declaredSitemapCount": 5, "sitemapChecks": [ { "url": "https://www.spyfu.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://www.spyfu.com/blog/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": null, "inspection": "plain", "looksLikeSitemap": true }, { "url": "https://www.spyfu.com/SpyGPT/SpyGPT_Software_Sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://surferseo.com", "robotsStatus": 200, "robotsBytes": 144, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://surferseo.com/sitemap.xml", "probeKind": "fallback", "status": 200, "contentType": "application/rss+xml; charset=utf-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://uptrends.com", "robotsStatus": 200, "robotsBytes": 147, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.uptrends.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://webpagetest.org", "robotsStatus": 200, "robotsBytes": 350, "declarationState": "observed", "declaredSitemapCount": 0, "sitemapChecks": [ { "url": "https://webpagetest.org/sitemap.xml", "probeKind": "fallback", "status": 404, "contentType": "text/html", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": false } ] }, { "origin": "https://whitespark.ca", "robotsStatus": 200, "robotsBytes": 111, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://whitespark.ca/sitemaps.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml;charset=UTF-8", "contentEncoding": "br", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://woorank.com", "robotsStatus": 200, "robotsBytes": 635, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.woorank.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "application/xml", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] }, { "origin": "https://yext.com", "robotsStatus": 200, "robotsBytes": 544, "declarationState": "observed", "declaredSitemapCount": 1, "sitemapChecks": [ { "url": "https://www.yext.com/sitemap.xml", "probeKind": "declared", "status": 200, "contentType": "text/xml; charset=utf-8", "contentEncoding": "gzip", "inspection": "plain", "looksLikeSitemap": true } ] } ] }