fix(seo): exclude documentation artifacts from the sitemap

src/utils/sitemap-docs.ts

@@ -0,0 +1,20 @@
+const LOW_VALUE_DOCS_SITEMAP_SEGMENTS = new Set(['examples', 'community'])
+
+const LOW_VALUE_DOCS_SITEMAP_SLUGS = new Set([
+  'community-resources',
+  'contributors',
+  'npm-stats',
+])
+
+export function isHighValueDocsSlug(slug: string) {
+  const segments = slug.split('/').filter(Boolean)
+
+  return (
+    segments.length > 0 &&
+    !segments.some((segment) =>
+      /^(?:drafts|readme)$|\.draft$/i.test(segment),
+    ) &&
+    !segments.some((segment) => LOW_VALUE_DOCS_SITEMAP_SEGMENTS.has(segment)) &&
+    !LOW_VALUE_DOCS_SITEMAP_SLUGS.has(slug)
+  )
+}

src/utils/sitemap.ts

@@ -1,3 +1,4 @@
+import { isHighValueDocsSlug } from './sitemap-docs'
 import { getStartExampleSitemapEntries } from './start-example-pages'
 import { getBranch, libraries } from '~/libraries'
 import type { LibrarySlim } from '~/libraries/types'
@@ -23,14 +24,6 @@ const HIGH_VALUE_NON_DOC_PAGES = [
   '/paid-support',
 ] as const satisfies ReadonlyArray<string>
 
-const LOW_VALUE_DOCS_SITEMAP_SEGMENTS = new Set(['examples', 'community'])
-
-const LOW_VALUE_DOCS_SITEMAP_SLUGS = new Set([
-  'community-resources',
-  'contributors',
-  'npm-stats',
-])
-
 function trimTrailingSlash(url: string) {
   return url.replace(/\/$/, '')
 }
@@ -62,16 +55,6 @@ function getLibraryEntries(): Array<SitemapEntry> {
   })
 }
 
-function isHighValueDocsSlug(slug: string) {
-  const segments = slug.split('/').filter(Boolean)
-
-  return (
-    segments.length > 0 &&
-    !segments.some((segment) => LOW_VALUE_DOCS_SITEMAP_SEGMENTS.has(segment)) &&
-    !LOW_VALUE_DOCS_SITEMAP_SLUGS.has(slug)
-  )
-}
-
 async function getLibraryDocsEntries(
   library: LibrarySlim,
 ): Promise<Array<SitemapEntry>> {

tests/sitemap-docs.test.ts

@@ -0,0 +1,36 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { isHighValueDocsSlug } from '../src/utils/sitemap-docs'
+
+test('sitemap excludes repository artifacts while retaining public guides', () => {
+  for (const slug of [
+    'how-to/drafts/README',
+    'how-to/drafts/search-params-in-forms.draft',
+    'how-to/README',
+    'contributing/designs/source-failure/README',
+    'guide.draft',
+    'guides/readme',
+  ]) {
+    assert.equal(isHighValueDocsSlug(slug), false, slug)
+  }
+  for (const slug of [
+    'overview',
+    'framework/react/guides/query-keys',
+    'guides/draft',
+    'guides/readme-format',
+  ]) {
+    assert.equal(isHighValueDocsSlug(slug), true, slug)
+  }
+})
+test('existing sitemap exclusions remain excluded', () => {
+  for (const slug of [
+    '',
+    'examples/basic',
+    'community/resources',
+    'community-resources',
+    'contributors',
+    'npm-stats',
+  ]) {
+    assert.equal(isHighValueDocsSlug(slug), false, slug)
+  }
+})