Files
supabase/packages/common/markdown-negotiation.test.ts
Pamela Chia cd52669f1f fix(docs): negotiate /guides/* markdown via shared helper (#45432)
## Summary

This brings docs `/guides/*` to full content negotiation for AI agents
(GROWTH-811):
RFC 9110 q-value parsing instead of a `.includes('text/markdown')`
substring match,
a 406 when the client rejects every type the route can produce, and
markdown rewrites
for known LLM user agents.

I implemented it by extracting the negotiation into a shared
`common/markdown-negotiation`
module consumed by both `apps/docs/middleware.ts` and
`apps/www/middleware.ts`, rather than
duplicating the helpers into docs and keeping them in sync by hand with
www (#45394). Single
source of truth, no re-sync burden. www is refactored onto the shared
helper with no behavior
change.

## Changes

### docs `/guides/*` content negotiation (GROWTH-811)

- Replace the `.includes('text/markdown')` substring match with RFC 9110
q-value parsing.
- Return 406 (`Cache-Control: no-store`, `Vary: Accept`) when Accept
excludes every type the
route serves. Bypassed for LLM user agents, the `.md` suffix, and
clients sending no Accept.
- Rewrite to `/api/guides-md/<slug>` for LLM user agents (Claude-User,
Claude-Web, ChatGPT-User,
  PerplexityBot) regardless of Accept.
- Preserve the existing `.md` suffix routing and the entire
`/reference/*` block.

### Shared negotiation helper

- New `packages/common/markdown-negotiation.ts`:
`negotiateMarkdown(signals, route)` returns
`'markdown' | 'not-acceptable' | 'pass'`. Internalizes q-value parsing,
the LLM user-agent
  match, the UA-length cap, and the markdown-vs-html preference.
- `apps/www/middleware.ts`: refactored to consume the shared helper; its
duplicated copy of the
negotiation helpers (added in #45394) is removed. `.md` early-return,
changelog routing, and
first-referrer cookie stamping are unchanged (no behavior change,
covered by its existing tests).

### Tests

- New `apps/docs/middleware.test.ts`: q-value priority, the 406 path,
`.md` suffix, LLM UA
override, browser default Accept, training-crawler and substring-embed
exclusion, and the
  `/reference/*` exemption.
- New `packages/common/markdown-negotiation.test.ts`: the same decision
matrix at the unit level
(q-values, 406, LLM UAs, `.md`, `*/*`, training crawlers, OWS,
out-of-range q).

## Testing (Vercel preview)

After Vercel posts a preview URL, save it once then run the probe set.

```bash
echo 'PREVIEW_HOST' > /tmp/growth-811-host.txt
HOST=$(cat /tmp/growth-811-host.txt)

# 1) Browser-style Accept -> HTML 200
curl -sI -A "Mozilla/5.0" \
  -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \
  "https://$HOST/docs/guides/auth"

# 2) Accept: text/markdown -> markdown 200
curl -sI -H 'Accept: text/markdown' "https://$HOST/docs/guides/auth"

# 3) text/html;q=1.0, text/markdown;q=0.5 -> HTML 200
curl -sI -H 'Accept: text/html;q=1.0, text/markdown;q=0.5' "https://$HOST/docs/guides/auth"

# 4) unsupported Accept -> 406 + Cache-Control: no-store + Vary: Accept
curl -sI -H 'Accept: application/x-content-negotiation-probe' "https://$HOST/docs/guides/auth"

# 5) User-Agent: Claude-User/1.0 (any Accept) -> markdown 200
curl -sI -A 'Claude-User/1.0' "https://$HOST/docs/guides/auth"
```

### After merge

Run
[acceptmarkdown.com/readiness-check](https://acceptmarkdown.com/readiness-check)
against `https://supabase.com/docs/guides/auth`: expect 100/100.

## Linear

- fixes GROWTH-811
2026-06-09 17:49:35 +08:00

168 lines
5.1 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { negotiateMarkdown } from './markdown-negotiation'
const BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
describe('negotiateMarkdown', () => {
describe('hasMarkdownVariant gate', () => {
it('passes when the route has no markdown variant, regardless of other signals', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/markdown', userAgent: 'Claude-User/1.0' },
{ hasMarkdownVariant: false, isMarkdownSuffix: true }
)
).toBe('pass')
})
})
describe('forced markdown', () => {
it('returns markdown for LLM user agents even when Accept rejects everything', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
})
it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/html', userAgent: '' },
{ hasMarkdownVariant: true, isMarkdownSuffix: true }
)
).toBe('markdown')
})
it.each([
'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
'Claude-Web/1.0',
'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
'PerplexityBot/1.0',
])('treats %s as an LLM agent', (userAgent) => {
expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
'markdown'
)
})
it.each([
'GPTBot/1.0',
'ClaudeBot/1.0',
'CCBot/2.0',
'chatgpt-userscript/2.0',
'NotPerplexityBot',
])('does not treat %s (training crawler / substring embed) as an LLM agent', (userAgent) => {
expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
'pass'
)
})
it('caps user-agent length before matching', () => {
const padded = 'x'.repeat(600) + 'Claude-User'
expect(
negotiateMarkdown({ acceptHeader: '', userAgent: padded }, { hasMarkdownVariant: true })
).toBe('pass')
})
})
describe('no Accept header', () => {
it('passes (serves HTML) when no Accept header is sent', () => {
expect(
negotiateMarkdown({ acceptHeader: '', userAgent: '' }, { hasMarkdownVariant: true })
).toBe('pass')
})
})
describe('406 not-acceptable', () => {
it('returns not-acceptable when Accept excludes every type we serve', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'application/x-content-negotiation-probe', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('not-acceptable')
})
it('does not 406 for bare */*', () => {
expect(
negotiateMarkdown({ acceptHeader: '*/*', userAgent: '' }, { hasMarkdownVariant: true })
).toBe('pass')
})
})
describe('q-value negotiation', () => {
it('serves HTML for browser-style Accept', () => {
expect(
negotiateMarkdown(
{ acceptHeader: BROWSER_ACCEPT, userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
})
it('serves markdown when explicitly requested', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/markdown', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
})
it('serves markdown when its q-value beats html', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
})
it('serves HTML when its q-value beats markdown', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
})
it('breaks an explicit md/html tie toward markdown', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/markdown, text/html, */*', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
})
it('does not serve markdown when the client rejects it (q=0)', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/markdown;q=0, text/html;q=1.0', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
})
it('tolerates OWS around the q parameter (RFC 9110)', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
})
it('ignores out-of-range q-values (falls back to 1.0; tie -> markdown)', () => {
expect(
negotiateMarkdown(
{ acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0', userAgent: '' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
})
})
})