mirror of
https://github.com/supabase/supabase.git
synced 2026-06-28 03:19:09 -04:00
cd52669f1f
## Summary
This brings docs `/guides/*` to full content negotiation for AI agents
(GROWTH-811):
RFC 9110 q-value parsing instead of a `.includes('text/markdown')`
substring match,
a 406 when the client rejects every type the route can produce, and
markdown rewrites
for known LLM user agents.
I implemented it by extracting the negotiation into a shared
`common/markdown-negotiation`
module consumed by both `apps/docs/middleware.ts` and
`apps/www/middleware.ts`, rather than
duplicating the helpers into docs and keeping them in sync by hand with
www (#45394). Single
source of truth, no re-sync burden. www is refactored onto the shared
helper with no behavior
change.
## Changes
### docs `/guides/*` content negotiation (GROWTH-811)
- Replace the `.includes('text/markdown')` substring match with RFC 9110
q-value parsing.
- Return 406 (`Cache-Control: no-store`, `Vary: Accept`) when Accept
excludes every type the
route serves. Bypassed for LLM user agents, the `.md` suffix, and
clients sending no Accept.
- Rewrite to `/api/guides-md/<slug>` for LLM user agents (Claude-User,
Claude-Web, ChatGPT-User,
PerplexityBot) regardless of Accept.
- Preserve the existing `.md` suffix routing and the entire
`/reference/*` block.
### Shared negotiation helper
- New `packages/common/markdown-negotiation.ts`:
`negotiateMarkdown(signals, route)` returns
`'markdown' | 'not-acceptable' | 'pass'`. Internalizes q-value parsing,
the LLM user-agent
match, the UA-length cap, and the markdown-vs-html preference.
- `apps/www/middleware.ts`: refactored to consume the shared helper; its
duplicated copy of the
negotiation helpers (added in #45394) is removed. `.md` early-return,
changelog routing, and
first-referrer cookie stamping are unchanged (no behavior change,
covered by its existing tests).
### Tests
- New `apps/docs/middleware.test.ts`: q-value priority, the 406 path,
`.md` suffix, LLM UA
override, browser default Accept, training-crawler and substring-embed
exclusion, and the
`/reference/*` exemption.
- New `packages/common/markdown-negotiation.test.ts`: the same decision
matrix at the unit level
(q-values, 406, LLM UAs, `.md`, `*/*`, training crawlers, OWS,
out-of-range q).
## Testing (Vercel preview)
After Vercel posts a preview URL, save it once then run the probe set.
```bash
echo 'PREVIEW_HOST' > /tmp/growth-811-host.txt
HOST=$(cat /tmp/growth-811-host.txt)
# 1) Browser-style Accept -> HTML 200
curl -sI -A "Mozilla/5.0" \
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \
"https://$HOST/docs/guides/auth"
# 2) Accept: text/markdown -> markdown 200
curl -sI -H 'Accept: text/markdown' "https://$HOST/docs/guides/auth"
# 3) text/html;q=1.0, text/markdown;q=0.5 -> HTML 200
curl -sI -H 'Accept: text/html;q=1.0, text/markdown;q=0.5' "https://$HOST/docs/guides/auth"
# 4) unsupported Accept -> 406 + Cache-Control: no-store + Vary: Accept
curl -sI -H 'Accept: application/x-content-negotiation-probe' "https://$HOST/docs/guides/auth"
# 5) User-Agent: Claude-User/1.0 (any Accept) -> markdown 200
curl -sI -A 'Claude-User/1.0' "https://$HOST/docs/guides/auth"
```
### After merge
Run
[acceptmarkdown.com/readiness-check](https://acceptmarkdown.com/readiness-check)
against `https://supabase.com/docs/guides/auth`: expect 100/100.
## Linear
- fixes GROWTH-811
168 lines
5.1 KiB
TypeScript
168 lines
5.1 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
|
|
import { negotiateMarkdown } from './markdown-negotiation'
|
|
|
|
const BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
|
|
|
describe('negotiateMarkdown', () => {
|
|
describe('hasMarkdownVariant gate', () => {
|
|
it('passes when the route has no markdown variant, regardless of other signals', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/markdown', userAgent: 'Claude-User/1.0' },
|
|
{ hasMarkdownVariant: false, isMarkdownSuffix: true }
|
|
)
|
|
).toBe('pass')
|
|
})
|
|
})
|
|
|
|
describe('forced markdown', () => {
|
|
it('returns markdown for LLM user agents even when Accept rejects everything', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
|
|
it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/html', userAgent: '' },
|
|
{ hasMarkdownVariant: true, isMarkdownSuffix: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
|
|
it.each([
|
|
'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
|
|
'Claude-Web/1.0',
|
|
'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
|
|
'PerplexityBot/1.0',
|
|
])('treats %s as an LLM agent', (userAgent) => {
|
|
expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
|
|
'markdown'
|
|
)
|
|
})
|
|
|
|
it.each([
|
|
'GPTBot/1.0',
|
|
'ClaudeBot/1.0',
|
|
'CCBot/2.0',
|
|
'chatgpt-userscript/2.0',
|
|
'NotPerplexityBot',
|
|
])('does not treat %s (training crawler / substring embed) as an LLM agent', (userAgent) => {
|
|
expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
|
|
'pass'
|
|
)
|
|
})
|
|
|
|
it('caps user-agent length before matching', () => {
|
|
const padded = 'x'.repeat(600) + 'Claude-User'
|
|
expect(
|
|
negotiateMarkdown({ acceptHeader: '', userAgent: padded }, { hasMarkdownVariant: true })
|
|
).toBe('pass')
|
|
})
|
|
})
|
|
|
|
describe('no Accept header', () => {
|
|
it('passes (serves HTML) when no Accept header is sent', () => {
|
|
expect(
|
|
negotiateMarkdown({ acceptHeader: '', userAgent: '' }, { hasMarkdownVariant: true })
|
|
).toBe('pass')
|
|
})
|
|
})
|
|
|
|
describe('406 not-acceptable', () => {
|
|
it('returns not-acceptable when Accept excludes every type we serve', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'application/x-content-negotiation-probe', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('not-acceptable')
|
|
})
|
|
|
|
it('does not 406 for bare */*', () => {
|
|
expect(
|
|
negotiateMarkdown({ acceptHeader: '*/*', userAgent: '' }, { hasMarkdownVariant: true })
|
|
).toBe('pass')
|
|
})
|
|
})
|
|
|
|
describe('q-value negotiation', () => {
|
|
it('serves HTML for browser-style Accept', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: BROWSER_ACCEPT, userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('pass')
|
|
})
|
|
|
|
it('serves markdown when explicitly requested', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/markdown', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
|
|
it('serves markdown when its q-value beats html', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
|
|
it('serves HTML when its q-value beats markdown', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('pass')
|
|
})
|
|
|
|
it('breaks an explicit md/html tie toward markdown', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/markdown, text/html, */*', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
|
|
it('does not serve markdown when the client rejects it (q=0)', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/markdown;q=0, text/html;q=1.0', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('pass')
|
|
})
|
|
|
|
it('tolerates OWS around the q parameter (RFC 9110)', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('pass')
|
|
})
|
|
|
|
it('ignores out-of-range q-values (falls back to 1.0; tie -> markdown)', () => {
|
|
expect(
|
|
negotiateMarkdown(
|
|
{ acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0', userAgent: '' },
|
|
{ hasMarkdownVariant: true }
|
|
)
|
|
).toBe('markdown')
|
|
})
|
|
})
|
|
})
|