mirror of
https://github.com/supabase/supabase.git
synced 2026-07-14 03:30:05 -04:00
2aa1b52234
## Problem Moving the Logs Explorer to ClickHouse means users' saved BigQuery queries no longer run. <img width="2430" height="1010" alt="CleanShot 2026-06-29 at 11 36 04@2x" src="https://github.com/user-attachments/assets/ae0ab155-7d3d-4ae9-81c3-22bf3a88cf8c" /> ## Fix Rewrite the query with AI instead of a SQL transpiler. AI handles the long tail of nested fields and dialect differences far better than a rule-based rewriter, and it needs no extra runtime dependency. - `rewriteLogsSqlWithAI` posts the current query to `/api/ai/code/complete` with `dialect: 'clickhouse'`. The endpoint skips the Postgres schema and best-practices for that dialect and uses logs-specific instructions and model so the output is ClickHouse logs SQL (FROM `logs` + `source` filter, no `unnest` joins, nested fields read from `log_attributes['...']`). - The query's `source` is detected and its real `log_attributes` keys are fetched and passed to the model, so it maps to exact paths instead of guessing. - The rewrite runs in the background and is proposed as a side-by-side accept/discard diff in the editor. The AI Assistant panel is not opened. - Entry points: a banner shown only for legacy-looking queries (dismissal persisted), and a "Fix Query" button next to Field Reference. - The Field Reference drawers discover `log_attributes` keys from real data so the listed fields match what the source actually emits. ## Dependencies Built on top of #47265 (Logs Explorer -> OTEL endpoint) — that is the base branch of this PR. Merge #47265 first. Behind `otelLegacyLogs` (off by default). Part of DEBUG-145 (split from #47087). ## How to test - Open the Logs Explorer with a BigQuery logs query (the templates have some), click "Fix Query", and confirm the diff shows valid ClickHouse SQL. Accept it and confirm the applied query runs. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added an OTEL legacy logs workflow (behind a feature flag) with an interactive banner and a “Fix Query” ClickHouse rewrite action, including an accept/discard diff review overlay. * Introduced OTEL-aware field reference rendering with dynamic discovery of `log_attributes` keys and updated OTEL source insertion behavior. * Enabled dialect-aware SQL completion for ClickHouse logs, using logs-specific instructions and output constraints. * **Bug Fixes** * Improved rewrite flow validation and handling, including log source detection and cleanup of AI-generated SQL formatting. * **Tests** * Added Vitest coverage for rewrite prompt generation, detection/classification utilities, SQL fence stripping, OTEL field mapping, and OTEL log attribute key discovery. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
103 lines
3.5 KiB
TypeScript
103 lines
3.5 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
|
|
import { otelFieldsFromKeys, toOtelFieldSchemas, type LogFieldSchema } from './Logs.fieldReference'
|
|
|
|
const edgeSchema: LogFieldSchema = {
|
|
name: 'API Gateway',
|
|
reference: 'edge_logs',
|
|
fields: [
|
|
{ path: 'id', type: 'string' },
|
|
{ path: 'timestamp', type: 'datetime' },
|
|
{ path: 'event_message', type: 'string' },
|
|
{ path: 'identifier', type: 'string' },
|
|
{ path: 'metadata.request.method', type: 'string' },
|
|
{ path: 'metadata.request.cf.asn', type: 'number' },
|
|
],
|
|
}
|
|
|
|
describe('toOtelFieldSchemas', () => {
|
|
it('keeps source name and reference', () => {
|
|
const [otel] = toOtelFieldSchemas([edgeSchema])
|
|
expect(otel.name).toBe('API Gateway')
|
|
expect(otel.reference).toBe('edge_logs')
|
|
})
|
|
|
|
it('keeps id/timestamp/event_message as real columns and adds source + severity_text', () => {
|
|
const [otel] = toOtelFieldSchemas([edgeSchema])
|
|
const paths = otel.fields.map((f) => f.path)
|
|
expect(paths.slice(0, 5)).toEqual([
|
|
'id',
|
|
'timestamp',
|
|
'event_message',
|
|
'severity_text',
|
|
'source',
|
|
])
|
|
})
|
|
|
|
it('moves non-column fields into log_attributes, dropping the metadata root', () => {
|
|
const [otel] = toOtelFieldSchemas([edgeSchema])
|
|
const paths = otel.fields.map((f) => f.path)
|
|
expect(paths).toContain("log_attributes['identifier']")
|
|
expect(paths).toContain("log_attributes['request.method']")
|
|
expect(paths).toContain("log_attributes['request.cf.asn']")
|
|
expect(paths).not.toContain('metadata.request.method')
|
|
expect(paths).not.toContain('identifier')
|
|
})
|
|
|
|
it('preserves the original field type for casting hints', () => {
|
|
const [otel] = toOtelFieldSchemas([edgeSchema])
|
|
const asn = otel.fields.find((f) => f.path === "log_attributes['request.cf.asn']")
|
|
expect(asn?.type).toBe('number')
|
|
})
|
|
})
|
|
|
|
describe('otelFieldsFromKeys', () => {
|
|
it('starts with the five base OTEL columns', () => {
|
|
const fields = otelFieldsFromKeys(['request.cf.asn'])
|
|
const paths = fields.map((f) => f.path)
|
|
expect(paths.slice(0, 5)).toEqual([
|
|
'id',
|
|
'timestamp',
|
|
'event_message',
|
|
'severity_text',
|
|
'source',
|
|
])
|
|
})
|
|
|
|
it('maps discovered keys to log_attributes lookups', () => {
|
|
const fields = otelFieldsFromKeys(['request.cf.asn', 'response.headers.x-foo'])
|
|
const paths = fields.map((f) => f.path)
|
|
expect(paths).toContain("log_attributes['request.cf.asn']")
|
|
expect(paths).toContain("log_attributes['response.headers.x-foo']")
|
|
})
|
|
|
|
it('excludes keys that are real OTEL columns', () => {
|
|
const fields = otelFieldsFromKeys([
|
|
'id',
|
|
'timestamp',
|
|
'event_message',
|
|
'severity_text',
|
|
'source',
|
|
'trace_id',
|
|
])
|
|
const paths = fields.map((f) => f.path)
|
|
for (const column of ['id', 'timestamp', 'event_message', 'severity_text', 'source']) {
|
|
expect(paths.filter((p) => p === column)).toHaveLength(1)
|
|
expect(paths).not.toContain(`log_attributes['${column}']`)
|
|
}
|
|
expect(paths).toContain("log_attributes['trace_id']")
|
|
})
|
|
|
|
it('escapes quotes and backslashes in discovered keys', () => {
|
|
const fields = otelFieldsFromKeys(["weird'key", 'back\\slash'])
|
|
const paths = fields.map((f) => f.path)
|
|
expect(paths).toContain("log_attributes['weird''key']")
|
|
expect(paths).toContain("log_attributes['back\\\\slash']")
|
|
})
|
|
|
|
it('returns only base fields for an empty key list', () => {
|
|
const fields = otelFieldsFromKeys([])
|
|
expect(fields).toHaveLength(5)
|
|
})
|
|
})
|