Files
supabase/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.test.ts
Jordi Enric 2aa1b52234 feat(studio): add feature to rewrite queries DEBUG-145 (#47266)
## Problem

Moving the Logs Explorer to ClickHouse means users' saved BigQuery
queries no longer run.
<img width="2430" height="1010" alt="CleanShot 2026-06-29 at 11 36
04@2x"
src="https://github.com/user-attachments/assets/ae0ab155-7d3d-4ae9-81c3-22bf3a88cf8c"
/>

## Fix

Rewrite the query with AI instead of a SQL transpiler. AI handles the
long tail of nested fields and dialect differences far better than a
rule-based rewriter, and it needs no extra runtime dependency.

- `rewriteLogsSqlWithAI` posts the current query to
`/api/ai/code/complete` with `dialect: 'clickhouse'`. The endpoint skips
the Postgres schema and best-practices for that dialect and uses
logs-specific instructions and model so the output is ClickHouse logs
SQL (FROM `logs` + `source` filter, no `unnest` joins, nested fields
read from `log_attributes['...']`).
- The query's `source` is detected and its real `log_attributes` keys
are fetched and passed to the model, so it maps to exact paths instead
of guessing.
- The rewrite runs in the background and is proposed as a side-by-side
accept/discard diff in the editor. The AI Assistant panel is not opened.
- Entry points: a banner shown only for legacy-looking queries
(dismissal persisted), and a "Fix Query" button next to Field Reference.
- The Field Reference drawers discover `log_attributes` keys from real
data so the listed fields match what the source actually emits.

## Dependencies

Built on top of #47265 (Logs Explorer -> OTEL endpoint) — that is the
base branch of this PR. Merge #47265 first. Behind `otelLegacyLogs` (off
by default).

Part of DEBUG-145 (split from #47087).

## How to test

- Open the Logs Explorer with a BigQuery logs query (the templates have
some), click "Fix Query", and confirm the diff shows valid ClickHouse
SQL. Accept it and confirm the applied query runs.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added an OTEL legacy logs workflow (behind a feature flag) with an
interactive banner and a “Fix Query” ClickHouse rewrite action,
including an accept/discard diff review overlay.
* Introduced OTEL-aware field reference rendering with dynamic discovery
of `log_attributes` keys and updated OTEL source insertion behavior.
* Enabled dialect-aware SQL completion for ClickHouse logs, using
logs-specific instructions and output constraints.
* **Bug Fixes**
* Improved rewrite flow validation and handling, including log source
detection and cleanup of AI-generated SQL formatting.
* **Tests**
* Added Vitest coverage for rewrite prompt generation,
detection/classification utilities, SQL fence stripping, OTEL field
mapping, and OTEL log attribute key discovery.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
2026-06-29 14:31:18 +02:00

103 lines
3.5 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { otelFieldsFromKeys, toOtelFieldSchemas, type LogFieldSchema } from './Logs.fieldReference'
const edgeSchema: LogFieldSchema = {
name: 'API Gateway',
reference: 'edge_logs',
fields: [
{ path: 'id', type: 'string' },
{ path: 'timestamp', type: 'datetime' },
{ path: 'event_message', type: 'string' },
{ path: 'identifier', type: 'string' },
{ path: 'metadata.request.method', type: 'string' },
{ path: 'metadata.request.cf.asn', type: 'number' },
],
}
describe('toOtelFieldSchemas', () => {
it('keeps source name and reference', () => {
const [otel] = toOtelFieldSchemas([edgeSchema])
expect(otel.name).toBe('API Gateway')
expect(otel.reference).toBe('edge_logs')
})
it('keeps id/timestamp/event_message as real columns and adds source + severity_text', () => {
const [otel] = toOtelFieldSchemas([edgeSchema])
const paths = otel.fields.map((f) => f.path)
expect(paths.slice(0, 5)).toEqual([
'id',
'timestamp',
'event_message',
'severity_text',
'source',
])
})
it('moves non-column fields into log_attributes, dropping the metadata root', () => {
const [otel] = toOtelFieldSchemas([edgeSchema])
const paths = otel.fields.map((f) => f.path)
expect(paths).toContain("log_attributes['identifier']")
expect(paths).toContain("log_attributes['request.method']")
expect(paths).toContain("log_attributes['request.cf.asn']")
expect(paths).not.toContain('metadata.request.method')
expect(paths).not.toContain('identifier')
})
it('preserves the original field type for casting hints', () => {
const [otel] = toOtelFieldSchemas([edgeSchema])
const asn = otel.fields.find((f) => f.path === "log_attributes['request.cf.asn']")
expect(asn?.type).toBe('number')
})
})
describe('otelFieldsFromKeys', () => {
it('starts with the five base OTEL columns', () => {
const fields = otelFieldsFromKeys(['request.cf.asn'])
const paths = fields.map((f) => f.path)
expect(paths.slice(0, 5)).toEqual([
'id',
'timestamp',
'event_message',
'severity_text',
'source',
])
})
it('maps discovered keys to log_attributes lookups', () => {
const fields = otelFieldsFromKeys(['request.cf.asn', 'response.headers.x-foo'])
const paths = fields.map((f) => f.path)
expect(paths).toContain("log_attributes['request.cf.asn']")
expect(paths).toContain("log_attributes['response.headers.x-foo']")
})
it('excludes keys that are real OTEL columns', () => {
const fields = otelFieldsFromKeys([
'id',
'timestamp',
'event_message',
'severity_text',
'source',
'trace_id',
])
const paths = fields.map((f) => f.path)
for (const column of ['id', 'timestamp', 'event_message', 'severity_text', 'source']) {
expect(paths.filter((p) => p === column)).toHaveLength(1)
expect(paths).not.toContain(`log_attributes['${column}']`)
}
expect(paths).toContain("log_attributes['trace_id']")
})
it('escapes quotes and backslashes in discovered keys', () => {
const fields = otelFieldsFromKeys(["weird'key", 'back\\slash'])
const paths = fields.map((f) => f.path)
expect(paths).toContain("log_attributes['weird''key']")
expect(paths).toContain("log_attributes['back\\\\slash']")
})
it('returns only base fields for an empty key list', () => {
const fields = otelFieldsFromKeys([])
expect(fields).toHaveLength(5)
})
})