*: Add embedded DuckDB storage backend

README.md

@@ -157,6 +157,8 @@ Flags:
       --grpc-headers=KEY=VALUE;...
                                   Additional gRPC headers to send with each
                                   request to the remote store (key=value pairs).
+      --storage-backend="clickhouse"
+                                  Storage backend for profile data.
       --clickhouse-address="localhost:9000"
                                   ClickHouse server address.
       --clickhouse-database="parca"
@@ -167,6 +169,10 @@ Flags:
       --clickhouse-table="stacktraces"
                                   ClickHouse table name for profile data.
       --clickhouse-secure         Use TLS for ClickHouse connection.
+      --duckdb-path=""            Filesystem path for the DuckDB database file.
+                                  Empty means an in-memory database (volatile).
+      --duckdb-table="stacktraces"
+                                  DuckDB table name for profile data.
 ```
 <!-- prettier-ignore-end -->
 

go.mod

@@ -120,6 +120,12 @@ require (
 	github.com/docker/docker v28.5.2+incompatible // indirect
 	github.com/docker/go-connections v0.6.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/duckdb/duckdb-go-bindings v0.1.21 // indirect
+	github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 // indirect
+	github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 // indirect
+	github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 // indirect
+	github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 // indirect
+	github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/efficientgo/core v1.0.0-rc.2 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -139,6 +145,7 @@ require (
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-resty/resty/v2 v2.16.5 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
@@ -183,6 +190,9 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/linode/linodego v1.52.1 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/marcboeker/go-duckdb/arrowmapping v0.0.21 // indirect
+	github.com/marcboeker/go-duckdb/mapping v0.0.21 // indirect
+	github.com/marcboeker/go-duckdb/v2 v2.4.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect

go.sum

@@ -225,6 +225,18 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
 github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/duckdb/duckdb-go-bindings v0.1.21 h1:bOb/MXNT4PN5JBZ7wpNg6hrj9+cuDjWDa4ee9UdbVyI=
+github.com/duckdb/duckdb-go-bindings v0.1.21/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc=
+github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 h1:Sjjhf2F/zCjPF53c2VXOSKk0PzieMriSoyr5wfvr9d8=
+github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA=
+github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 h1:IUk0FFUB6dpWLhlN9hY1mmdPX7Hkn3QpyrAmn8pmS8g=
+github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI=
+github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 h1:Qpc7ZE3n6Nwz30KTvaAwI6nGkXjXmMxBTdFpC8zDEYI=
+github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk=
+github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 h1:eX2DhobAZOgjXkh8lPnKAyrxj8gXd2nm+K71f6KV/mo=
+github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w=
+github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 h1:hhziFnGV7mpA+v5J5G2JnYQ+UWCCP3NQ+OTvxFX10D8=
+github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -332,6 +344,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
 github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
@@ -587,6 +601,12 @@ github.com/m1gwings/treedrawer v0.3.3-beta h1:VeeQ4I90+NL0G2Tga3H4EY4hbOyVP3ID4T
 github.com/m1gwings/treedrawer v0.3.3-beta/go.mod h1:Sebh5tCtjQWAG/B9xWct163vB9pCbBcA1ykaUErDUTY=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/marcboeker/go-duckdb/arrowmapping v0.0.21 h1:geHnVjlsAJGczSWEqYigy/7ARuD+eBtjd0kLN80SPJQ=
+github.com/marcboeker/go-duckdb/arrowmapping v0.0.21/go.mod h1:flFTc9MSqQCh2Xm62RYvG3Kyj29h7OtsTb6zUx1CdK8=
+github.com/marcboeker/go-duckdb/mapping v0.0.21 h1:6woNXZn8EfYdc9Vbv0qR6acnt0TM1s1eFqnrJZVrqEs=
+github.com/marcboeker/go-duckdb/mapping v0.0.21/go.mod h1:q3smhpLyv2yfgkQd7gGHMd+H/Z905y+WYIUjrl29vT4=
+github.com/marcboeker/go-duckdb/v2 v2.4.3 h1:bHUkphPsAp2Bh/VFEdiprGpUekxBNZiWWtK+Bv/ljRk=
+github.com/marcboeker/go-duckdb/v2 v2.4.3/go.mod h1:taim9Hktg2igHdNBmg5vgTfHAlV26z3gBI0QXQOcuyI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

pkg/duckdb/client.go

@@ -0,0 +1,73 @@
+// Copyright 2026 The Parca Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package duckdb
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	_ "github.com/marcboeker/go-duckdb/v2"
+)
+
+// Config holds DuckDB connection configuration.
+//
+// Path is the on-disk file path. An empty Path uses an in-memory database.
+// Table is the name of the profile data table.
+type Config struct {
+	Path  string
+	Table string
+}
+
+// Client wraps a DuckDB database/sql connection.
+type Client struct {
+	db  *sql.DB
+	cfg Config
+}
+
+// NewClient opens a DuckDB connection at cfg.Path (file) or in memory if
+// the path is empty. The on-disk file is created if it doesn't exist.
+func NewClient(_ context.Context, cfg Config) (*Client, error) {
+	dsn := cfg.Path // empty string == in-memory per duckdb-go convention
+	db, err := sql.Open("duckdb", dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open duckdb: %w", err)
+	}
+
+	// DuckDB is single-writer per process. Pin connection count to 1 so
+	// every Appender / Query lands on the same connection and we don't
+	// race against a transient one. Reads still work fine because the
+	// embedded engine is single-process anyway.
+	db.SetMaxOpenConns(1)
+	db.SetMaxIdleConns(1)
+
+	return &Client{db: db, cfg: cfg}, nil
+}
+
+// Close closes the underlying database/sql connection.
+func (c *Client) Close() error { return c.db.Close() }
+
+// DB returns the underlying *sql.DB.
+func (c *Client) DB() *sql.DB { return c.db }
+
+// Table returns the profile data table name.
+func (c *Client) Table() string { return c.cfg.Table }
+
+// EnsureSchema creates the profile data table if it doesn't already exist.
+func (c *Client) EnsureSchema(ctx context.Context) error {
+	if _, err := c.db.ExecContext(ctx, CreateTableSQL(c.cfg.Table)); err != nil {
+		return fmt.Errorf("create table %q: %w", c.cfg.Table, err)
+	}
+	return nil
+}

pkg/duckdb/duckdb_test.go

@@ -0,0 +1,217 @@
+// Copyright 2026 The Parca Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package duckdb_test
+
+import (
+	"context"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/go-kit/log"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+
+	"github.com/parca-dev/parca/pkg/duckdb"
+	"github.com/parca-dev/parca/pkg/profile"
+	"github.com/parca-dev/parca/pkg/symbolizer"
+)
+
+// nopSymbolizer is a no-op symbolizer for tests — synthetic rows ingested
+// here already carry resolved function info.
+type nopSymbolizer struct{}
+
+func (nopSymbolizer) Symbolize(context.Context, symbolizer.SymbolizationRequest) error { return nil }
+
+// buildSampleRecord builds a one-row Arrow record matching the parca write
+// schema. Caller must Release() the returned record.
+func buildSampleRecord(t *testing.T, mem memory.Allocator, ts int64) arrow.RecordBatch {
+	t.Helper()
+
+	schema := profile.BuildArrowSchema([]string{"job"})
+	b := array.NewRecordBuilder(mem, schema)
+	defer b.Release()
+
+	for i, field := range schema.Fields() {
+		switch field.Name {
+		case profile.ColumnDuration:
+			b.Field(i).(*array.Int64Builder).Append(int64(time.Second))
+		case profile.ColumnName:
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("process_cpu"))
+		case profile.ColumnPeriod:
+			b.Field(i).(*array.Int64Builder).Append(10_000_000)
+		case profile.ColumnPeriodType:
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("cpu"))
+		case profile.ColumnPeriodUnit:
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("nanoseconds"))
+		case profile.ColumnSampleType:
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("cpu"))
+		case profile.ColumnSampleUnit:
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("nanoseconds"))
+		case profile.ColumnStacktrace:
+			lb := b.Field(i).(*array.ListBuilder)
+			vb := lb.ValueBuilder().(*array.BinaryDictionaryBuilder)
+			lb.Append(true)
+			require.NoError(t, vb.Append(encodeLocation(0xdeadbeef, "test-build-id", "/lib/test", "main.run")))
+		case profile.ColumnTimestamp:
+			b.Field(i).(*array.Int64Builder).Append(ts)
+		case profile.ColumnTimeNanos:
+			b.Field(i).(*array.Int64Builder).Append(ts * int64(time.Millisecond))
+		case profile.ColumnValue:
+			b.Field(i).(*array.Int64Builder).Append(42)
+		case profile.ColumnLabelsPrefix + "job":
+			require.NoError(t, b.Field(i).(*array.BinaryDictionaryBuilder).AppendString("test"))
+		}
+	}
+
+	return b.NewRecordBatch()
+}
+
+// encodeLocation produces a varint-encoded location blob in the same
+// shape produced by the symbolizer and decoded by the ingester.
+//
+// Layout (matches pkg/profile + pkg/clickhouse decoders):
+//
+//	addr (uvarint)
+//	numLines (uvarint)
+//	hasMapping byte (0|1)
+//	  if hasMapping: buildID (len+bytes), filename (len+bytes), 3 zero uvarints
+//	per line:
+//	  lineNumber (uvarint)
+//	  hasFunction byte (0|1)
+//	  if hasFunction: startLine (uvarint), name, systemName, filename (each len+bytes)
+func encodeLocation(addr uint64, buildID, mappingFile, fnName string) []byte {
+	var out []byte
+	out = appendUvarint(out, addr)
+	out = appendUvarint(out, 1) // 1 line
+	out = append(out, 0x01)     // hasMapping
+	out = appendBytes(out, []byte(buildID))
+	out = appendBytes(out, []byte(mappingFile))
+	out = appendUvarint(out, 0) // memoryStart
+	out = appendUvarint(out, 0) // memoryLength
+	out = appendUvarint(out, 0) // mappingOffset
+
+	out = appendUvarint(out, 7) // line number
+	out = append(out, 0x01)     // hasFunction
+	out = appendUvarint(out, 1) // startLine
+	out = appendBytes(out, []byte(fnName))
+	out = appendBytes(out, []byte(fnName)) // system name
+	out = appendBytes(out, []byte("main.go"))
+	return out
+}
+
+func appendUvarint(b []byte, v uint64) []byte {
+	for v >= 0x80 {
+		b = append(b, byte(v)|0x80)
+		v >>= 7
+	}
+	return append(b, byte(v))
+}
+
+func appendBytes(b []byte, payload []byte) []byte {
+	b = appendUvarint(b, uint64(len(payload)))
+	return append(b, payload...)
+}
+
+func newTestClient(t *testing.T) *duckdb.Client {
+	t.Helper()
+	c, err := duckdb.NewClient(context.Background(), duckdb.Config{Path: "", Table: "stacktraces"})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = c.Close() })
+	require.NoError(t, c.EnsureSchema(context.Background()))
+	return c
+}
+
+// TestIngestAndQueryRoundTrip ingests one synthetic profile row and
+// verifies the live Querier methods return sensible results against it.
+func TestIngestAndQueryRoundTrip(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	client := newTestClient(t)
+	ctx := context.Background()
+	logger := log.NewNopLogger()
+	tracer := noop.NewTracerProvider().Tracer("")
+
+	const tsMillis int64 = 1_700_000_000_000
+	rec := buildSampleRecord(t, mem, tsMillis)
+	defer rec.Release()
+
+	ing := duckdb.NewIngester(logger, client)
+	require.NoError(t, ing.Ingest(ctx, rec))
+
+	q := duckdb.NewQuerier(client, logger, tracer, mem, nopSymbolizer{})
+
+	// HasProfileData / ProfileTypes
+	has, err := q.HasProfileData(ctx)
+	require.NoError(t, err)
+	require.True(t, has)
+
+	types, err := q.ProfileTypes(ctx, time.UnixMilli(0), time.UnixMilli(0))
+	require.NoError(t, err)
+	require.Len(t, types, 1)
+	require.Equal(t, "process_cpu", types[0].Name)
+	require.Equal(t, "cpu", types[0].SampleType)
+	require.True(t, types[0].Delta)
+
+	// Labels / Values
+	labels, err := q.Labels(ctx, nil, time.UnixMilli(0), time.UnixMilli(0), "")
+	require.NoError(t, err)
+	require.Equal(t, []string{"job"}, labels)
+
+	values, err := q.Values(ctx, "job", nil, time.UnixMilli(0), time.UnixMilli(0), "")
+	require.NoError(t, err)
+	sort.Strings(values)
+	require.Equal(t, []string{"test"}, values)
+
+	// QueryRange — wide window covering tsMillis.
+	start := time.UnixMilli(tsMillis - 1_000)
+	end := time.UnixMilli(tsMillis + 1_000)
+	queryStr := `process_cpu:cpu:nanoseconds:cpu:nanoseconds:delta{job="test"}`
+	series, err := q.QueryRange(ctx, queryStr, start, end, time.Second, 0, []string{"job"})
+	require.NoError(t, err)
+	require.Len(t, series, 1)
+	require.Equal(t, "job", series[0].Labelset.Labels[0].Name)
+	require.Equal(t, "test", series[0].Labelset.Labels[0].Value)
+	require.NotEmpty(t, series[0].Samples)
+	require.Equal(t, int64(42), series[0].Samples[0].Value)
+
+	// QuerySingle — exact timestamp match.
+	single, err := q.QuerySingle(ctx, queryStr, time.UnixMilli(tsMillis), false)
+	require.NoError(t, err)
+	require.NotEmpty(t, single.Samples)
+	for _, r := range single.Samples {
+		r.Release()
+	}
+
+	// QueryMerge — sum across the window.
+	merged, err := q.QueryMerge(ctx, queryStr, start, end, []string{"job"}, false, "")
+	require.NoError(t, err)
+	require.NotEmpty(t, merged.Samples)
+	for _, r := range merged.Samples {
+		r.Release()
+	}
+
+	// Metadata helpers
+	mappings, err := q.GetProfileMetadataMappings(ctx, queryStr, start, end)
+	require.NoError(t, err)
+	require.Equal(t, []string{"/lib/test"}, mappings)
+
+	metaLabels, err := q.GetProfileMetadataLabels(ctx, queryStr, start, end)
+	require.NoError(t, err)
+	require.Equal(t, []string{"job"}, metaLabels)
+}