From 9a6a5a520032bdc2fcebc22375facb74d7814dec Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:11:33 +0000 Subject: [PATCH 1/3] datagen: child_of columns for hierarchical labels A `child_of: ` column with `fan_out: f` nests under an earlier string column: its value is the parent row's value, a `.`, and a child index in [0, f) drawn from the column's own distribution. Bad indices, a fan_out that disagrees with the domain, and fields that would not mean what they say on a child are refused before drawing. Adds configs/datagen/hydra_hier.yaml (region 4 -> service 25 -> endpoint 25, status 4, user_id value) for the hydra rows, docs, and tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-cli/src/external.rs | 2 + aqpbm-cli/src/flatten_record.rs | 2 + aqpbm-cli/src/main.rs | 2 + aqpbm-cli/src/repeat.rs | 2 + aqpbm-core/schema/merged_record.schema.json | 18 +++ aqpbm-core/src/atomic_costs.rs | 2 + aqpbm-core/src/benchmark_result/schema.rs | 2 + aqpbm-core/src/erp.rs | 2 + aqpbm-datagen/src/column.rs | 91 +++++++++++++ aqpbm-datagen/src/lib.rs | 144 ++++++++++++++++++++ aqpbm-datagen/src/table.rs | 29 +++- configs/datagen/hydra_hier.yaml | 74 ++++++++++ docs/aqpbm-datagen.md | 27 ++++ 13 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 configs/datagen/hydra_hier.yaml diff --git a/aqpbm-cli/src/external.rs b/aqpbm-cli/src/external.rs index 50eb22b1..3f65ca8d 100644 --- a/aqpbm-cli/src/external.rs +++ b/aqpbm-cli/src/external.rs @@ -269,6 +269,8 @@ fn shape_column(data_type: &str) -> ColumnSpec { special_rule: RULE_NONE, data_type: data_type.to_string(), string: None, + child_of: None, + fan_out: None, } } diff --git a/aqpbm-cli/src/flatten_record.rs b/aqpbm-cli/src/flatten_record.rs index c69b1004..00076d05 100644 --- a/aqpbm-cli/src/flatten_record.rs +++ b/aqpbm-cli/src/flatten_record.rs @@ -249,6 +249,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1000, ); diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 8334d4dd..c6315d19 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -258,6 +258,8 @@ fn dataset_spec(args: &SketchbenchArgs) -> Result { }; (opts != StringOpts::default()).then_some(opts) }, + child_of: None, + fan_out: None, }, args.size.expect("clap requires --size") as u64, ))) diff --git a/aqpbm-cli/src/repeat.rs b/aqpbm-cli/src/repeat.rs index 19ae79e9..4ebcec1f 100644 --- a/aqpbm-cli/src/repeat.rs +++ b/aqpbm-cli/src/repeat.rs @@ -191,6 +191,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1000, ); diff --git a/aqpbm-core/schema/merged_record.schema.json b/aqpbm-core/schema/merged_record.schema.json index 4b858a32..dd09e6e3 100644 --- a/aqpbm-core/schema/merged_record.schema.json +++ b/aqpbm-core/schema/merged_record.schema.json @@ -393,6 +393,15 @@ "format": "uint64", "minimum": 0 }, + "child_of": { + "description": "Index of an earlier `string` column this one nests under, for\nhierarchical labels. A value is the parent row's value, a `.`, and a\nchild index in `[0, fan_out)` drawn from this column's distribution.", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0 + }, "data_type": { "description": "Which [`ColumnData`] variant this column renders into.", "type": "string" @@ -400,6 +409,15 @@ "distribution": { "$ref": "#/$defs/DataDistribution" }, + "fan_out": { + "description": "How many children each parent value has. Required with `child_of`, and\nchecked against the distribution's domain like `cardinality` is.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, "shift": { "description": "Added to every value, for a column that should not start where its\ndistribution naturally does. Under the monotonic rule it is the value the\nseries starts from.", "type": [ diff --git a/aqpbm-core/src/atomic_costs.rs b/aqpbm-core/src/atomic_costs.rs index fa5c8ee7..449b0d2f 100644 --- a/aqpbm-core/src/atomic_costs.rs +++ b/aqpbm-core/src/atomic_costs.rs @@ -418,6 +418,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1_000_000, ) diff --git a/aqpbm-core/src/benchmark_result/schema.rs b/aqpbm-core/src/benchmark_result/schema.rs index 6b644160..84472aab 100644 --- a/aqpbm-core/src/benchmark_result/schema.rs +++ b/aqpbm-core/src/benchmark_result/schema.rs @@ -490,6 +490,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1_000_000, ) diff --git a/aqpbm-core/src/erp.rs b/aqpbm-core/src/erp.rs index 6a4e42c8..9bddc906 100644 --- a/aqpbm-core/src/erp.rs +++ b/aqpbm-core/src/erp.rs @@ -181,6 +181,8 @@ mod tests { special_rule: 0, data_type: "u64".into(), string: None, + child_of: None, + fan_out: None, }, 100_000, ), diff --git a/aqpbm-datagen/src/column.rs b/aqpbm-datagen/src/column.rs index a68168c9..07e8899d 100644 --- a/aqpbm-datagen/src/column.rs +++ b/aqpbm-datagen/src/column.rs @@ -36,6 +36,15 @@ pub struct ColumnSpec { /// Rendering options for `data_type: string`. Absent means the defaults. #[serde(default, skip_serializing_if = "Option::is_none")] pub string: Option, + /// Index of an earlier `string` column this one nests under, for + /// hierarchical labels. A value is the parent row's value, a `.`, and a + /// child index in `[0, fan_out)` drawn from this column's distribution. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub child_of: Option, + /// How many children each parent value has. Required with `child_of`, and + /// checked against the distribution's domain like `cardinality` is. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fan_out: Option, } impl ColumnSpec { @@ -53,6 +62,21 @@ impl ColumnSpec { ))); } + match (self.child_of, self.fan_out) { + (None, None) => {} + (Some(_), None) => { + return Err(DataGenError::BadParam( + "child_of: needs fan_out, the number of children per parent value".into(), + )) + } + (None, Some(_)) => { + return Err(DataGenError::BadParam( + "fan_out: applies only to a child_of column".into(), + )) + } + (Some(_), Some(fan_out)) => self.validate_child(fan_out)?, + } + // `cardinality` restates the domain. Silently preferring one over the // other would let a report cite a key space the column never had. if let Some(claimed) = self.cardinality { @@ -98,6 +122,54 @@ impl ColumnSpec { Ok(()) } + /// The checks a `child_of` column adds. Its index of the parent is checked + /// by the table, which knows the other columns. + fn validate_child(&self, fan_out: u64) -> Result<(), DataGenError> { + if self.data_type != "string" { + return Err(DataGenError::BadParam(format!( + "child_of: a child value is text, so data_type has to be `string`, not '{}'", + self.data_type + ))); + } + if fan_out == 0 { + return Err(DataGenError::BadParam("fan_out must be > 0".into())); + } + match self.distribution.domain() { + Some(domain) if domain.size == fan_out => {} + Some(domain) => { + return Err(DataGenError::BadParam(format!( + "fan_out {fan_out} disagrees with the {} domain, which holds {}; the \ + distribution draws the child index", + self.distribution.tag(), + domain.size, + ))) + } + None => { + return Err(DataGenError::BadParam(format!( + "child_of: the child index needs a bounded domain, and {} has none", + self.distribution.tag(), + ))) + } + } + // A child column holds parent cardinality × fan_out values and is not + // rendered from a rank, so neither field would mean what it says. + if self.cardinality.is_some() { + return Err(DataGenError::BadParam( + "child_of: cardinality would restate fan_out, not the column's distinct \ + count (parent cardinality × fan_out); drop it" + .into(), + )); + } + if self.string.is_some() { + return Err(DataGenError::BadParam( + "child_of: a child value is its parent's plus an index, so a `string:` \ + block has nothing to render; drop it" + .into(), + )); + } + Ok(()) + } + /// The raw draw stream: `row_num` values straight from the distribution, /// before any rule, shift or rendering. Split out because a /// `column_connected` group draws this once and every member column renders @@ -163,4 +235,23 @@ impl ColumnSpec { ))), } } + + /// Render a `child_of` column: each row's parent value, a `.`, and the + /// draw's 0-based rank in the domain as the child index. + pub(crate) fn render_child( + &self, + raw: &[f64], + parents: &[String], + ) -> Result { + let domain = self + .distribution + .domain() + .expect("child domains were validated above"); + Ok(ColumnData::String( + raw.iter() + .zip(parents) + .map(|(draw, parent)| format!("{parent}.{}", (draw - domain.lower) as u64)) + .collect(), + )) + } } diff --git a/aqpbm-datagen/src/lib.rs b/aqpbm-datagen/src/lib.rs index 867131a3..f5e2679d 100644 --- a/aqpbm-datagen/src/lib.rs +++ b/aqpbm-datagen/src/lib.rs @@ -61,6 +61,8 @@ mod tests { special_rule: RULE_NONE, data_type: data_type.into(), string: None, + child_of: None, + fan_out: None, } } @@ -502,6 +504,148 @@ mod tests { assert!(err.contains("used twice"), "{err}"); } + // ---------- child_of ---------- + + fn child(parent: usize, fan_out: u64, distribution: DataDistribution) -> ColumnSpec { + ColumnSpec { + child_of: Some(parent), + fan_out: Some(fan_out), + ..column(distribution, "string") + } + } + + /// region (4) → service (fan-out 25, Zipf) → endpoint (fan-out 5), then a value. + fn hier_table() -> TableDescription { + TableDescription { + column_num: 4, + column_label: vec![ + "region".into(), + "service".into(), + "endpoint".into(), + "value".into(), + ], + column_spec: vec![ + column(uniform(0.0, 4.0, 1), "string"), + child(0, 25, zipf(25, 1.2, 2)), + child(1, 5, uniform(0.0, 5.0, 3)), + column(zipf(1000, 1.1, 4), "i64"), + ], + column_connected: Vec::new(), + row_num: 20_000, + } + } + + /// Each child splits into its parent, a `.`, and an index below `fan_out`; + /// returns the distinct children seen under each parent. + fn children_by_parent( + parents: &[String], + children: &[String], + fan_out: u64, + ) -> std::collections::HashMap> { + let mut by_parent: std::collections::HashMap<_, std::collections::HashSet<_>> = + Default::default(); + for (parent, child) in parents.iter().zip(children) { + let (head, index) = child.rsplit_once('.').expect("a child has a `.`"); + assert_eq!(head, parent, "'{child}' does not start with its parent"); + assert!(index.parse::().unwrap() < fan_out, "'{child}'"); + by_parent + .entry(parent.clone()) + .or_default() + .insert(child.clone()); + } + by_parent + } + + #[test] + fn a_child_table_is_deterministic() { + let d = hier_table(); + assert_eq!(d.generate().unwrap(), d.generate().unwrap()); + } + + #[test] + fn every_child_extends_its_parent_within_its_fan_out() { + let t = hier_table().generate().unwrap(); + let region = t.data[0].as_string().unwrap(); + let service = t.data[1].as_string().unwrap(); + let endpoint = t.data[2].as_string().unwrap(); + + let services = children_by_parent(region, service, 25); + assert_eq!(services.len(), 4); + // 20k draws over 25 Zipf(1.2) ranks reach every service under every region. + assert!(services.values().all(|s| s.len() == 25), "{services:?}"); + + let endpoints = children_by_parent(service, endpoint, 5); + assert_eq!(endpoints.len(), 100); + assert!(endpoints.values().all(|e| e.len() <= 5)); + } + + /// The child index is the column's own draw, ranked: the distribution is + /// what picks it, and the parent plays no part. + #[test] + fn the_child_index_follows_the_columns_distribution() { + let d = hier_table(); + let draws = d.column_spec[1].draw(d.row_num as usize).unwrap(); + let t = d.generate().unwrap(); + let service = t.data[1].as_string().unwrap(); + let mut counts = [0u64; 25]; + for (draw, s) in draws.iter().zip(service) { + let index: u64 = s.rsplit_once('.').unwrap().1.parse().unwrap(); + assert_eq!(index, (draw - 1.0) as u64); + counts[index as usize] += 1; + } + // Zipf(1.2) over 25 ranks gives rank 1 about 30% of the mass. + let head = counts[0] as f64 / d.row_num as f64; + assert!((0.25..0.35).contains(&head), "{head}"); + assert!(counts[0] > counts[1] && counts[1] > counts[24]); + } + + #[test] + fn hydra_hier_yaml_generates_its_hierarchy() { + let d = + TableDescription::from_path(std::path::Path::new("../configs/datagen/hydra_hier.yaml")) + .unwrap(); + let t = d.generate().unwrap(); + let col = |i: usize| t.data[i].as_string().unwrap(); + assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4); + assert_eq!(children_by_parent(col(1), col(2), 25).len(), 100); + } + + #[test] + fn bad_child_columns_are_refused_by_name() { + type Edit = fn(&mut TableDescription); + let cases: Vec<(&str, Edit)> = vec![ + ("earlier column", |d| d.column_spec[1].child_of = Some(1)), + ("earlier column", |d| d.column_spec[1].child_of = Some(9)), + ("not a string column", |d| { + d.column_spec[0].data_type = "i64".into() + }), + ("needs fan_out", |d| d.column_spec[1].fan_out = None), + ("only to a child_of", |d| d.column_spec[1].child_of = None), + ("fan_out must be > 0", |d| { + d.column_spec[1].fan_out = Some(0) + }), + ("fan_out 24 disagrees", |d| { + d.column_spec[1].fan_out = Some(24) + }), + ("bounded domain", |d| { + d.column_spec[1].distribution = normal(0.0, 1.0, 2) + }), + ("has to be `string`", |d| { + d.column_spec[1].data_type = "u64".into() + }), + ("cardinality", |d| d.column_spec[1].cardinality = Some(25)), + ("`string:` block", |d| { + d.column_spec[1].string = Some(StringOpts::default()) + }), + ]; + for (want, edit) in cases { + let mut d = hier_table(); + edit(&mut d); + let err = d.generate().unwrap_err().to_string(); + assert!(err.contains(want), "wanted '{want}' in: {err}"); + } + } + // ---------- serde ---------- #[test] diff --git a/aqpbm-datagen/src/table.rs b/aqpbm-datagen/src/table.rs index 9b336c41..cb64c73d 100644 --- a/aqpbm-datagen/src/table.rs +++ b/aqpbm-datagen/src/table.rs @@ -119,6 +119,27 @@ impl TableDescription { })?; } + // A child is generated after its parent, which an earlier index + // guarantees and which also rules out a column nesting under itself. + for (i, spec) in self.column_spec.iter().enumerate() { + let Some(parent) = spec.child_of else { + continue; + }; + let label = &self.column_label[i]; + if parent >= i { + return Err(DataGenError::BadParam(format!( + "column '{label}': child_of {parent} has to name an earlier column, \ + one of 0..{i}" + ))); + } + if self.column_spec[parent].data_type != "string" { + return Err(DataGenError::BadParam(format!( + "column '{label}': child_of names '{}', which is not a string column", + self.column_label[parent] + ))); + } + } + let mut claimed: HashMap<&str, usize> = HashMap::new(); for (g, group) in self.column_connected.iter().enumerate() { if group.len() < 2 { @@ -217,7 +238,7 @@ impl TableDescription { group_bits.insert(g, bits); } - let mut data = Vec::with_capacity(self.column_spec.len()); + let mut data: Vec = Vec::with_capacity(self.column_spec.len()); for (i, spec) in self.column_spec.iter().enumerate() { // Only one of these buffers is initialized. Either one lives just // long enough for this column's render pass. @@ -244,7 +265,11 @@ impl TableDescription { &own_draw } }; - data.push(spec.render(raw).map_err(|e| { + let rendered = match spec.child_of { + Some(parent) => spec.render_child(raw, data[parent].as_string()?), + None => spec.render(raw), + }; + data.push(rendered.map_err(|e| { DataGenError::BadParam(format!("column '{}': {e}", self.column_label[i])) })?); } diff --git a/configs/datagen/hydra_hier.yaml b/configs/datagen/hydra_hier.yaml new file mode 100644 index 00000000..968e6ecd --- /dev/null +++ b/configs/datagen/hydra_hier.yaml @@ -0,0 +1,74 @@ +# Hierarchical labels for the record-ingesting rows (the `hydra-*` algorithms): +# region → service → endpoint, plus an independent status label, then the value +# column. As in `hydra_columns.yaml`, the last column is the value and the ones +# before it are labels. +# +# approxbench sketchbench --variant hydra-cms --library lib \ +# --spec configs/datagen/hydra_hier.yaml \ +# --config "rows=3 cols=1024 cell_rows=3 cell_cols=1024" \ +# --operations query --metrics accuracy +# +# A `child_of` column nests under an earlier label: its value is the parent row's +# value, a `.`, and a child index in `[0, fan_out)` drawn from its own +# distribution. So a service reads `.7` and an endpoint `.7.3`, +# and each parent value has at most `fan_out` children. The index is drawn +# independently of the parent, so every region shares the same service skew. +# +# Four labels cost `2^4 - 1 = 15` cell insertions per record. Distinct label +# values: region 4, service 4 × 25 = 100, endpoint 100 × 25 = 2500, status 4. + +column_num: 5 +column_label: [region, service, endpoint, status, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # endpoint: 25 per service, skewed. + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.2 + population_size: 25 + seed: 3 + + # status: 4 values, heavily skewed toward the first (the "200"). + - data_type: string + string: + min_len: 3 + max_len: 3 + distribution: + kind: zipf + skewness: 2.0 + population_size: 4 + seed: 4 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 diff --git a/docs/aqpbm-datagen.md b/docs/aqpbm-datagen.md index fcccfa78..3ff1f839 100644 --- a/docs/aqpbm-datagen.md +++ b/docs/aqpbm-datagen.md @@ -35,6 +35,8 @@ pub struct ColumnSpec { pub special_rule: u32, // bit mask about special_rule; 0 means no special rules pub data_type: String, // defines which case of enum ColumnData should be chosen from pub string: Option, // if this column is about string data, description about how string looks like + pub child_of: Option, // index of an earlier string column this one nests under; see child_of below + pub fan_out: Option, // children per parent value; required with child_of } enum DataDistribution { @@ -83,6 +85,31 @@ Thus, the field `column_connected` is used to describe such relation. The requirement is: columns related to each other needs to have identical distribution and distribution parameter. They will be generated once, and processed to meet the need. +### child_of + +Labels are often hierarchical: a service lives in one region, an endpoint in one service. +A column with `child_of: ` and `fan_out: f` nests under the earlier column at that index: +its value is the parent row's value, a `.`, and a child index in `[0, f)`. +The child index is drawn from the column's own `distribution` (its 0-based rank in the domain), so the usual skew and seed options pick it, +independently of the parent: every parent value shares one child distribution and has at most `f` distinct children. + +```yaml +- data_type: string + child_of: 0 # nests under column 0, e.g. "abcd" -> "abcd.7" + fan_out: 25 + distribution: {kind: zipf, skewness: 1.1, population_size: 25, seed: 2} +``` + +Each of the following is a description error: + +- `child_of` without `fan_out`, or `fan_out` without `child_of` +- `child_of` naming the column itself, a later column, or no column (a parent is generated first) +- a parent that is not `data_type: string`, or a child that is not +- `fan_out` of 0, or one that disagrees with the distribution's domain size (an unbounded distribution has none) +- `cardinality` or a `string:` block on a child column (its distinct count is the parent's times `fan_out`, and it is not rendered from a rank) + +`configs/datagen/hydra_hier.yaml` uses it for a region → service → endpoint hierarchy. + ## Output The output is a struct containing data: From 4b40e31c995a708d0dbf0e935544f02b45cdfdc2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:18:40 +0000 Subject: [PATCH 2/3] datagen: whole uniform bounds for child_of; 2- and 3-label hierarchy cuts A uniform draw is continuous, so `uniform{0.0, 2.9}` with fan_out 2 drew child indices 0, 1 and 2; child_of now requires whole uniform bounds. hydra_hier_d2.yaml and hydra_hier_d3.yaml cut hydra_hier.yaml to its first 2 and 3 labels for the Hydra study's schema-width sweep. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-datagen/src/column.rs | 10 +++++++ aqpbm-datagen/src/lib.rs | 16 ++++++++++ configs/datagen/hydra_hier_d2.yaml | 38 +++++++++++++++++++++++ configs/datagen/hydra_hier_d3.yaml | 48 ++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 configs/datagen/hydra_hier_d2.yaml create mode 100644 configs/datagen/hydra_hier_d3.yaml diff --git a/aqpbm-datagen/src/column.rs b/aqpbm-datagen/src/column.rs index 07e8899d..092bea87 100644 --- a/aqpbm-datagen/src/column.rs +++ b/aqpbm-datagen/src/column.rs @@ -134,6 +134,16 @@ impl ColumnSpec { if fan_out == 0 { return Err(DataGenError::BadParam("fan_out must be > 0".into())); } + // A uniform draw is continuous; only whole bounds keep its index in + // `[0, fan_out)` (`uniform{0.0, 2.9}` would draw 0, 1 and 2). + if let DataDistribution::Uniform(p) = &self.distribution { + if p.lower_bound.fract() != 0.0 || p.upper_bound.fract() != 0.0 { + return Err(DataGenError::BadParam(format!( + "child_of: uniform bounds must be whole numbers, got [{}, {})", + p.lower_bound, p.upper_bound + ))); + } + } match self.distribution.domain() { Some(domain) if domain.size == fan_out => {} Some(domain) => { diff --git a/aqpbm-datagen/src/lib.rs b/aqpbm-datagen/src/lib.rs index f5e2679d..e90ac559 100644 --- a/aqpbm-datagen/src/lib.rs +++ b/aqpbm-datagen/src/lib.rs @@ -610,6 +610,19 @@ mod tests { assert_eq!(children_by_parent(col(1), col(2), 25).len(), 100); } + /// The schema-width cuts keep the first labels and the value column. + #[test] + fn hydra_hier_cuts_keep_their_leading_labels() { + for (file, labels) in [("hydra_hier_d2.yaml", 2), ("hydra_hier_d3.yaml", 3)] { + let path = format!("../configs/datagen/{file}"); + let d = TableDescription::from_path(std::path::Path::new(&path)).unwrap(); + assert_eq!(d.column_num, labels + 1, "{file}"); + let t = d.generate().unwrap(); + let col = |i: usize| t.data[i].as_string().unwrap(); + assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4, "{file}"); + } + } + #[test] fn bad_child_columns_are_refused_by_name() { type Edit = fn(&mut TableDescription); @@ -633,6 +646,9 @@ mod tests { ("has to be `string`", |d| { d.column_spec[1].data_type = "u64".into() }), + ("whole numbers", |d| { + d.column_spec[2].distribution = uniform(0.0, 5.9, 3) + }), ("cardinality", |d| d.column_spec[1].cardinality = Some(25)), ("`string:` block", |d| { d.column_spec[1].string = Some(StringOpts::default()) diff --git a/configs/datagen/hydra_hier_d2.yaml b/configs/datagen/hydra_hier_d2.yaml new file mode 100644 index 00000000..e0454cc6 --- /dev/null +++ b/configs/datagen/hydra_hier_d2.yaml @@ -0,0 +1,38 @@ +# `hydra_hier.yaml` cut to 2 labels (region, service), for the Hydra +# study's schema-width sweep: `2^2 - 1 = 3` cell insertions per record. +# Same columns, distributions and seeds as `hydra_hier.yaml`, which has 4 labels. + +column_num: 3 +column_label: [region, service, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 diff --git a/configs/datagen/hydra_hier_d3.yaml b/configs/datagen/hydra_hier_d3.yaml new file mode 100644 index 00000000..b9257cb7 --- /dev/null +++ b/configs/datagen/hydra_hier_d3.yaml @@ -0,0 +1,48 @@ +# `hydra_hier.yaml` cut to 3 labels (region, service, endpoint), for the Hydra +# study's schema-width sweep: `2^3 - 1 = 7` cell insertions per record. +# Same columns, distributions and seeds as `hydra_hier.yaml`, which has 4 labels. + +column_num: 4 +column_label: [region, service, endpoint, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # endpoint: 25 per service, skewed. + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.2 + population_size: 25 + seed: 3 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 From 0f99c402c55cf5da29208feeec8fd709dee4a3c2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:39:55 +0000 Subject: [PATCH 3/3] datagen: the hydra_hier usage example passes --dtype Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- configs/datagen/hydra_hier.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/datagen/hydra_hier.yaml b/configs/datagen/hydra_hier.yaml index 968e6ecd..62c6a6aa 100644 --- a/configs/datagen/hydra_hier.yaml +++ b/configs/datagen/hydra_hier.yaml @@ -4,7 +4,7 @@ # before it are labels. # # approxbench sketchbench --variant hydra-cms --library lib \ -# --spec configs/datagen/hydra_hier.yaml \ +# --spec configs/datagen/hydra_hier.yaml --dtype i64 \ # --config "rows=3 cols=1024 cell_rows=3 cell_cols=1024" \ # --operations query --metrics accuracy #