datagen: child_of columns for hierarchical labels

aqpbm-cli/src/external.rs

@@ -269,6 +269,8 @@ fn shape_column(data_type: &str) -> ColumnSpec {
         special_rule: RULE_NONE,
         data_type: data_type.to_string(),
         string: None,
+        child_of: None,
+        fan_out: None,
     }
 }
 

aqpbm-cli/src/flatten_record.rs

@@ -249,6 +249,8 @@ mod tests {
                 special_rule: aqpbm_datagen::RULE_NONE,
                 data_type: "i64".into(),
                 string: None,
+                child_of: None,
+                fan_out: None,
             },
             1000,
         );

aqpbm-cli/src/main.rs

@@ -258,6 +258,8 @@ fn dataset_spec(args: &SketchbenchArgs) -> Result<InputDataSetSpec> {
                 };
                 (opts != StringOpts::default()).then_some(opts)
             },
+            child_of: None,
+            fan_out: None,
         },
         args.size.expect("clap requires --size") as u64,
     )))

aqpbm-cli/src/repeat.rs

@@ -191,6 +191,8 @@ mod tests {
                 special_rule: aqpbm_datagen::RULE_NONE,
                 data_type: "i64".into(),
                 string: None,
+                child_of: None,
+                fan_out: None,
             },
             1000,
         );

aqpbm-core/schema/merged_record.schema.json

@@ -393,13 +393,31 @@
           "format": "uint64",
           "minimum": 0
         },
+        "child_of": {
+          "description": "Index of an earlier `string` column this one nests under, for\nhierarchical labels. A value is the parent row's value, a `.`, and a\nchild index in `[0, fan_out)` drawn from this column's distribution.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint",
+          "minimum": 0
+        },
         "data_type": {
           "description": "Which [`ColumnData`] variant this column renders into.",
           "type": "string"
         },
         "distribution": {
           "$ref": "#/$defs/DataDistribution"
         },
+        "fan_out": {
+          "description": "How many children each parent value has. Required with `child_of`, and\nchecked against the distribution's domain like `cardinality` is.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0
+        },
         "shift": {
           "description": "Added to every value, for a column that should not start where its\ndistribution naturally does. Under the monotonic rule it is the value the\nseries starts from.",
           "type": [

aqpbm-core/src/atomic_costs.rs

@@ -418,6 +418,8 @@ mod tests {
                 special_rule: aqpbm_datagen::RULE_NONE,
                 data_type: "i64".into(),
                 string: None,
+                child_of: None,
+                fan_out: None,
             },
             1_000_000,
         )

aqpbm-core/src/benchmark_result/schema.rs

@@ -490,6 +490,8 @@ mod tests {
                 special_rule: aqpbm_datagen::RULE_NONE,
                 data_type: "i64".into(),
                 string: None,
+                child_of: None,
+                fan_out: None,
             },
             1_000_000,
         )

aqpbm-core/src/erp.rs

@@ -181,6 +181,8 @@ mod tests {
                     special_rule: 0,
                     data_type: "u64".into(),
                     string: None,
+                    child_of: None,
+                    fan_out: None,
                 },
                 100_000,
             ),

aqpbm-datagen/src/column.rs

@@ -36,6 +36,15 @@ pub struct ColumnSpec {
     /// Rendering options for `data_type: string`. Absent means the defaults.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub string: Option<StringOpts>,
+    /// Index of an earlier `string` column this one nests under, for
+    /// hierarchical labels. A value is the parent row's value, a `.`, and a
+    /// child index in `[0, fan_out)` drawn from this column's distribution.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub child_of: Option<usize>,
+    /// How many children each parent value has. Required with `child_of`, and
+    /// checked against the distribution's domain like `cardinality` is.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fan_out: Option<u64>,
 }
 
 impl ColumnSpec {
@@ -53,6 +62,21 @@ impl ColumnSpec {
             )));
         }
 
+        match (self.child_of, self.fan_out) {
+            (None, None) => {}
+            (Some(_), None) => {
+                return Err(DataGenError::BadParam(
+                    "child_of: needs fan_out, the number of children per parent value".into(),
+                ))
+            }
+            (None, Some(_)) => {
+                return Err(DataGenError::BadParam(
+                    "fan_out: applies only to a child_of column".into(),
+                ))
+            }
+            (Some(_), Some(fan_out)) => self.validate_child(fan_out)?,
+        }
+
         // `cardinality` restates the domain. Silently preferring one over the
         // other would let a report cite a key space the column never had.
         if let Some(claimed) = self.cardinality {
@@ -98,6 +122,64 @@ impl ColumnSpec {
         Ok(())
     }
 
+    /// The checks a `child_of` column adds. Its index of the parent is checked
+    /// by the table, which knows the other columns.
+    fn validate_child(&self, fan_out: u64) -> Result<(), DataGenError> {
+        if self.data_type != "string" {
+            return Err(DataGenError::BadParam(format!(
+                "child_of: a child value is text, so data_type has to be `string`, not '{}'",
+                self.data_type
+            )));
+        }
+        if fan_out == 0 {
+            return Err(DataGenError::BadParam("fan_out must be > 0".into()));
+        }
+        // A uniform draw is continuous; only whole bounds keep its index in
+        // `[0, fan_out)` (`uniform{0.0, 2.9}` would draw 0, 1 and 2).
+        if let DataDistribution::Uniform(p) = &self.distribution {
+            if p.lower_bound.fract() != 0.0 || p.upper_bound.fract() != 0.0 {
+                return Err(DataGenError::BadParam(format!(
+                    "child_of: uniform bounds must be whole numbers, got [{}, {})",
+                    p.lower_bound, p.upper_bound
+                )));
+            }
+        }
+        match self.distribution.domain() {
+            Some(domain) if domain.size == fan_out => {}
+            Some(domain) => {
+                return Err(DataGenError::BadParam(format!(
+                    "fan_out {fan_out} disagrees with the {} domain, which holds {}; the \
+                     distribution draws the child index",
+                    self.distribution.tag(),
+                    domain.size,
+                )))
+            }
+            None => {
+                return Err(DataGenError::BadParam(format!(
+                    "child_of: the child index needs a bounded domain, and {} has none",
+                    self.distribution.tag(),
+                )))
+            }
+        }
+        // A child column holds parent cardinality × fan_out values and is not
+        // rendered from a rank, so neither field would mean what it says.
+        if self.cardinality.is_some() {
+            return Err(DataGenError::BadParam(
+                "child_of: cardinality would restate fan_out, not the column's distinct \
+                 count (parent cardinality × fan_out); drop it"
+                    .into(),
+            ));
+        }
+        if self.string.is_some() {
+            return Err(DataGenError::BadParam(
+                "child_of: a child value is its parent's plus an index, so a `string:` \
+                 block has nothing to render; drop it"
+                    .into(),
+            ));
+        }
+        Ok(())
+    }
+
     /// The raw draw stream: `row_num` values straight from the distribution,
     /// before any rule, shift or rendering. Split out because a
     /// `column_connected` group draws this once and every member column renders
@@ -163,4 +245,23 @@ impl ColumnSpec {
             ))),
         }
     }
+
+    /// Render a `child_of` column: each row's parent value, a `.`, and the
+    /// draw's 0-based rank in the domain as the child index.
+    pub(crate) fn render_child(
+        &self,
+        raw: &[f64],
+        parents: &[String],
+    ) -> Result<ColumnData, DataGenError> {
+        let domain = self
+            .distribution
+            .domain()
+            .expect("child domains were validated above");
+        Ok(ColumnData::String(
+            raw.iter()
+                .zip(parents)
+                .map(|(draw, parent)| format!("{parent}.{}", (draw - domain.lower) as u64))
+                .collect(),
+        ))
+    }
 }

aqpbm-datagen/src/lib.rs

@@ -61,6 +61,8 @@ mod tests {
             special_rule: RULE_NONE,
             data_type: data_type.into(),
             string: None,
+            child_of: None,
+            fan_out: None,
         }
     }
 
@@ -502,6 +504,164 @@ mod tests {
         assert!(err.contains("used twice"), "{err}");
     }
 
+    // ---------- child_of ----------
+
+    fn child(parent: usize, fan_out: u64, distribution: DataDistribution) -> ColumnSpec {
+        ColumnSpec {
+            child_of: Some(parent),
+            fan_out: Some(fan_out),
+            ..column(distribution, "string")
+        }
+    }
+
+    /// region (4) → service (fan-out 25, Zipf) → endpoint (fan-out 5), then a value.
+    fn hier_table() -> TableDescription {
+        TableDescription {
+            column_num: 4,
+            column_label: vec![
+                "region".into(),
+                "service".into(),
+                "endpoint".into(),
+                "value".into(),
+            ],
+            column_spec: vec![
+                column(uniform(0.0, 4.0, 1), "string"),
+                child(0, 25, zipf(25, 1.2, 2)),
+                child(1, 5, uniform(0.0, 5.0, 3)),
+                column(zipf(1000, 1.1, 4), "i64"),
+            ],
+            column_connected: Vec::new(),
+            row_num: 20_000,
+        }
+    }
+
+    /// Each child splits into its parent, a `.`, and an index below `fan_out`;
+    /// returns the distinct children seen under each parent.
+    fn children_by_parent(
+        parents: &[String],
+        children: &[String],
+        fan_out: u64,
+    ) -> std::collections::HashMap<String, std::collections::HashSet<String>> {
+        let mut by_parent: std::collections::HashMap<_, std::collections::HashSet<_>> =
+            Default::default();
+        for (parent, child) in parents.iter().zip(children) {
+            let (head, index) = child.rsplit_once('.').expect("a child has a `.`");
+            assert_eq!(head, parent, "'{child}' does not start with its parent");
+            assert!(index.parse::<u64>().unwrap() < fan_out, "'{child}'");
+            by_parent
+                .entry(parent.clone())
+                .or_default()
+                .insert(child.clone());
+        }
+        by_parent
+    }
+
+    #[test]
+    fn a_child_table_is_deterministic() {
+        let d = hier_table();
+        assert_eq!(d.generate().unwrap(), d.generate().unwrap());
+    }
+
+    #[test]
+    fn every_child_extends_its_parent_within_its_fan_out() {
+        let t = hier_table().generate().unwrap();
+        let region = t.data[0].as_string().unwrap();
+        let service = t.data[1].as_string().unwrap();
+        let endpoint = t.data[2].as_string().unwrap();
+
+        let services = children_by_parent(region, service, 25);
+        assert_eq!(services.len(), 4);
+        // 20k draws over 25 Zipf(1.2) ranks reach every service under every region.
+        assert!(services.values().all(|s| s.len() == 25), "{services:?}");
+
+        let endpoints = children_by_parent(service, endpoint, 5);
+        assert_eq!(endpoints.len(), 100);
+        assert!(endpoints.values().all(|e| e.len() <= 5));
+    }
+
+    /// The child index is the column's own draw, ranked: the distribution is
+    /// what picks it, and the parent plays no part.
+    #[test]
+    fn the_child_index_follows_the_columns_distribution() {
+        let d = hier_table();
+        let draws = d.column_spec[1].draw(d.row_num as usize).unwrap();
+        let t = d.generate().unwrap();
+        let service = t.data[1].as_string().unwrap();
+        let mut counts = [0u64; 25];
+        for (draw, s) in draws.iter().zip(service) {
+            let index: u64 = s.rsplit_once('.').unwrap().1.parse().unwrap();
+            assert_eq!(index, (draw - 1.0) as u64);
+            counts[index as usize] += 1;
+        }
+        // Zipf(1.2) over 25 ranks gives rank 1 about 30% of the mass.
+        let head = counts[0] as f64 / d.row_num as f64;
+        assert!((0.25..0.35).contains(&head), "{head}");
+        assert!(counts[0] > counts[1] && counts[1] > counts[24]);
+    }
+
+    #[test]
+    fn hydra_hier_yaml_generates_its_hierarchy() {
+        let d =
+            TableDescription::from_path(std::path::Path::new("../configs/datagen/hydra_hier.yaml"))
+                .unwrap();
+        let t = d.generate().unwrap();
+        let col = |i: usize| t.data[i].as_string().unwrap();
+        assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4);
+        assert_eq!(children_by_parent(col(1), col(2), 25).len(), 100);
+    }
+
+    /// The schema-width cuts keep the first labels and the value column.
+    #[test]
+    fn hydra_hier_cuts_keep_their_leading_labels() {
+        for (file, labels) in [("hydra_hier_d2.yaml", 2), ("hydra_hier_d3.yaml", 3)] {
+            let path = format!("../configs/datagen/{file}");
+            let d = TableDescription::from_path(std::path::Path::new(&path)).unwrap();
+            assert_eq!(d.column_num, labels + 1, "{file}");
+            let t = d.generate().unwrap();
+            let col = |i: usize| t.data[i].as_string().unwrap();
+            assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4, "{file}");
+        }
+    }
+
+    #[test]
+    fn bad_child_columns_are_refused_by_name() {
+        type Edit = fn(&mut TableDescription);
+        let cases: Vec<(&str, Edit)> = vec![
+            ("earlier column", |d| d.column_spec[1].child_of = Some(1)),
+            ("earlier column", |d| d.column_spec[1].child_of = Some(9)),
+            ("not a string column", |d| {
+                d.column_spec[0].data_type = "i64".into()
+            }),
+            ("needs fan_out", |d| d.column_spec[1].fan_out = None),
+            ("only to a child_of", |d| d.column_spec[1].child_of = None),
+            ("fan_out must be > 0", |d| {
+                d.column_spec[1].fan_out = Some(0)
+            }),
+            ("fan_out 24 disagrees", |d| {
+                d.column_spec[1].fan_out = Some(24)
+            }),
+            ("bounded domain", |d| {
+                d.column_spec[1].distribution = normal(0.0, 1.0, 2)
+            }),
+            ("has to be `string`", |d| {
+                d.column_spec[1].data_type = "u64".into()
+            }),
+            ("whole numbers", |d| {
+                d.column_spec[2].distribution = uniform(0.0, 5.9, 3)
+            }),
+            ("cardinality", |d| d.column_spec[1].cardinality = Some(25)),
+            ("`string:` block", |d| {
+                d.column_spec[1].string = Some(StringOpts::default())
+            }),
+        ];
+        for (want, edit) in cases {
+            let mut d = hier_table();
+            edit(&mut d);
+            let err = d.generate().unwrap_err().to_string();
+            assert!(err.contains(want), "wanted '{want}' in: {err}");
+        }
+    }
+
     // ---------- serde ----------
 
     #[test]