fix: item db duplicate handling is stable over insertion order

ck3-tiger/benches/criterion.rs

@@ -22,12 +22,23 @@ struct Config {
     sample_size: Option<usize>,
 }
 
+fn workspace_path(s: &str) -> PathBuf {
+    let p = PathBuf::from(s);
+    if p.is_relative() {
+        PathBuf::from("..").join(p)
+    } else {
+        p
+    }
+}
+
 fn bench_multiple(c: &mut Criterion) {
     let content = fs::read_to_string(CONFIG_PATH).unwrap();
     let config: Config = toml::from_str(&content).unwrap();
-    let mut modfile_paths = config.modfile_paths.iter().map(PathBuf::from).collect::<Vec<_>>();
+    let mut modfile_paths =
+        config.modfile_paths.iter().map(|p| workspace_path(p)).collect::<Vec<_>>();
 
     if let Some(modfile_dir) = config.modfile_dir {
+        let modfile_dir = workspace_path(&modfile_dir);
         let iter =
             fs::read_dir(modfile_dir).unwrap().filter_map(|entry| entry.ok()).filter_map(|entry| {
                 entry.file_name().to_string_lossy().ends_with(".mod").then(|| entry.path())

ck3-tiger/src/bin/scan-mod-ck3-tiger.rs

@@ -50,12 +50,13 @@ fn main() -> Result<()> {
     let mut grand_json = json!({});
     for itype in Item::iter() {
         let mut vec = Vec::new();
-        for token in everything.iter_keys(itype).filter(|token| token.loc.kind == FileKind::Mod) {
+        for token in everything.iter_keys(itype).filter(|token| token.loc.ptr.kind == FileKind::Mod)
+        {
             let json = json!({
                 "key": token.to_string(),
                 "file": token.loc.pathname(),
-                "line": if token.loc.line == 0 { None } else { Some(token.loc.line) },
-                "column": if token.loc.column == 0 { None } else { Some(token.loc.column) },
+                "line": if token.loc.ptr.line == 0 { None } else { Some(token.loc.ptr.line) },
+                "column": if token.loc.ptr.column == 0 { None } else { Some(token.loc.ptr.column) },
             });
             vec.push(json);
         }

src/block.rs

@@ -3,7 +3,7 @@
 use crate::date::Date;
 use crate::macros::MACRO_MAP;
 use crate::parse::pdxfile::{parse_pdx_macro, MacroComponent, MacroComponentKind, PdxfileMemory};
-use crate::token::{Loc, Token};
+use crate::token::{LocStack, Token};
 
 mod blockitem;
 mod bv;
@@ -39,7 +39,7 @@ pub struct Block {
     /// It is in a `Box` to save space in blocks that don't have a tag, which is most of them.
     pub tag: Option<Box<Token>>,
     /// The location of the start of the block. Used mostly for error reporting.
-    pub loc: Loc,
+    pub loc: LocStack,
     /// If the block is a top-level block and contains macro substitutions, this field will
     /// hold the original source for re-parsing.
     /// The source has already been split into a vec that alternates content with macro parameters.
@@ -50,7 +50,7 @@ pub struct Block {
 
 impl Block {
     /// Open a new `Block` at the given location.
-    pub fn new(loc: Loc) -> Self {
+    pub fn new(loc: LocStack) -> Self {
         Block { v: Vec::new(), tag: None, loc, source: None }
     }
 
@@ -463,7 +463,7 @@ impl Block {
     pub fn expand_macro(
         &self,
         args: &[(&str, Token)],
-        loc: Loc,
+        loc: LocStack,
         global: &PdxfileMemory,
     ) -> Option<Block> {
         let link_index = MACRO_MAP.get_or_insert_loc(loc);
@@ -484,7 +484,7 @@ impl Block {
                                 let mut val = val.clone();
                                 let orig_loc = val.loc;
                                 val.loc = token.loc;
-                                val.loc.column -= 1; // point at the $, it looks better
+                                val.loc.ptr.column -= 1; // point at the $, it looks better
                                 val.loc.link_idx = Some(MACRO_MAP.get_or_insert_loc(orig_loc));
                                 content.push(val);
                                 break;

src/ck3/data/characters.rs

@@ -488,7 +488,7 @@ impl Character {
             if birth.is_none() && event != Birth {
                 let msg = format!("{character} was not born yet on {date}");
                 let mut loc = token.loc;
-                loc.column = 0;
+                loc.ptr.column = 0;
                 warn(ErrorKey::History).msg(msg).loc(loc).push();
             }
 
@@ -498,7 +498,7 @@ impl Character {
                         "{character} was not alive on {date}, had already died on {death_date}"
                     );
                     let mut loc = token.loc;
-                    loc.column = 0;
+                    loc.ptr.column = 0;
                     warn(ErrorKey::History)
                         .msg(msg)
                         .loc(loc)
@@ -510,7 +510,7 @@ impl Character {
             match event {
                 Birth => {
                     let mut loc = token.loc;
-                    loc.column = 0;
+                    loc.ptr.column = 0;
 
                     if let Some((birth_date, birth_loc)) = birth {
                         let msg = format!(
@@ -551,7 +551,7 @@ impl Character {
                 }
                 Death => {
                     let mut loc = token.loc;
-                    loc.column = 0;
+                    loc.ptr.column = 0;
                     death = Some((date, loc));
                 }
                 Posthumous => {

src/ck3/data/doctrines.rs

@@ -30,7 +30,7 @@ pub struct Doctrines {
 impl Doctrines {
     fn load_item(&mut self, key: Token, block: Block) {
         if let Some(other) = self.categories.get(key.as_str()) {
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &other.key, "doctrine category");
             }
         }
@@ -128,7 +128,7 @@ impl FileHandler<Block> for Doctrines {
                 }
 
                 if let Some(other) = self.doctrines.get(doctrine.as_str()) {
-                    if other.key.loc.kind >= doctrine.loc.kind {
+                    if other.key.loc.ptr.kind >= doctrine.loc.ptr.kind {
                         dup_error(doctrine, &other.key, "doctrine");
                     }
                 }

src/ck3/data/gameconcepts.rs

@@ -20,7 +20,7 @@ pub struct GameConcepts {
 impl GameConcepts {
     pub fn load_item(&mut self, key: Token, block: Block) {
         if let Some(other) = self.concepts.get(key.as_str()) {
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &other.key, "game concept");
             }
         }

src/ck3/data/interaction_cats.rs

@@ -21,7 +21,7 @@ pub struct CharacterInteractionCategories {
 impl CharacterInteractionCategories {
     pub fn load_item(&mut self, key: Token, block: Block) {
         if let Some(other) = self.categories.get(key.as_str()) {
-            if other.key.loc.kind == key.loc.kind {
+            if other.key.loc.ptr.kind == key.loc.ptr.kind {
                 dup_error(&key, &other.key, "interaction category");
             }
         }

src/ck3/data/maa.rs

@@ -27,7 +27,7 @@ pub struct MenAtArmsTypes {
 impl MenAtArmsTypes {
     pub fn load_item(&mut self, key: Token, block: Block) {
         if let Some(other) = self.menatarmstypes.get(key.as_str()) {
-            if other.key.loc.kind == key.loc.kind {
+            if other.key.loc.ptr.kind == key.loc.ptr.kind {
                 dup_error(&key, &other.key, "men-at-arms type");
             }
         }

src/ck3/data/prov_history.rs

@@ -26,7 +26,7 @@ impl ProvinceHistories {
     fn load_item(&mut self, id: ProvId, key: Token, mut block: Block) {
         if let Some(province) = self.provinces.get_mut(&id) {
             // Multiple entries are valid but could easily be a mistake.
-            if province.key.loc.kind >= key.loc.kind {
+            if province.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &province.key, "province");
             }
             province.block.append(&mut block);

src/ck3/data/prov_terrain.rs

@@ -8,7 +8,7 @@ use crate::item::Item;
 use crate::parse::ParserMemory;
 use crate::pdxfile::PdxFile;
 use crate::report::{warn, ErrorKey, Severity};
-use crate::token::{Loc, Token};
+use crate::token::{LocStack, Token};
 use crate::validator::Validator;
 
 use super::provinces::ProvId;
@@ -18,14 +18,14 @@ const DEFAULT_TERRAINS: &[&str] = &["default_land", "default_sea", "default_coas
 #[derive(Clone, Debug, Default)]
 pub struct ProvinceTerrains {
     provinces: TigerHashMap<ProvId, ProvinceTerrain>,
-    file_loc: Option<Loc>,
+    file_loc: Option<LocStack>,
     defaults: [Option<Token>; DEFAULT_TERRAINS.len()],
 }
 
 impl ProvinceTerrains {
     fn load_item(&mut self, id: ProvId, key: Token, value: Token) {
         if let Some(province) = self.provinces.get_mut(&id) {
-            if province.key.loc.kind >= key.loc.kind {
+            if province.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &province.key, "province");
             }
             *province = ProvinceTerrain::new(key, value);
@@ -79,7 +79,7 @@ impl FileHandler<Block> for ProvinceTerrains {
                 self.load_item(id, key, value);
             } else if let Some(index) = DEFAULT_TERRAINS.iter().position(|&x| x == key.as_str()) {
                 if let Some(default) = &self.defaults[index] {
-                    if default.loc.kind >= key.loc.kind {
+                    if default.loc.ptr.kind >= key.loc.ptr.kind {
                         dup_error(&key, default, "default terrain");
                     }
                 } else {
@@ -119,7 +119,7 @@ impl ProvinceProperties {
     fn load_item(&mut self, id: ProvId, key: Token, mut block: Block) {
         if let Some(province) = self.provinces.get_mut(&id) {
             // Multiple entries are valid but could easily be a mistake.
-            if province.key.loc.kind >= key.loc.kind {
+            if province.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &province.key, "province");
             }
             province.block.append(&mut block);

src/ck3/data/provinces.rs

@@ -17,7 +17,7 @@ use crate::parse::csv::{parse_csv, read_csv};
 use crate::parse::ParserMemory;
 use crate::pdxfile::{PdxEncoding, PdxFile};
 use crate::report::{err, fatal, report, untidy, warn, ErrorKey, Severity};
-use crate::token::{Loc, Token};
+use crate::token::{LocStack, Token};
 use crate::validator::Validator;
 
 pub type ProvId = u32;
@@ -383,7 +383,7 @@ pub struct Coords {
 #[allow(dead_code)] // TODO
 #[derive(Clone, Debug)]
 pub struct Adjacency {
-    line: Loc,
+    line: LocStack,
     from: ProvId,
     to: ProvId,
     /// TODO: check type is sea or `river_large`

src/ck3/data/religions.rs

@@ -287,7 +287,7 @@ impl DbKind for Faith {
         _data: &Everything,
     ) -> bool {
         if property == "is_modded" {
-            return key.loc.kind == FileKind::Mod;
+            return key.loc.ptr.kind == FileKind::Mod;
         }
         false
     }

src/ck3/data/title_history.rs

@@ -25,7 +25,7 @@ impl TitleHistories {
     pub fn load_item(&mut self, key: Token, mut block: Block) {
         if let Some(other) = self.histories.get_mut(key.as_str()) {
             // Multiple entries are valid but could easily be a mistake.
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 warn(ErrorKey::DuplicateItem)
                     .msg("title has two definition blocks, they will be added together")
                     .loc(&other.key)

src/ck3/data/titles.rs

@@ -82,7 +82,7 @@ impl Titles {
         is_county_capital: bool,
     ) {
         if let Some(other) = self.titles.get(key.as_str()) {
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &other.key, "title");
             }
         }

src/ck3/data/traits.rs

@@ -38,7 +38,7 @@ pub struct Traits {
 impl Traits {
     fn load_item(&mut self, key: Token, block: Block) {
         if let Some(other) = self.traits.get(key.as_str()) {
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &other.key, "trait");
             }
         }

src/config_load.rs

@@ -11,16 +11,16 @@ use crate::block::{Block, BlockItem, Comparator, Eq::*, Field, BV};
 use crate::helpers::stringify_list;
 use crate::report::{
     err, set_predicate, set_show_loaded_mods, set_show_vanilla, Confidence, ErrorKey, ErrorLoc,
-    FilterRule, PointedMessage, Severity,
+    FilterRule, PointedMessageStack, Severity,
 };
 
 /// Checks for legacy ignore blocks (that no longer work) and report an error if they are present.
 pub fn check_for_legacy_ignore(config: &Block) {
     // First, report errors if legacy ignore blocks are detected:
-    let pointers: Vec<PointedMessage> = config
+    let pointers: Vec<PointedMessageStack> = config
         .get_keys("ignore")
         .into_iter()
-        .map(|key| PointedMessage::new(key.into_loc()))
+        .map(|key| PointedMessageStack::new(key.into_loc()))
         .collect();
     if !pointers.is_empty() {
         err(ErrorKey::Config)
@@ -406,10 +406,10 @@ pub fn assert_one_key(assert_key: &str, block: &Block) {
         let pointers = keys
             .iter()
             .enumerate()
-            .map(|(index, key)| PointedMessage {
+            .map(|(index, key)| PointedMessageStack {
                 loc: key.into_loc(),
                 length: 1,
-                msg: Some((if index == 0 { "It occurs here" } else { "and here" }).to_owned()),
+                msg: Some((if index == 0 { "It occurs here" } else { "and here" }).into()),
             })
             .collect();
         err(ErrorKey::Config)

src/data/assets.rs

@@ -31,7 +31,7 @@ impl Assets {
     pub fn load_item(&mut self, key: &Token, block: &Block) {
         if let Some(name) = block.get_field_value("name") {
             if let Some(other) = self.assets.get(name.as_str()) {
-                if other.key.loc.kind >= name.loc.kind {
+                if other.key.loc.ptr.kind >= name.loc.ptr.kind {
                     dup_error(name, &other.key, "asset");
                 }
             }

src/data/coa.rs

@@ -31,7 +31,7 @@ impl Coas {
             if let Some(block) = bv.expect_block() {
                 for (key, block) in block.iter_definitions_warn() {
                     if let Some(other) = self.templates.get(key.as_str()) {
-                        if other.key.loc.kind >= key.loc.kind {
+                        if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                             if let BV::Block(otherblock) = &other.bv {
                                 if otherblock.equivalent(block) {
                                     exact_dup_advice(key, &other.key, "coa template");
@@ -49,7 +49,7 @@ impl Coas {
             }
         } else {
             if let Some(other) = self.coas.get(key.as_str()) {
-                if other.key.loc.kind >= key.loc.kind {
+                if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                     if other.bv.equivalent(bv) {
                         exact_dup_advice(key, &other.key, "coat of arms");
                     } else {

src/data/data_binding.rs

@@ -33,7 +33,7 @@ impl DataBindings {
             return;
         }
         if let Some(other) = self.bindings.get(key.as_str()) {
-            if other.key.loc.kind >= key.loc.kind {
+            if other.key.loc.ptr.kind >= key.loc.ptr.kind {
                 dup_error(&key, &other.key, "data binding");
             }
         }

src/data/defines.rs

@@ -23,7 +23,7 @@ impl Defines {
     pub fn load_item(&mut self, group: Token, name: Token, bv: &BV) {
         let key = format!("{}|{}", &group, &name);
         if let Some(other) = self.defines.get(&key) {
-            if other.name.loc.kind >= name.loc.kind && !bv.equivalent(&other.bv) {
+            if other.name.loc.ptr.kind >= name.loc.ptr.kind && !bv.equivalent(&other.bv) {
                 dup_error(&name, &other.name, "define");
             }
         }

src/data/events.rs

@@ -39,7 +39,7 @@ impl Events {
                     if Game::is_vic3() {
                         // Earlier events override later ones in vic3.
                         // The game will complain but it does work, so don't warn unless warranted.
-                        if other.key.loc.kind <= key.loc.kind {
+                        if other.key.loc.ptr.kind <= key.loc.ptr.kind {
                             dup_error(&other.key, &key, "event");
                         }
                         return;
@@ -58,15 +58,15 @@ impl Events {
     }
 
     fn load_scripted_trigger(&mut self, key: Token, block: Block) {
-        let index = (key.loc.idx, key.as_str());
+        let index = (key.loc.ptr.idx, key.as_str());
         if let Some(other) = self.triggers.get(&index) {
             dup_error(&key, &other.key, "scripted trigger");
         }
         self.triggers.insert(index, Trigger::new(key, block, None));
     }
 
     fn load_scripted_effect(&mut self, key: Token, block: Block) {
-        let index = (key.loc.idx, key.as_str());
+        let index = (key.loc.ptr.idx, key.as_str());
         if let Some(other) = self.effects.get(&index) {
             dup_error(&key, &other.key, "scripted effect");
         }
@@ -87,13 +87,13 @@ impl Events {
 
     #[cfg(feature = "ck3")]
     pub fn get_trigger(&self, key: &Token) -> Option<&Trigger> {
-        let index = (key.loc.idx, key.as_str());
+        let index = (key.loc.ptr.idx, key.as_str());
         self.triggers.get(&index)
     }
 
     #[cfg(feature = "ck3")]
     pub fn get_effect(&self, key: &Token) -> Option<&Effect> {
-        let index = (key.loc.idx, key.as_str());
+        let index = (key.loc.ptr.idx, key.as_str());
         self.effects.get(&index)
     }