diff --git a/docs/svs-v4.md b/docs/svs-v4.md new file mode 100644 index 00000000..2972cf68 --- /dev/null +++ b/docs/svs-v4.md @@ -0,0 +1,463 @@ +# State Vector Sync (SVS) v4 Specification + +SVS v4 is a revision of SVS v3 for large synchronization groups. It introduces +a membership hash (`mhash`), two embedded state-vector encodings (`FULL` and +`PARTIAL`), and a third publish-only form that references a retrievable full +vector. The protocol is self-contained: there is no compatibility mode with +plain SVS v3 peers — every Sync Data carries `mhash` and a `VectorType`. + +--- + +## 1. Basic Protocol Design + +### 1.1 Small groups + +For most deployments, the complete State Vector fits in one Sync packet. +Nodes exchange **full** State Vectors using steady-state, suppression, merge, +and `OnUpdate` semantics inherited from SVS v3. + +### 1.2 Large groups + +When the encoded State Vector exceeds **`SyncVectorThreshold`** (an +application-configured size budget in bytes), nodes use three dissemination +modes: + +| Mode | Trigger | Wire shape | +|------|---------|------------| +| **Embedded FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | +| **Embedded PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | +| **Publish + pull** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | + +**MemberSetHash (`mhash`)** is always carried inside `SvsData`. It is a +**membership hash**, not a hash of the full State Vector. + +**Full state recovery** uses publish + pull when: + +1. `mhash` differs from the local membership hash, or +2. Periodic sync runs while the local FULL encoding exceeds + `SyncVectorThreshold`, or +3. An embedded `VectorType = FULL` State Vector is outdated per §6.2. + +Link-level fragmentation (NDNLPv2) is below this layer. Publishers use the +ndnd object segmentation APIs when retrievable full-vector Data is large. + +--- + +## 2. Format and Naming + +### 2.1 Sync Interest + +**Sync Interest Name:** + +``` +//v=4 +``` + +Implementations MAY append additional name components after `v=4`. The +Interest nonce is carried in Interest packet fields, not as a name component. + +- Signed Sync Data is carried in `ApplicationParameters`. +- Interest Lifetime is 1 second. +- Sync Interests are unacknowledged. + +### 2.2 Sync Data (in ApplicationParameters) + +**Sync Data Name** (signing identity for the Sync message): + +``` +//// +``` + +- **`version`:** microsecond timestamp. No hash suffix is used. + +**Sync Data Content:** encoded `SvsData` (§3) — either embedded form (FULL +or PARTIAL) or publish-only form. + +### 2.3 Application publication Data + +``` +////seq= +``` + +Application-level naming may vary. Sync vector Data lives in a separate +namespace distinguished by the `32=sv` keyword (§2.4). + +### 2.4 Published full State Vector Data + +Retrievable full State Vector objects use a dedicated sync namespace: + +**Name:** + +``` +////32=sv/ +``` + +**Content:** signed `SvsData` in embedded FULL form: `mhash` + +`VectorType = FULL` + complete `StateVector`. + +**Publish + pull procedure** (periodic sync, `mhash` recovery, join when +FULL exceeds threshold): + +1. Produce the full-vector Data at + `////32=sv/` (ndnd segmentation handles + large content). +2. Send a Sync Interest whose AppParam Sync Data contains publish-only + `SvsData`: `mhash` + `SvsDataRef` pointing at the published name (§3.1). +3. Receivers pull the referenced Data, validate, and merge. + +A Sync message carries either an embedded StateVector or a publish-only +reference — not both. + +--- + +## 3. Packet Specification + +### 3.1 `SvsData` + +`SvsData` has two forms: embedded (FULL or PARTIAL) and publish-only. The +`mhash` field is present in both forms. `VectorType` is only meaningful in +the embedded form. + +#### 3.1.1 Embedded form (FULL or PARTIAL) + +Used when the State Vector (full or a publication-time PARTIAL subset) is +carried inline in Sync Data, or in published full-vector Data at +`32=sv/`. + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + VectorType + StateVector +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `VectorType` | `0xCD` | `0` = FULL, `1` = PARTIAL | +| `StateVector` | `0xC9` | See §3.2 | + +#### 3.1.2 Publish-only form + +Used when Sync Data advertises a retrievable full-vector Data name (periodic +sync, `mhash` recovery). `VectorType` and `StateVector` are absent. + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + SvsDataRef +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `SvsDataRef` | `0x07` (Name) | Name of the published full-vector Data. The receiver strips the trailing version component and uses the resulting `32=sv` prefix as the trust anchor for that sender's retrievable full vectors. | + +The inline layout extends ndnd v3 `SvsData` with `MemberSetHash` and +`VectorType` before `StateVector`, matching the Python strawman (`mhash` at +`0xCB`, vector at `0xC9`/`0xCA`). + +### 3.2 `StateVector` + +``` +StateVector = STATE-VECTOR-TYPE TLV-LENGTH + *StateVectorEntry + +StateVectorEntry = STATE-VECTOR-ENTRY-TYPE TLV-LENGTH + Name + *SeqNoEntry + +SeqNoEntry = SEQ-NO-ENTRY-TYPE TLV-LENGTH + BootstrapTime + SeqNo +``` + +| TLV | Type (decimal) | Type (hex) | +|-----|----------------|------------| +| `STATE-VECTOR-TYPE` | 201 | `0xC9` | +| `STATE-VECTOR-ENTRY-TYPE` | 202 | `0xCA` | +| `SEQ-NO-ENTRY-TYPE` | 210 | `0xD2` | +| `BOOTSTRAP-TIME-TYPE` | 212 | `0xD4` | +| `SEQ-NO-TYPE` | 214 | `0xD6` | + +**Rules:** + +- Sequence numbers are 1-indexed. +- Bootstrap time is seconds since Unix epoch. +- If an entry is absent, its sequence number is treated as 0 for comparison. +- If any received `BootstrapTime` is more than 86400s in the future, the + entire `StateVector` SHOULD be ignored. + +### 3.3 `MemberSetHash` (`mhash`) + +`mhash` is a **membership hash**. It is not a hash of the full State Vector +and not a hash of sequence numbers. + +**Membership** is the set of participants, each identified by: + +``` +(Producer Name, Bootstrap Time) +``` + +**Computation:** + +``` +members = { (Name, BootstrapTime) | node knows this member in the sync group } +sort by NDN canonical order of Name, then by BootstrapTime ascending +mhash = SHA-256( concatenation of canonical TLV bytes of each (Name, BootstrapTime) pair ) +``` + +Recompute `mhash` whenever membership changes (member added, removed, or new +bootstrap time for a name). + +The Python strawman hashes sorted producer names only. SVS v4 includes +Bootstrap Time in each membership tuple, consistent with SVS v3 identity. + +Membership data and State Vector data are separate concepts. Membership is +carried implicitly in the full State Vector. `mhash` summarizes membership +for quick comparison. + +### 3.4 `VectorType` (embedded form) + +| Value | Name | Meaning | +|-------|------|---------| +| `0` | **FULL** | `StateVector` contains the complete advertised state (§4.1 ordering). | +| `1` | **PARTIAL** | `StateVector` contains a subset (§4.2). Used for new publication only when FULL exceeds threshold. | + +`VectorType` is required on the wire because it lets a receiver skip the +more expensive subset-evaluation code path when it sees `FULL`, and lets a +sender guarantee the receiver knows whether missing names imply partition +(FULL) or merely "not included in this subset" (PARTIAL). `mhash` alone +cannot convey this — two parties with identical membership but different +subscription views may legitimately disagree on what subset was sent. + +`mhash` is present in both embedded and publish-only `SvsData` messages. + +--- + +## 4. State Vector Encoding + +### 4.1 FULL State Vector + +- Include all known members and their latest sequence numbers per bootstrap. +- Entries ordered in NDN canonical order of `Name`. +- Set `VectorType = FULL`. + +### 4.2 PARTIAL State Vector + +Used on new publication when +`encoded_size(embedded FULL SvsData) > SyncVectorThreshold`. + +- Set `VectorType = PARTIAL`. +- **Entry `[0]`** is the sender's own `StateVectorEntry`. +- **Entries `[1…n]`** are in NDN canonical order among included peers. + +If the sender-only baseline already exceeds `SyncVectorThreshold`, the +sender falls back to publish + pull rather than emit a PARTIAL vector that +omits the required entry `[0]`. + +An implementation MAY use the following selection priority: + +| Priority | Include | +|----------|---------| +| 1 | Sender (always) | +| 2 | Repair targets | +| 3 | Propagation targets | +| 4 | Random inactive producers | +| 5 | Others by recency | + +Stop adding entries when the estimated embedded `SvsData` size approaches +`SyncVectorThreshold`. + +### 4.3 `SyncVectorThreshold` + +- Configurable implementation parameter (application packet size budget) in + bytes. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use embedded FULL + (with `mhash` and `VectorType=FULL`). +- When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL + (publication) or publish + pull (periodic sync and recovery). + +The wire format is independent of `SyncVectorThreshold`. All Sync messages +carry `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). +`SyncVectorThreshold <= 0` selects the default 1200-byte budget. + +> **Future work:** the spec currently treats `SyncVectorThreshold` as a +> static application-level constant. Auto-sizing it from observed MTU is a +> planned extension and is intentionally out of scope for v4. + +--- + +## 5. State Sync + +Sections 5.1–5.4 inherit their behavior from SVS v3 [Section 4](https://named-data.github.io/StateVectorSync/Specification.html). +SVS v4 adds Sections 5.5–5.9. + +### 5.1 Sync Interest timer + +- `PeriodicTimeout` default 30s (±10% jitter). +- `SuppressionPeriod` default 200ms. +- `SuppressionTimeout` exponential decay. + +### 5.2 Send Sync Interest on new publication + +When the node generates a new publication, it immediately emits a Sync +Interest and resets the timer to `PeriodicTimeout`. + +| Trigger | Action | +|---------|--------| +| `encoded_size(embedded FULL) ≤ SyncVectorThreshold` | Send embedded FULL (`mhash` + `VectorType=FULL` + `StateVector`) | +| `encoded_size(embedded FULL) > SyncVectorThreshold` | Send embedded PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | + +### 5.3 Sync Ack policy + +Sync Interests are unacknowledged. + +### 5.4 Steady state and suppression (embedded FULL) + +For incoming Sync Data with embedded `VectorType = FULL`, apply SVS v3 +steady-state and suppression rules. + +### 5.5 PARTIAL State Vector processing + +When `VectorType = PARTIAL`: + +1. Parse `mhash` and `StateVector`. +2. Names omitted from the partial `StateVector` are interpreted as "not + included in this subset" — they do not imply producer removal, outdated + sender, or sequence rollback. +3. For each present entry, merge newer sequence numbers into local state + (§6.1). +4. If `mhash` differs from local `mhash`, perform publish + pull recovery + (§5.6). + +This is the receive-side change versus SVS v3. + +### 5.6 Full state recovery (publish + pull) + +**Triggers:** + +| # | Trigger | Action | +|---|---------|--------| +| 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Publish + pull | +| 2 | Embedded `VectorType = FULL` is outdated per §6.2 | Merge embedded if complete; otherwise publish + pull | +| 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Publish + pull (§5.8) | + +Recovery always fetches the complete State Vector from the referenced +`32=sv/` Data. + +**Sender procedure** (on `mhash` mismatch or periodic large-group sync): + +1. Produce full-vector Data at `////32=sv/` + with embedded FULL `SvsData`. +2. Send Sync Interest with publish-only `SvsData` (`mhash` + `SvsDataRef`). + +**Receiver procedure:** + +1. Identify the sender from the Sync Data signature, or — when the Sync + Data is PARTIAL — from PARTIAL entry `[0]`, which is the sender's own + entry per §4.2. +2. If the Sync Data is embedded FULL and complete: merge directly. +3. If the Sync Data is publish-only: read `SvsDataRef`; express Interest for + that name; validate; merge; update local `mhash`. +4. Continue application data fetch via SvsALO (`OnUpdate`) as today. + +> **Implementation note:** A consumer may receive many publish-only Sync +> messages that all cross the `mhash` boundary simultaneously. To bound the +> resulting pull fan-in, implementations commonly debounce per-sender pull +> attempts (e.g., 5 seconds per sender prefix). This is a local +> implementation detail and does not affect protocol correctness — a +> debounced pull is equivalent to a slightly delayed pull. + +Use ndnd segmentation when fetched Data content is large. + +### 5.7 New node join + +1. Joining node **N** multicasts Sync Interest whose embedded State Vector + contains only itself: `(Name=N, SeqNo=0)`. The Sync Data's `mhash` is + the SHA-256 of N's single-member membership list. +2. Existing members receive the announcement. +3. Suppression limits duplicate responses; typically one member **A** + provides recovery state. +4. If FULL fits inline: **A** responds with embedded `VectorType = FULL`. +5. If FULL exceeds `SyncVectorThreshold`: **A** uses publish + pull + (produce at `32=sv/`, then publish-only Sync Data). +6. Normal synchronization proceeds through SvsALO. + +### 5.8 Periodic sync in large groups + +| Local FULL size | Periodic Sync behavior | +|-----------------|------------------------| +| `≤ SyncVectorThreshold` | Embedded FULL | +| `> SyncVectorThreshold` | Publish + pull (produce full-vector Data, then publish-only Sync Data) | + +Periodic sync does not send embedded PARTIAL vectors. + +### 5.9 Summary of sync triggers + +| Event | `size ≤ threshold` | `size > threshold` | +|-------|--------------------|--------------------| +| **New publication** | Embedded FULL | Embedded PARTIAL (or publish + pull fallback) | +| **Periodic sync** | Embedded FULL | Publish + pull | +| **`mhash` mismatch** | Publish + pull (if recovery needed) | Publish + pull | + +--- + +## 6. Comparing and Merging State Vectors + +### 6.1 Merge rule + +For each matching `(Name, BootstrapTime)`, retain the maximum `SeqNo`. + +### 6.2 Outdated vector (embedded FULL only) + +State Vector `A` is outdated to `B` if: + +- `A` is missing a name present in `B`, or +- `A` has a strictly smaller `SeqNo` for any entry. + +For `VectorType = PARTIAL`, the missing-name rule does not apply to names +omitted from the partial message. + +--- + +## 7. Examples + +### 7.1 Small group + +Three nodes `A`, `B`, `C`. Full State Vector fits. `A` publishes; sends +embedded FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. + +### 7.2 Large group + +Group exceeds `SyncVectorThreshold`. Producer `P` publishes: + +- `P` sends embedded PARTIAL `SvsData { mhash, VectorType=PARTIAL, + StateVector=[P:…, A:…, …] }`. +- Receiver merges present entries only. +- If `mhash` differs, `P` (or receiver per policy) triggers publish + pull + (§5.6). + +### 7.3 Large group + +- `A` produces full vector at `/group/A/boot/32=sv/`. +- `A` sends publish-only Sync Data `{ mhash, + SvsDataRef=/group/A/boot/32=sv/ }`. +- Peers pull and merge. + +### 7.4 New node join + +- `N` sends self-only vector `[N:0]` with `mhash`. +- `A` responds with embedded FULL or publish + pull. +- `N` merges and synchronizes via SvsALO. + +--- + +## 8. Interoperability + +SVS v4 defines a single wire profile. It does not interoperate with plain +SVS v3 peers in the same sync group: deployments upgrade all nodes to a +v4-conformant implementation at the same time. Every Sync Data carries +`mhash` and a `VectorType` (or `SvsDataRef` for publish-only). The +implementation never emits a legacy `StateVector`-only `SvsData`, regardless +of `SyncVectorThreshold` (a `Threshold ≤ 0` selects the 1200-byte default). \ No newline at end of file diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index 3f7777bc..4412d154 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -3,9 +3,22 @@ package svs import ( enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/types/optional" +) + +// VectorType values for inline SvsData (TLV 0xCD). +const ( + VectorTypeFull uint64 = 0 + VectorTypePartial uint64 = 1 ) type SvsData struct { + //+field:binary:optional + MemberSetHash []byte `tlv:"0xcb"` + //+field:natural:optional + VectorType optional.Optional[uint64] `tlv:"0xcd"` + //+field:name + SvsDataRef enc.Name `tlv:"0x07"` //+field:struct:StateVector StateVector *StateVector `tlv:"0xc9"` } @@ -29,6 +42,14 @@ type SeqNoEntry struct { SeqNo uint64 `tlv:"0xd6"` } +// MembershipTuple is one (Name, BootstrapTime) pair used to compute MemberSetHash. +type MembershipTuple struct { + //+field:name + Name enc.Name `tlv:"0x07"` + //+field:natural + BootstrapTime uint64 `tlv:"0xd4"` +} + // +tlv-model:nocopy type PassiveState struct { //+field:sequence:[]byte:binary:[]byte diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 0fb2e7fc..6d4590e7 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -11,6 +11,7 @@ import ( type SvsDataEncoder struct { Length uint + SvsDataRef_length uint StateVector_encoder StateVectorEncoder } @@ -19,11 +20,32 @@ type SvsDataParsingContext struct { } func (encoder *SvsDataEncoder) Init(value *SvsData) { + + if value.SvsDataRef != nil { + encoder.SvsDataRef_length = 0 + for _, c := range value.SvsDataRef { + encoder.SvsDataRef_length += uint(c.EncodingLength()) + } + } if value.StateVector != nil { encoder.StateVector_encoder.Init(value.StateVector) } l := uint(0) + if value.MemberSetHash != nil { + l += 1 + l += uint(enc.TLNum(len(value.MemberSetHash)).EncodingLength()) + l += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + l += 1 + l += uint(1 + enc.Nat(optval).EncodingLength()) + } + if value.SvsDataRef != nil { + l += 1 + l += uint(enc.TLNum(encoder.SvsDataRef_length).EncodingLength()) + l += encoder.SvsDataRef_length + } if value.StateVector != nil { l += 1 l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) @@ -34,6 +56,7 @@ func (encoder *SvsDataEncoder) Init(value *SvsData) { } func (context *SvsDataParsingContext) Init() { + context.StateVector_context.Init() } @@ -41,6 +64,29 @@ func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { pos := uint(0) + if value.MemberSetHash != nil { + buf[pos] = byte(203) + pos += 1 + pos += uint(enc.TLNum(len(value.MemberSetHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.MemberSetHash) + pos += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + buf[pos] = byte(205) + pos += 1 + + buf[pos] = byte(enc.Nat(optval).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + + } + if value.SvsDataRef != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.SvsDataRef_length).EncodeInto(buf[pos:])) + for _, c := range value.SvsDataRef { + pos += uint(c.EncodeInto(buf[pos:])) + } + } if value.StateVector != nil { buf[pos] = byte(201) pos += 1 @@ -64,6 +110,9 @@ func (encoder *SvsDataEncoder) Encode(value *SvsData) enc.Wire { func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*SvsData, error) { + var handled_MemberSetHash bool = false + var handled_VectorType bool = false + var handled_SvsDataRef bool = false var handled_StateVector bool = false progress := -1 @@ -91,6 +140,43 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical err = nil if handled := false; true { switch typ { + case 203: + if true { + handled = true + handled_MemberSetHash = true + value.MemberSetHash = make([]byte, l) + _, err = reader.ReadFull(value.MemberSetHash) + } + case 205: + if true { + handled = true + handled_VectorType = true + { + optval := uint64(0) + optval = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + optval = uint64(optval<<8) | uint64(x) + } + } + value.VectorType.Set(optval) + } + } + case 7: + if true { + handled = true + handled_SvsDataRef = true + delegate := reader.Delegate(int(l)) + value.SvsDataRef, err = delegate.ReadName() + } case 201: if true { handled = true @@ -115,6 +201,15 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical startPos = reader.Pos() err = nil + if !handled_MemberSetHash && err == nil { + value.MemberSetHash = nil + } + if !handled_VectorType && err == nil { + value.VectorType.Unset() + } + if !handled_SvsDataRef && err == nil { + value.SvsDataRef = nil + } if !handled_StateVector && err == nil { value.StateVector = nil } @@ -742,6 +837,172 @@ func ParseSeqNoEntry(reader enc.WireView, ignoreCritical bool) (*SeqNoEntry, err return context.Parse(reader, ignoreCritical) } +type MembershipTupleEncoder struct { + Length uint + + Name_length uint +} + +type MembershipTupleParsingContext struct { +} + +func (encoder *MembershipTupleEncoder) Init(value *MembershipTuple) { + if value.Name != nil { + encoder.Name_length = 0 + for _, c := range value.Name { + encoder.Name_length += uint(c.EncodingLength()) + } + } + + l := uint(0) + if value.Name != nil { + l += 1 + l += uint(enc.TLNum(encoder.Name_length).EncodingLength()) + l += encoder.Name_length + } + l += 1 + l += uint(1 + enc.Nat(value.BootstrapTime).EncodingLength()) + encoder.Length = l + +} + +func (context *MembershipTupleParsingContext) Init() { + +} + +func (encoder *MembershipTupleEncoder) EncodeInto(value *MembershipTuple, buf []byte) { + + pos := uint(0) + + if value.Name != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.Name_length).EncodeInto(buf[pos:])) + for _, c := range value.Name { + pos += uint(c.EncodeInto(buf[pos:])) + } + } + buf[pos] = byte(212) + pos += 1 + + buf[pos] = byte(enc.Nat(value.BootstrapTime).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) +} + +func (encoder *MembershipTupleEncoder) Encode(value *MembershipTuple) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *MembershipTupleParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + + var handled_Name bool = false + var handled_BootstrapTime bool = false + + progress := -1 + _ = progress + + value := &MembershipTuple{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7: + if true { + handled = true + handled_Name = true + delegate := reader.Delegate(int(l)) + value.Name, err = delegate.ReadName() + } + case 212: + if true { + handled = true + handled_BootstrapTime = true + value.BootstrapTime = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.BootstrapTime = uint64(value.BootstrapTime<<8) | uint64(x) + } + } + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Name && err == nil { + value.Name = nil + } + if !handled_BootstrapTime && err == nil { + err = enc.ErrSkipRequired{Name: "BootstrapTime", TypeNum: 212} + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *MembershipTuple) Encode() enc.Wire { + encoder := MembershipTupleEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *MembershipTuple) Bytes() []byte { + return value.Encode().Join() +} + +func ParseMembershipTuple(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + context := MembershipTupleParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + type PassiveStateEncoder struct { Length uint diff --git a/std/sync/svs.go b/std/sync/svs.go index 60e71015..e87455aa 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -40,6 +40,13 @@ type SvSync struct { // Channel for incoming state vectors recvSv chan svSyncRecvSvArgs + // Prefix for published full State Vector Data (.../32=sv). + fullVectorPrefix enc.Name + + // lastPullTime debounces pullFullVector per sender so a sync storm across + // many peers does not generate thousands of redundant segment-0 fetches. + lastPullTime map[string]time.Time + // cancellation for face hook faceCancel func() } @@ -58,6 +65,13 @@ type SvSyncOpts struct { // If not provided, the GroupPrefix will be used instead. SyncDataName enc.Name + // FullVectorPrefix is the publish/serve prefix for retrievable FULL + // StateVector Data used by SvsDataRef publish+pull recovery. The + // version component is appended when producing the Data. + // If not provided, it defaults to SyncDataName with the trailing + // "32=svs" component (if present) replaced by "32=sv". + FullVectorPrefix enc.Name + // Initial state vector from persistence InitialState *spec_svs.StateVector // Boot time from persistence @@ -73,6 +87,14 @@ type SvSyncOpts struct { UseSignatureTime optional.Optional[bool] // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] + + // SyncVectorThreshold is the max embedded SvsData size (bytes) above + // which the sender switches to PARTIAL (on publication) or + // publish+pull (on periodic sync and recovery). When <= 0, the + // default (1200 bytes) is used. SVS v4 always emits `mhash` and a + // `VectorType` on the wire; there is no legacy StateVector-only + // mode. + SyncVectorThreshold int } type SvSyncUpdate struct { @@ -83,8 +105,11 @@ type SvSyncUpdate struct { } type svSyncRecvSvArgs struct { - sv *spec_svs.StateVector - data enc.Wire + sv *spec_svs.StateVector + data enc.Wire + vectorType optional.Optional[uint64] + mhash []byte + svsDataRef enc.Name } // NewSvSync creates a new SV Sync instance. @@ -124,6 +149,9 @@ func NewSvSync(opts SvSyncOpts) *SvSync { if len(opts.SyncDataName) == 0 { opts.SyncDataName = opts.GroupPrefix } + if opts.SyncVectorThreshold <= 0 { + opts.SyncVectorThreshold = 1200 + } return &SvSync{ o: opts, @@ -135,7 +163,7 @@ func NewSvSync(opts SvSyncOpts) *SvSync { mutex: sync.Mutex{}, state: initialState, mtime: make(map[string]time.Time), - prefix: opts.GroupPrefix.Append(enc.NewVersionComponent(3)), + prefix: opts.GroupPrefix.Append(enc.NewVersionComponent(4)), suppress: false, merge: NewSvMap[uint64](0), @@ -145,6 +173,10 @@ func NewSvSync(opts SvSyncOpts) *SvSync { recvSv: make(chan svSyncRecvSvArgs, 128), + fullVectorPrefix: resolveFullVectorPrefix(opts.FullVectorPrefix, opts.SyncDataName), + + lastPullTime: make(map[string]time.Time), + faceCancel: func() {}, } } @@ -169,7 +201,6 @@ func (s *SvSync) Start() (err error) { return nil } -// (AI GENERATED DESCRIPTION): Runs the SvSync event loop: it performs the initial sync (or passive load), registers periodic timer ticks and face‑up callbacks, processes received state vectors, and exits cleanly when signalled to stop. func (s *SvSync) main() { // Cleanup on exit defer s.o.Client.Engine().DetachHandler(s.prefix) @@ -180,7 +211,7 @@ func (s *SvSync) main() { // Notify everyone when we are back online s.faceCancel = s.o.Client.Engine().Face().OnUp(func() { - time.AfterFunc(100*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(100*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) }) defer s.faceCancel() @@ -190,7 +221,7 @@ func (s *SvSync) main() { go s.loadPassiveWires() } else { // Send the initial Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendOther) } for { @@ -242,7 +273,7 @@ func (s *SvSync) SetSeqNo(name enc.Name, seqNo uint64) error { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest s.state.Set(hash, s.o.BootTime, seqNo) - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return nil } @@ -264,17 +295,15 @@ func (s *SvSync) IncrSeqNo(name enc.Name) uint64 { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return entry } -// (AI GENERATED DESCRIPTION): Returns the boot time value stored in the SvSync instance. func (s *SvSync) GetBootTime() uint64 { return s.o.BootTime } -// (AI GENERATED DESCRIPTION): Returns a thread‑safe slice of all names currently stored in the SvSync state. func (s *SvSync) GetNames() []enc.Name { s.mutex.Lock() defer s.mutex.Unlock() @@ -287,7 +316,6 @@ func (s *SvSync) GetNames() []enc.Name { return names } -// (AI GENERATED DESCRIPTION): Processes an incoming state vector, updating the local state vector, notifying the application of any changes, and handling suppression and passive‑sync logic while ensuring updates are delivered in order. func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // Deliver the updates after this call is done // This ensures the mutex is not held during the callback @@ -372,7 +400,16 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // The above checks each node in the incoming state vector, but // does not check if a node is missing from the incoming state vector. - if !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { + // + // [Spec] For embedded SvsData, VectorType is required by the protocol: + // publish-only Sync Data carries no StateVector and is filtered out + // earlier (see onSyncData). So args.vectorType is guaranteed present + // here; we default missing values to FULL rather than branch on `ok`. + isPartial := args.vectorType.GetOr(spec_svs.VectorTypeFull) == spec_svs.VectorTypePartial + if len(args.mhash) > 0 { + s.handleMhashMismatch(args, recvSv) + } + if !isPartial && !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { isOutdated = true canDrop = false } @@ -402,7 +439,6 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { s.ticker.Reset(s.getSuppressionTimeout()) } -// (AI GENERATED DESCRIPTION): Handles a timer expiry by checking suppression state, potentially transitioning to steady state, and asynchronously sending a Sync Interest with the current local state vector. func (s *SvSync) timerExpired() { s.mutex.Lock() defer s.mutex.Unlock() @@ -420,11 +456,10 @@ func (s *SvSync) timerExpired() { // [Spec] On expiration of timer emit a Sync Interest // with the current local state vector. - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPeriodic) } -// (AI GENERATED DESCRIPTION): Sends a sync Interest: if passive mode is enabled, it publishes all buffered state updates without duplicates; otherwise, it encodes the current state vector into a wire and transmits it, provided the sync service is running. -func (s *SvSync) sendSyncInterest() { +func (s *SvSync) sendSyncInterest(reason syncSendReason, pubName ...enc.Name) { if !s.running.Load() { return } @@ -437,12 +472,16 @@ func (s *SvSync) sendSyncInterest() { return } + var sender enc.Name + if reason == syncSendPublication && len(pubName) > 0 { + sender = pubName[0] + } + // Encode and sign the current state vector - wire := s.encodeSyncData() + wire := s.encodeSyncData(reason, sender) s.sendSyncInterestWith(wire) } -// (AI GENERATED DESCRIPTION): Sends a sync Interest carrying the supplied data wire payload with a 1‑second lifetime, using the object’s prefix, and logs any construction or transmission errors. func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { if dataWire == nil { return @@ -465,21 +504,52 @@ func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { } } -// (AI GENERATED DESCRIPTION): Builds a signed Data packet containing the current state vector for SVS v3 synchronization. -func (s *SvSync) encodeSyncData() enc.Wire { - // Critical section - sv := func() *spec_svs.StateVector { - s.mutex.Lock() - defer s.mutex.Unlock() - - // [Spec*] Sending always triggers Steady State - s.enterSteadyState() - - return s.state.Encode(func(s uint64) uint64 { return s }) - }() - svWire := (&spec_svs.SvsData{StateVector: sv}).Encode() +func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire { + s.mutex.Lock() + s.enterSteadyState() + stateSnap := cloneSvMap(s.state) + mtimeSnap := make(map[string]time.Time, len(s.mtime)) + for k, v := range s.mtime { + mtimeSnap[k] = v + } + repair, propagation := s.partialTargets() + s.mutex.Unlock() + + var svsData *spec_svs.SvsData + if shouldUseAnnouncePull(reason, s.o.SyncVectorThreshold, stateSnap) { + ref, err := s.publishFullVectorData(stateSnap) + if err != nil { + log.Error(s, "publishFullVectorData failed", "err", err) + return nil + } + svsData = buildAnnounceSvsData(stateSnap, ref) + } else { + svsData = buildSvsDataForSend(svsSendInput{ + State: stateSnap, + Reason: reason, + Threshold: s.o.SyncVectorThreshold, + Sender: sender, + Repair: repair, + Propagation: propagation, + Mtime: mtimeSnap, + }) + if svsData == nil { + // [Spec] Publication-triggered PARTIAL encoding could not fit + // even the sender-only baseline: fall back to publish+pull. + ref, err := s.publishFullVectorData(stateSnap) + if err != nil { + log.Error(s, "publishFullVectorData failed (fallback)", "err", err) + return nil + } + svsData = buildAnnounceSvsData(stateSnap, ref) + } + } + if svsData == nil { + return nil + } + svWire := svsData.Encode() - // SVS v3 Sync Data + // SVS v4 Sync Data name := s.o.SyncDataName.WithVersion(enc.VersionUnixMicro) // Sign Sync Data @@ -500,7 +570,6 @@ func (s *SvSync) encodeSyncData() enc.Wire { return data.Wire } -// (AI GENERATED DESCRIPTION): Handles a received sync Interest by checking the running state, extracting its AppParam, and passing that payload to the sync‑data processing routine. func (s *SvSync) onSyncInterest(interest ndn.Interest) { if !s.running.Load() { return @@ -516,7 +585,6 @@ func (s *SvSync) onSyncInterest(interest ndn.Interest) { s.onSyncData(interest.AppParam()) } -// (AI GENERATED DESCRIPTION): Processes a received SyncData packet by parsing it, validating the signature, extracting the state vector, and forwarding the vector and original data to the receiver channel. func (s *SvSync) onSyncData(dataWire enc.Wire) { data, sigCov, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) if err != nil { @@ -536,18 +604,36 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } - // Decode state vector + // Decode SvsData (embedded FULL, embedded PARTIAL, or publish-only ref). svWire := data.Content().Join() params, err := spec_svs.ParseSvsData(enc.NewBufferView(svWire), false) - if err != nil || params.StateVector == nil { - log.Warn(s, "onSyncInterest failed to parse StateVec", "err", err) + if err != nil { + log.Warn(s, "onSyncInterest failed to parse SvsData", "err", err) + return + } + + // Publish-only ref: advertise that the full vector is retrievable. + if params.StateVector == nil && len(params.SvsDataRef) > 0 { + trustPrefix := pullRefFromSyncDataWire(dataWire) + go s.pullFullVector(params.SvsDataRef, trustPrefix) + return + } + if params.StateVector == nil { + log.Warn(s, "onSyncInterest SvsData has no StateVector") return } - s.recvSv <- svSyncRecvSvArgs{ - sv: params.StateVector, - data: dataWire, + args := svSyncRecvSvArgs{ + sv: params.StateVector, + data: dataWire, + mhash: params.MemberSetHash, + svsDataRef: params.SvsDataRef, } + if vt, ok := params.VectorType.Get(); ok { + args.vectorType = optional.Some(vt) + } + + s.recvSv <- args }, }) } @@ -559,7 +645,6 @@ func (s *SvSync) enterSteadyState() { s.ticker.Reset(s.getPeriodicTimeout()) } -// (AI GENERATED DESCRIPTION): Returns a duration uniformly randomized within ±10% of the configured periodic timeout. func (s *SvSync) getPeriodicTimeout() time.Duration { // [Spec] ±10% uniform jitter jitter := s.o.PeriodicTimeout / 10 @@ -568,7 +653,6 @@ func (s *SvSync) getPeriodicTimeout() time.Duration { return time.Duration(rand.Int64N(int64(max-min))) + min } -// (AI GENERATED DESCRIPTION): Calculates a random suppression timeout duration using an exponential‑decay function based on the configured SuppressionPeriod. func (s *SvSync) getSuppressionTimeout() time.Duration { // [Spec] Exponential decay function // [Spec] c = SuppressionPeriod // constant factor @@ -664,5 +748,16 @@ func (s *SvSync) loadPassiveWires() { } // This is hacky but pragmatic - wait for the state to be processed - time.AfterFunc(500*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(500*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) +} + +// partialTargets returns repair and propagation name targets from suppression merge state. +func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { + if !s.suppress { + return nil, nil + } + for name := range s.merge.Iter() { + repair = append(repair, name) + } + return repair, nil } diff --git a/std/sync/svs_alo_data.go b/std/sync/svs_alo_data.go index 27543385..7be450b8 100644 --- a/std/sync/svs_alo_data.go +++ b/std/sync/svs_alo_data.go @@ -25,7 +25,6 @@ type svsDataState struct { SnapBlock int } -// (AI GENERATED DESCRIPTION): Builds the full name for a data object by appending the node identifier, boot‑timestamp, and sequence number to the group’s prefix and marking the resulting name as immutable. func (s *SvsALO) objectName(node enc.Name, boot uint64, seq uint64) enc.Name { return s.GroupPrefix(). Append(node...). @@ -34,7 +33,6 @@ func (s *SvsALO) objectName(node enc.Name, boot uint64, seq uint64) enc.Name { WithVersion(enc.VersionImmutable) } -// (AI GENERATED DESCRIPTION): Publishes a new Data object with the supplied content, updates the SVS state vector and snapshot strategy, and returns the produced name and the instance’s serialized state. func (s *SvsALO) produceObject(content enc.Wire) (enc.Name, enc.Wire, error) { // This instance owns the underlying SVS instance. // So we can be sure that the sequence number does not @@ -118,6 +116,8 @@ func (s *SvsALO) consumeObject(node enc.Name, boot uint64, seq uint64) { fetchName := s.objectName(node, boot, seq) s.client.ConsumeExt(ndn.ConsumeExtArgs{ Name: fetchName, + TryStore: true, + NoMetadata: true, // fetch name includes version+seq; metadata would only block on timeout UseSignatureTime: s.opts.Svs.UseSignatureTime, IgnoreValidity: s.opts.Svs.IgnoreValidity, Callback: func(status ndn.ConsumeState) { diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go new file mode 100644 index 00000000..2fcb394c --- /dev/null +++ b/std/sync/svs_encode.go @@ -0,0 +1,258 @@ +package sync + +import ( + "cmp" + "math/rand/v2" + "slices" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +// syncSendReason distinguishes why a Sync Interest is being sent. +type syncSendReason int + +const ( + syncSendOther syncSendReason = iota + syncSendPublication + syncSendPeriodic + syncSendRecovery +) + +// PartialEncodeOpts configures subset selection for inline PARTIAL vectors. +type PartialEncodeOpts struct { + Sender enc.Name + Threshold int + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + +// svsSendInput carries everything needed to build inline Sync Data for send. +type svsSendInput struct { + State SvMap[uint64] + Reason syncSendReason + Threshold int + Sender enc.Name + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + +// buildSvsDataForSend picks embedded FULL or PARTIAL SvsData for an outgoing +// Sync message. Returns nil when publication-triggered PARTIAL encoding cannot +// fit even the sender-only baseline: the caller MUST fall back to publish+pull +// (see shouldUseAnnouncePull). Other reasons always return a non-nil result. +func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { + fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) + fullData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(in.State), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: fullSv, + } + + if in.Reason != syncSendPublication || len(fullData.Encode().Join()) <= in.Threshold { + return fullData + } + + partialSv := encodePartialStateVector(in.State, PartialEncodeOpts{ + Sender: in.Sender, + Threshold: in.Threshold, + Repair: in.Repair, + Propagation: in.Propagation, + Mtime: in.Mtime, + }) + if partialSv == nil { + // Baseline exceeded Threshold; caller must use publish+pull. + return nil + } + return &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(in.State), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: partialSv, + } +} + +// encodePartialStateVector builds a PARTIAL StateVector for new publication. +// Entry [0] is the sender; entries [1..n] are in NDN canonical order. +// +// Returns nil if the sender-only baseline itself exceeds Threshold: +// callers MUST fall back to publish+pull in that case, because including +// the sender entry is required by §4.2 of the v4 spec. +func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { + seq := func(v uint64) uint64 { return v } + senderHash := opts.Sender.TlvStr() + + senderEntry := state.encodeNameEntry(opts.Sender, seq) + if senderEntry == nil { + senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} + } + + // Sender-only baseline must always fit when possible. + baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} + baselineData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: baseline, + } + if len(baselineData.Encode().Join()) > opts.Threshold { + // Caller falls back to publish+pull because we cannot satisfy + // the §4.2 "entry [0] is the sender" rule at this size budget. + return nil + } + + candidates := partialCandidateNames(state, senderHash, opts) + included := map[string]bool{senderHash: true} + entries := []*spec_svs.StateVectorEntry{senderEntry} + + for _, name := range candidates { + hash := name.TlvStr() + if included[hash] { + continue + } + entry := state.encodeNameEntry(name, seq) + if entry == nil { + continue + } + + trial := append(slices.Clone(entries), entry) + sortPartialTail(trial) + trialSv := &spec_svs.StateVector{Entries: trial} + trialData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: trialSv, + } + if len(trialData.Encode().Join()) > opts.Threshold { + break + } + + entries = trial + included[hash] = true + } + + sortPartialTail(entries) + return &spec_svs.StateVector{Entries: entries} +} + +// partialCandidateNames returns the producer names considered for inclusion +// in a PARTIAL StateVector, in priority order: +// +// 1. Repair targets from the suppression-merge state (newest entries first). +// 2. Propagation targets (the most recently updated producers). +// 3. Inactive producers (zero-value entries) in randomized order — these +// are included only when bandwidth allows, so randomization is fair. +// 4. Remaining active producers, sorted by (a) recency descending then +// (b) canonical NDN name ascending. +// +// The sender is excluded — it is always included at entries[0]. +func partialCandidateNames(state SvMap[uint64], senderHash string, opts PartialEncodeOpts) []enc.Name { + seen := map[string]bool{senderHash: true} + out := make([]enc.Name, 0, len(state)) + + appendUnique := func(names []enc.Name) { + for _, name := range names { + hash := name.TlvStr() + if seen[hash] { + continue + } + if _, ok := state[hash]; !ok { + continue + } + seen[hash] = true + out = append(out, name) + } + } + + appendUnique(opts.Repair) + appendUnique(opts.Propagation) + + inactive := make([]enc.Name, 0) + remaining := make([]enc.Name, 0) + for name, vals := range state.Iter() { + hash := name.TlvStr() + if seen[hash] { + continue + } + if isInactiveProducer(vals) { + inactive = append(inactive, name) + continue + } + remaining = append(remaining, name) + } + + rand.Shuffle(len(inactive), func(i, j int) { + inactive[i], inactive[j] = inactive[j], inactive[i] + }) + appendUnique(inactive) + + slices.SortFunc(remaining, func(a, b enc.Name) int { + return a.Compare(b) + }) + slices.SortFunc(remaining, func(a, b enc.Name) int { + return cmp.Compare(recencyScore(opts.Mtime, b), recencyScore(opts.Mtime, a)) + }) + appendUnique(remaining) + + return out +} + +func isInactiveProducer(vals []SvMapVal[uint64]) bool { + for _, val := range vals { + if val.Value > 0 { + return false + } + } + return true +} + +func recencyScore(mtime map[string]time.Time, name enc.Name) int64 { + if mtime == nil { + return 0 + } + t, ok := mtime[name.TlvStr()] + if !ok { + return 0 + } + return t.UnixNano() +} + +// sortPartialTail keeps entry [0] fixed and sorts [1..n] in canonical name order. +// +// [Spec §4.2] Entry [0] of a PARTIAL StateVector is the sender; remaining +// entries are NOT ordered by membership hash like MemberSet entries are — +// they are ordered by canonical NDN name comparison. StateVectorEntry +// ordering is independent of mhash ordering. +func sortPartialTail(entries []*spec_svs.StateVectorEntry) { + if len(entries) <= 1 { + return + } + slices.SortFunc(entries[1:], func(a, b *spec_svs.StateVectorEntry) int { + return a.Name.Compare(b.Name) + }) +} + +// encodeNameEntry encodes one producer name from the map. +func (m SvMap[V]) encodeNameEntry(name enc.Name, seq func(V) uint64) *spec_svs.StateVectorEntry { + hash := name.TlvStr() + vals, ok := m[hash] + if !ok { + return nil + } + + entry := &spec_svs.StateVectorEntry{ + Name: name, + SeqNoEntries: make([]*spec_svs.SeqNoEntry, 0, len(vals)), + } + for _, val := range vals { + if seqNo := seq(val.Value); seqNo > 0 { + entry.SeqNoEntries = append(entry.SeqNoEntries, &spec_svs.SeqNoEntry{ + BootstrapTime: val.Boot, + SeqNo: seqNo, + }) + } + } + return entry +} diff --git a/std/sync/svs_map.go b/std/sync/svs_map.go index be1ffd0c..3be8e1eb 100644 --- a/std/sync/svs_map.go +++ b/std/sync/svs_map.go @@ -19,7 +19,6 @@ type SvMapVal[V any] struct { Value V } -// (AI GENERATED DESCRIPTION): Compares the Boot field of two SvMapVal[V] values, returning a negative, zero, or positive integer to indicate their ordering. func (*SvMapVal[V]) Cmp(a, b SvMapVal[V]) int { return cmp.Compare(a.Boot, b.Boot) } @@ -29,6 +28,15 @@ func NewSvMap[V any](size int) SvMap[V] { return make(SvMap[V], size) } +// cloneSvMap returns a shallow copy safe for use without holding SvSync.mutex. +func cloneSvMap[V any](m SvMap[V]) SvMap[V] { + out := NewSvMap[V](len(m)) + for hash, vals := range m { + out[hash] = slices.Clone(vals) + } + return out +} + // Get seq entry for a bootstrap time. func (m SvMap[V]) Get(hash string, boot uint64) (value V) { entry := SvMapVal[V]{boot, value} @@ -39,7 +47,6 @@ func (m SvMap[V]) Get(hash string, boot uint64) (value V) { return value } -// (AI GENERATED DESCRIPTION): Adds or updates a value in the sorted list for a given hash, inserting the new entry or replacing the existing one while maintaining the slice sorted by the boot field. func (m SvMap[V]) Set(hash string, boot uint64, value V) { entry := SvMapVal[V]{boot, value} i, match := slices.BinarySearchFunc(m[hash], entry, entry.Cmp) @@ -50,7 +57,6 @@ func (m SvMap[V]) Set(hash string, boot uint64, value V) { m[hash] = slices.Insert(m[hash], i, entry) } -// (AI GENERATED DESCRIPTION): Clears all key/value pairs from the SvMap, safely handling nil maps by doing nothing if the map is nil. func (m SvMap[V]) Clear() { if m != nil { clear(m) @@ -118,7 +124,6 @@ func (m SvMap[V]) Encode(seq func(V) uint64) *spec_svs.StateVector { return &spec_svs.StateVector{Entries: entries} } -// (AI GENERATED DESCRIPTION): Iter returns an iterator over the SvMap that yields each decoded name and its associated slice of SvMapVal values. func (m SvMap[V]) Iter() iter.Seq2[enc.Name, []SvMapVal[V]] { return func(yield func(enc.Name, []SvMapVal[V]) bool) { for hash, val := range m { diff --git a/std/sync/svs_membership_hash.go b/std/sync/svs_membership_hash.go new file mode 100644 index 00000000..908729a9 --- /dev/null +++ b/std/sync/svs_membership_hash.go @@ -0,0 +1,42 @@ +package sync + +import ( + "crypto/sha256" + "slices" + + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" +) + +// ComputeMembershipHash returns the membership hash over all (Name, BootstrapTime) pairs in state. +// Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime +// children) using the ndnd standard TLV codec. +func ComputeMembershipHash(state SvMap[uint64]) []byte { + tuples := make([]*spec_svs.MembershipTuple, 0) + for name, vals := range state.Iter() { + for _, val := range vals { + tuples = append(tuples, &spec_svs.MembershipTuple{ + Name: name, + BootstrapTime: val.Boot, + }) + } + } + + slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { + if c := a.Name.Compare(b.Name); c != 0 { + return c + } + if a.BootstrapTime < b.BootstrapTime { + return -1 + } + if a.BootstrapTime > b.BootstrapTime { + return 1 + } + return 0 + }) + + h := sha256.New() + for _, t := range tuples { + h.Write(t.Encode().Join()) + } + return h.Sum(nil) +} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go new file mode 100644 index 00000000..edf73638 --- /dev/null +++ b/std/sync/svs_pull.go @@ -0,0 +1,272 @@ +package sync + +import ( + "bytes" + "fmt" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +const ( + syncDataKeyword = "svs" + fullVectorKeyword = "sv" +) + +// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). +func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { + if len(syncDataName) == 0 { + return nil + } + base := syncDataName + if base.At(-1).IsKeyword(syncDataKeyword) { + base = base.Prefix(-1) + } + return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) +} + +// resolveFullVectorPrefix returns the explicit FullVectorPrefix if set, +// otherwise derives it from SyncDataName. +func resolveFullVectorPrefix(explicit, syncDataName enc.Name) enc.Name { + if len(explicit) > 0 { + return explicit.Clone() + } + return deriveFullVectorPrefix(syncDataName) +} + +// pullRefFromSyncDataWire returns the trust prefix for fetching a publish-only +// SvsDataRef. The "ref" field of Sync Data is the published full-vector name +// (.../32=sv/); the trust prefix is the same name with the version +// component stripped, which corresponds to the sender's .../32=sv prefix used +// for all its published full vectors and is what an authorized consumer must +// trust to follow the reference. +func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { + data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) + if err != nil { + return nil + } + name := data.Name() + if len(name) == 0 { + return nil + } + if name.At(-1).IsVersion() { + name = name.Prefix(-1) + } + return deriveFullVectorPrefix(name) +} + +func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { + return &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + SvsDataRef: ref, + } +} + +// shouldUseAnnouncePull reports whether the sender should publish at .../32=sv +// and emit publish-only Sync Data (mhash + SvsDataRef, no embedded vector) +// instead of an embedded FULL or PARTIAL StateVector. +func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if reason == syncSendRecovery { + return true + } + if reason == syncSendPublication { + return false + } + sv := state.Encode(func(seq uint64) uint64 { return seq }) + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + return len(full.Encode().Join()) > threshold +} + +// publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. +func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { + if len(s.fullVectorPrefix) == 0 { + return nil, fmt.Errorf("full vector prefix unset") + } + sv := state.Encode(func(seq uint64) uint64 { return seq }) + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode() + name := s.fullVectorPrefix.WithVersion(enc.VersionUnixMicro) + return s.o.Client.Produce(ndn.ProduceArgs{ + Name: name, + Content: content, + }) +} + +// pullFullVectorMinInterval debounces pullFullVector per sender. During convergence on a +// large group, every sync that crosses the membership hash boundary schedules a pull; without +// gating, a node can accumulate redundant segment-0 fetches for the same content, which +// exhausts retry budgets under network load. We allow at most one pull per sender per +// pullFullVectorMinInterval. +const pullFullVectorMinInterval = 5 * time.Second + +// pullFullVector fetches a published full State Vector and merges it on the main loop. +// trustPrefix is the sender's .../32=sv prefix; ref must be equal to or below it. +// +// [Impl] The 5s per-sender debounce (pullFullVectorMinInterval) is an +// implementation detail documented in §5.6 of the v4 spec: it limits the +// fan-in when many peers cross an mhash boundary at the same time and is +// safe to relax provided the consumer's retry budget scales accordingly. +func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { + if len(ref) == 0 { + return + } + if !isTrustedSvsDataRef(ref, trustPrefix) { + log.Warn(s, "pullFullVector rejected untrusted SvsDataRef", "ref", ref, "trust", trustPrefix) + return + } + + // Debounce per sender: drop the pull if one is already in flight or completed recently. + senderHash := trustPrefix.TlvStr() + s.mutex.Lock() + if last, ok := s.lastPullTime[senderHash]; ok && time.Since(last) < pullFullVectorMinInterval { + s.mutex.Unlock() + return + } + s.lastPullTime[senderHash] = time.Now() + s.mutex.Unlock() + + s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ + Name: ref.Clone(), + TryStore: true, + NoMetadata: true, + UseSignatureTime: s.o.UseSignatureTime, + IgnoreValidity: s.o.IgnoreValidity, + Callback: func(st ndn.ConsumeState) { + if st.Error() != nil { + log.Warn(s, "pullFullVector failed", "ref", ref, "err", st.Error()) + return + } + if !st.IsComplete() { + return + } + s.onPulledFullVector(st.Content().Join()) + }, + }) +} + +// onPulledFullVector merges a fetched inline FULL SvsData into local state. +// Segment signatures are validated by client.ConsumeExt during fetch. +func (s *SvSync) onPulledFullVector(content []byte) { + params, err := parseFullVectorContent(content) + if err != nil { + log.Warn(s, "onPulledFullVector parse failed", "err", err) + return + } + + s.recvSv <- svSyncRecvSvArgs{ + sv: params.StateVector, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: params.MemberSetHash, + } +} + +func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { + params, err := spec_svs.ParseSvsData(enc.NewBufferView(content), false) + if err != nil { + return nil, err + } + if params.StateVector == nil { + return nil, fmt.Errorf("full vector content has no StateVector") + } + if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { + return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) + } + if len(params.MemberSetHash) > 0 { + computed := ComputeMembershipHash(stateVectorToMap(params.StateVector)) + if !bytes.Equal(params.MemberSetHash, computed) { + return nil, fmt.Errorf("full vector mhash mismatch") + } + } + return params, nil +} + +func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { + m := NewSvMap[uint64](len(sv.Entries)) + for _, node := range sv.Entries { + hash := node.Name.TlvStr() + for _, entry := range node.SeqNoEntries { + m.Set(hash, entry.BootstrapTime, entry.SeqNo) + } + } + return m +} + +// sendRecoveryAnnounce publishes at 32=sv and emits announce-only Sync Data (mhash recovery). +func (s *SvSync) sendRecoveryAnnounce() { + if !s.running.Load() || s.o.Passive { + return + } + wire := s.encodeSyncData(syncSendRecovery, enc.Name{}) + s.sendSyncInterestWith(wire) +} + +// handleMhashMismatch schedules announce or pull recovery on membership mismatch. +func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { + localMhash := ComputeMembershipHash(s.state) + if bytes.Equal(localMhash, args.mhash) { + return + } + + trustPrefix := pullRefFromSyncDataWire(args.data) + if len(trustPrefix) == 0 { + trustPrefix = s.fullVectorPrefix + } + + localTuples, remoteTuples := membershipTupleCount(s.state), membershipTupleCount(recvSv) + if localTuples > remoteTuples && membershipContains(s.state, recvSv) { + go s.sendRecoveryAnnounce() + return + } + + // [Spec] Inline FULL is already merged in onReceiveStateVector. + // Pull only when the sender provided a retrievable SvsDataRef (publish-only sync). + if len(args.svsDataRef) == 0 { + return + } + go s.pullFullVector(args.svsDataRef, trustPrefix) +} + +func membershipContains(outer, inner SvMap[uint64]) bool { + for hash, vals := range inner { + for _, v := range vals { + found := false + for _, ov := range outer[hash] { + if ov.Boot == v.Boot { + found = true + break + } + } + if !found { + return false + } + } + } + return true +} + +func membershipTupleCount(m SvMap[uint64]) int { + n := 0 + for _, vals := range m { + n += len(vals) + } + return n +} + +func isTrustedSvsDataRef(ref, senderFullVectorPrefix enc.Name) bool { + if len(ref) == 0 || len(senderFullVectorPrefix) == 0 { + return false + } + return senderFullVectorPrefix.IsPrefix(ref) +} diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go new file mode 100644 index 00000000..daf5d56b --- /dev/null +++ b/std/sync/svs_test.go @@ -0,0 +1,479 @@ +package sync + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + sig "github.com/named-data/ndnd/std/security/signer" + "github.com/named-data/ndnd/std/types/optional" + tu "github.com/named-data/ndnd/std/utils/testutils" +) + +// --- shared test helpers and encode tests --- + +func testSvMapAliceBob() SvMap[uint64] { + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 5) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + return m +} + +func TestBuildInlineFullSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + data := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + + require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) + vt, ok := data.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) + require.NotNil(t, data.StateVector) + require.Len(t, data.StateVector.Entries, 2) +} + +func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + // PARTIAL with only bob; local knows alice — must not enter suppression. + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.False(t, s.suppress) +} + +func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: fullSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.True(t, s.suppress) +} + +func TestEncodePartialSenderFirst(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + carol := tu.NoErr(enc.NameFromStr("/ndn/carol")) + + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + m.Set(bob.TlvStr(), 150, 3) + m.Set(carol.TlvStr(), 150, 7) + + // Threshold large enough for sender + one peer. + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) - 1 + + partial := encodePartialStateVector(m, PartialEncodeOpts{ + Sender: carol, + Threshold: threshold, + Mtime: map[string]time.Time{ + alice.TlvStr(): time.Unix(10, 0), + bob.TlvStr(): time.Unix(20, 0), + }, + }) + + require.NotEmpty(t, partial.Entries) + require.Equal(t, carol, partial.Entries[0].Name) + if len(partial.Entries) > 2 { + require.Less(t, partial.Entries[1].Name.Compare(partial.Entries[2].Name), 0) + } +} + +func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + for i := range 20 { + name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/ndn/peer%d", i))) + m.Set(name.TlvStr(), 150, uint64(i+1)) + } + + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) / 2 + + pub := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, + }) + vt, ok := pub.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypePartial, vt) + require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) + + periodic := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, + }) + vt, ok = periodic.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) +} + +func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + s.state.Set(alice.TlvStr(), 100, 1) + s.state.Set(bob.TlvStr(), 150, 1) + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(bob.TlvStr(), 150, 4) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.Len(t, updates, 1) + require.Equal(t, bob, updates[0].Name) + require.EqualValues(t, 4, updates[0].High) + require.EqualValues(t, 1, s.state.Get(alice.TlvStr(), 100)) + require.EqualValues(t, 4, s.state.Get(bob.TlvStr(), 150)) +} + +// --- mhash tests --- + +func TestComputeMembershipHashStable(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + h1 := ComputeMembershipHash(m) + h2 := ComputeMembershipHash(m) + require.Equal(t, h1, h2) + require.Len(t, h1, 32) +} + +func TestComputeMembershipHashOrderIndependent(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + require.Equal(t, ComputeMembershipHash(m1), ComputeMembershipHash(m2)) +} + +func TestComputeMembershipHashChangesOnMembership(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + before := ComputeMembershipHash(m) + + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + after := ComputeMembershipHash(m) + require.NotEqual(t, before, after) +} + +func TestComputeMembershipHashIgnoresSeqNo(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 99) + + require.Equal(t, ComputeMembershipHash(m1), ComputeMembershipHash(m2)) +} + +func TestSvsDataInlineTLV(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + sv := m.Encode(func(s uint64) uint64 { return s }) + + original := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) + require.Equal(t, original.VectorType, parsed.VectorType) + require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) +} + +func TestSvsDataAnnounceTLV(t *testing.T) { + tu.SetT(t) + + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) + mhash := make([]byte, 32) + + original := &spec_svs.SvsData{ + MemberSetHash: mhash, + SvsDataRef: ref, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, mhash, parsed.MemberSetHash) + require.Equal(t, ref.String(), parsed.SvsDataRef.String()) + require.Nil(t, parsed.StateVector) + require.False(t, parsed.VectorType.IsSet()) +} + +func TestSvsDataLegacyParse(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} + wire := legacy.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Nil(t, parsed.MemberSetHash) + require.NotNil(t, parsed.StateVector) +} + +// --- pull / recovery tests --- + +func TestDeriveFullVectorPrefix(t *testing.T) { + tu.SetT(t) + + syncData := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")) + prefix := deriveFullVectorPrefix(syncData) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", prefix.String()) +} + +func TestPullRefFromSyncDataWire(t *testing.T) { + tu.SetT(t) + + syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")). + Append(enc.NewVersionComponent(12345)) + dataWire, err := spec.Spec{}.MakeData( + syncDataName, + &ndn.DataConfig{ContentType: optional.Some(ndn.ContentTypeBlob)}, + enc.Wire{enc.Buffer{0x01}}, + sig.NewSha256Signer(), + ) + require.NoError(t, err) + + ref := pullRefFromSyncDataWire(dataWire.Wire) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) +} + +func TestBuildAnnounceSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) + data := buildAnnounceSvsData(m, ref) + + require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) + require.True(t, ref.Equal(data.SvsDataRef)) + require.Nil(t, data.StateVector) + require.False(t, data.VectorType.IsSet()) + + wire := data.Encode().Join() + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) + require.True(t, ref.Equal(parsed.SvsDataRef)) + require.Nil(t, parsed.StateVector) +} + +func TestShouldUseAnnouncePull(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode().Join()) + + require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) +} + +func TestIsTrustedSvsDataRef(t *testing.T) { + tu.SetT(t) + + trust := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv")) + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/999")) + bad := tu.NoErr(enc.NameFromStr("/ndn/evil/32=sv/1")) + + require.True(t, isTrustedSvsDataRef(ref, trust)) + require.True(t, isTrustedSvsDataRef(trust, trust)) + require.False(t, isTrustedSvsDataRef(bad, trust)) + require.False(t, isTrustedSvsDataRef(ref, nil)) +} + +func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: []byte("not-a-valid-mhash-padding-000000"), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + +func TestParseFullVectorContent(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + parsed, err := parseFullVectorContent(wire) + require.NoError(t, err) + require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) + require.Len(t, parsed.StateVector.Entries, 2) +} + +func TestOnPulledFullVectorMergesState(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + recvSv: make(chan svSyncRecvSvArgs, 1), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + s.state.Set(alice.TlvStr(), 100, 1) + + remote := testSvMapAliceBob() + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(remote), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: remote.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join() + + go func() { + s.onPulledFullVector(content) + }() + + s.onReceiveStateVector(<-s.recvSv) + + require.Len(t, updates, 2) + require.EqualValues(t, 3, s.state.Get(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150)) + require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) +} + +func TestEncodeSyncDataAnnounceMode(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join()) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + + announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) + require.Nil(t, announce.StateVector) + vt, ok := announce.VectorType.Get() + require.False(t, ok || vt == spec_svs.VectorTypePartial) +}