Separate EBML from Matroska, add another EBML-based format: MOSAIC

.golangci.yml

@@ -81,7 +81,7 @@ linters:
       # ignore warnings in code from crypto/tls and zmap/zcrypto
       - format/tls/tlsdecrypt
       - format/fuzz_test.go
-      - format/matroska/ebml/gen/main.go
+      - format/ebml/gen/main.go
 formatters:
   enable:
     - goimports

README.md

@@ -115,6 +115,7 @@ markdown,
 matroska,
 midi,
 moc3,
+mosaic,
 mp3,
 mp3_frame,
 mp3_frame_vbri,

format/all/all.go

@@ -39,6 +39,7 @@ import (
 	_ "github.com/wader/fq/format/matroska"
 	_ "github.com/wader/fq/format/midi"
 	_ "github.com/wader/fq/format/moc3"
+	_ "github.com/wader/fq/format/mosaic"
 	_ "github.com/wader/fq/format/mp3"
 	_ "github.com/wader/fq/format/mpeg"
 	_ "github.com/wader/fq/format/msgpack"

format/matroska/ebml/ebml.go → format/ebml/ebml.go

@@ -1,6 +1,11 @@
 package ebml
 
-import "time"
+import (
+	"time"
+
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/scalar"
+)
 
 // 2001-01-01T00:00:00.000000000 UTC
 var EpochDate = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)
@@ -13,6 +18,7 @@ type Element interface {
 	GetParentID() ID
 	GetName() string
 	GetDefinition() string
+	IsSingleton() bool
 }
 
 type Enum struct {
@@ -25,13 +31,15 @@ type ElementType struct {
 	ParentID   ID
 	Name       string
 	Definition string
+	Singleton  bool
 }
 
 func (e *ElementType) GetType() string       { return "" }
 func (e *ElementType) GetID() ID             { return e.ID }
 func (e *ElementType) GetParentID() ID       { return e.ParentID }
 func (e *ElementType) GetName() string       { return e.Name }
 func (e *ElementType) GetDefinition() string { return e.Definition }
+func (e *ElementType) IsSingleton() bool     { return e.Singleton }
 
 type ElementScalarType[T comparable] struct {
 	ElementType
@@ -80,6 +88,29 @@ type Master struct {
 func (e *Master) GetType() string           { return "master" }
 func (e *Master) GetMaster() map[ID]Element { return e.Master }
 
+// Simplifies the resolution of id to an Element by looking into standard EBML elements
+// if id is not found in schema-specific types.
+func (e *Master) LookupByID(id ID) (Element, bool) {
+	if match, ok := e.Master[id]; ok {
+		return match, true
+	}
+	if match, ok := Global.Master[id]; ok {
+		return match, true
+	}
+	return nil, false
+}
+
+// Peek the next element and sets `out`
+func (e *Master) PeekNextElement(d *decode.D, out *Element) uint64 {
+	n := PeekRawVint(d)
+	var ok bool
+	*out, ok = e.LookupByID(ID(n))
+	if !ok {
+		*out = &Unknown{}
+	}
+	return n
+}
+
 const (
 	RootID = 0
 
@@ -89,12 +120,13 @@ const (
 
 var Global = &Master{
 	ElementType: ElementType{
-		ID:       -1,
-		ParentID: -1,
-		Name:     "",
+		ID:        -1,
+		ParentID:  -1,
+		Name:      "",
+		Singleton: true,
 	},
 	Master: map[ID]Element{
-		CRC32ID: &Binary{ElementType: ElementType{Name: "crc32"}},
+		CRC32ID: &Binary{ElementType: ElementType{Name: "crc32", Singleton: true}},
 		VoidID:  &Binary{ElementType: ElementType{Name: "void"}},
 	},
 }
@@ -112,18 +144,19 @@ const (
 
 var Header = &Master{
 	ElementType: ElementType{
-		ID:       HeaderID,
-		ParentID: RootID,
-		Name:     "ebml",
+		ID:        HeaderID,
+		ParentID:  RootID,
+		Name:      "ebml",
+		Singleton: true,
 	},
 	Master: map[ID]Element{
-		EBMLVersionID:        &Uinteger{ElementType: ElementType{Name: "ebml_version", Definition: "EBML Version"}},
-		EBMLReadVersionID:    &Uinteger{ElementType: ElementType{Name: "ebml_read_version", Definition: "Minimum EBML reader version"}},
-		EBMLMaxIDLengthID:    &Uinteger{ElementType: ElementType{Name: "ebml_max_id_length", Definition: "Maximum id length"}},
-		EBMLMaxSizeLengthID:  &Uinteger{ElementType: ElementType{Name: "ebml_max_size_length", Definition: "Maximum body length"}},
-		DocTypeID:            &String{ElementType: ElementType{Name: "doc_type", Definition: "Document content type"}},
-		DocTypeVersionID:     &Uinteger{ElementType: ElementType{Name: "doc_type_version", Definition: "Document type version"}},
-		DocTypeReadVersionID: &Uinteger{ElementType: ElementType{Name: "doc_type_read_version", Definition: "Minimum document reader version"}},
+		EBMLVersionID:        &Uinteger{ElementType: ElementType{Name: "ebml_version", Definition: "EBML Version", Singleton: true}},
+		EBMLReadVersionID:    &Uinteger{ElementType: ElementType{Name: "ebml_read_version", Definition: "Minimum EBML reader version", Singleton: true}},
+		EBMLMaxIDLengthID:    &Uinteger{ElementType: ElementType{Name: "ebml_max_id_length", Definition: "Maximum id length", Singleton: true}},
+		EBMLMaxSizeLengthID:  &Uinteger{ElementType: ElementType{Name: "ebml_max_size_length", Definition: "Maximum body length", Singleton: true}},
+		DocTypeID:            &String{ElementType: ElementType{Name: "doc_type", Definition: "Document content type", Singleton: true}},
+		DocTypeVersionID:     &Uinteger{ElementType: ElementType{Name: "doc_type_version", Definition: "Document type version", Singleton: true}},
+		DocTypeReadVersionID: &Uinteger{ElementType: ElementType{Name: "doc_type_read_version", Definition: "Minimum document reader version", Singleton: true}},
 	},
 }
 
@@ -142,3 +175,43 @@ func FindParentID(idToElement map[ID]Element, startID ID, id ID) (Element, bool)
 	}
 	return nil, false
 }
+
+// TODO: smarter?
+func DecodeRawVintWidth(d *decode.D) (uint64, int) {
+	n := d.U8()
+	w := 1
+	for i := 0; i <= 7 && (n&(1<<(7-i))) == 0; i++ {
+		w++
+	}
+	for i := 1; i < w; i++ {
+		n = n<<8 | d.U8()
+	}
+	return n, w
+}
+
+func DecodeRawVint(d *decode.D) uint64 {
+	n, _ := DecodeRawVintWidth(d)
+	return n
+}
+
+func PeekRawVint(d *decode.D) uint64 {
+	n, w := DecodeRawVintWidth(d)
+	d.SeekRel(int64(-w) * 8)
+	return n
+}
+
+func DecodeVint(d *decode.D) uint64 {
+	n, w := DecodeRawVintWidth(d)
+	m := (uint64(1<<((w-1)*8+(8-w))) - 1)
+	return n & m
+}
+
+// returns a scalar.UintMapper for EBML tags' display
+func ElementIDMapper(e Element) scalar.UintMapper {
+	return scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+		s.DisplayFormat = scalar.NumberHex
+		s.Sym = e.GetName()
+		s.Description = e.GetDefinition()
+		return s, nil
+	})
+}

format/ebml/ebml.md

@@ -0,0 +1,19 @@
+## EBML helpers
+
+This module does not implements a decoder, but helper functions to decode EBML-based
+formats (in particular, Matroska).
+
+### Elements generator
+
+`format/ebml/gen` is a program that can generate FQ elements from an EBML schema written
+as XML:
+
+    go run format/ebml/gen/main.go \
+        format/matroska/ebml_matroska/ebml_matroska.xml \
+        ebml_matroska \
+        github.com/wader/fq/format/ebml Segment \
+        | gofmt -s > format/matroska/ebml_matroska/ebml_matroska_gen.go
+
+### References
+
+- https://www.rfc-editor.org/info/rfc8794

format/matroska/ebml/gen/main.go → format/ebml/gen/main.go

@@ -81,6 +81,7 @@ func findDefintion(docs []Documentation) (string, bool) {
 			s = whitespaceRE.ReplaceAllLiteralString(s, " ")
 			s = quotesRE.ReplaceAllLiteralString(s, "")
 			s = strings.TrimRight(s, " .")
+			s = strings.TrimLeft(s, " ")
 
 			if i := strings.IndexAny(s, ".,;"); i != -1 {
 				s = s[0:i]
@@ -134,6 +135,7 @@ func main() {
 	fmt.Printf("    ID: RootID,\n")
 	fmt.Printf("    ParentID: -1,\n")
 	fmt.Printf("    Name: \"\",\n")
+	fmt.Printf("    Singleton: true,\n")
 	fmt.Printf("  },\n")
 	fmt.Printf("  Master: map[ebml.ID]ebml.Element{\n")
 	fmt.Printf("    ebml.HeaderID: ebml.Header,\n")
@@ -189,6 +191,7 @@ func main() {
 			fmt.Printf("    ParentID: RootID,\n")
 		}
 		fmt.Printf("    Name: %q,\n", camelToSnake(e.Name))
+		fmt.Printf("    Singleton: %t,\n", e.MaxOccurs == "1")
 		if def, defOk := findDefintion(e.Documentations); defOk {
 			fmt.Printf("    Definition: %q,\n", newLineRE.ReplaceAllString(def, " "))
 		}
@@ -227,6 +230,7 @@ func main() {
 			fmt.Printf("    ID: %sID,\n", c.Name)
 			fmt.Printf("    ParentID: %sID,\n", e.Name)
 			fmt.Printf("    Name: %q,\n", camelToSnake(c.Name))
+			fmt.Printf("    Singleton: %t,\n", c.MaxOccurs == "1")
 			def, defOk := findDefintion(c.Documentations)
 			if defOk {
 				fmt.Printf("  Definition: %q,\n", newLineRE.ReplaceAllString(def, " "))

format/format.go

@@ -135,6 +135,7 @@ var (
 	Matroska            = &decode.Group{Name: "matroska"}
 	MIDI                = &decode.Group{Name: "midi"}
 	MOC3                = &decode.Group{Name: "moc3"}
+	MOSAIC              = &decode.Group{Name: "mosaic"}
 	MP3                 = &decode.Group{Name: "mp3"}
 	MP3_Frame           = &decode.Group{Name: "mp3_frame"}
 	MP3_Frame_VBRI      = &decode.Group{Name: "mp3_frame_vbri"}

format/matroska/ebml_matroska/ebml_matroska.go

@@ -1,4 +1,4 @@
 package ebml_matroska
 
 // https://raw.githubusercontent.com/ietf-wg-cellar/matroska-specification/master/ebml_matroska.xml
-//go:generate sh -c "go run ../ebml/gen/main.go ebml_matroska.xml ebml_matroska github.com/wader/fq/format/matroska/ebml Segment | gofmt -s > ebml_matroska_gen.go"
+//go:generate sh -c "go run ../../ebml/gen/main.go ebml_matroska.xml ebml_matroska github.com/wader/fq/format/ebml Segment | gofmt -s > ebml_matroska_gen.go"