Separate EBML from Matroska, add another EBML-based format: MOSAIC

.golangci.yml

@@ -81,7 +81,7 @@ linters:
       # ignore warnings in code from crypto/tls and zmap/zcrypto
       - format/tls/tlsdecrypt
       - format/fuzz_test.go
-      - format/matroska/ebml/gen/main.go
+      - format/ebml/gen/main.go
 formatters:
   enable:
     - goimports

README.md

@@ -115,6 +115,7 @@ markdown,
 matroska,
 midi,
 moc3,
+mosaic,
 mp3,
 mp3_frame,
 mp3_frame_vbri,

format/all/all.go

@@ -39,6 +39,7 @@ import (
 	_ "github.com/wader/fq/format/matroska"
 	_ "github.com/wader/fq/format/midi"
 	_ "github.com/wader/fq/format/moc3"
+	_ "github.com/wader/fq/format/mosaic"
 	_ "github.com/wader/fq/format/mp3"
 	_ "github.com/wader/fq/format/mpeg"
 	_ "github.com/wader/fq/format/msgpack"

format/matroska/ebml/ebml.go → format/ebml/ebml.go

@@ -1,6 +1,11 @@
 package ebml
 
-import "time"
+import (
+	"time"
+
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/scalar"
+)
 
 // 2001-01-01T00:00:00.000000000 UTC
 var EpochDate = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)
@@ -13,6 +18,7 @@ type Element interface {
 	GetParentID() ID
 	GetName() string
 	GetDefinition() string
+	IsSingleton() bool
 }
 
 type Enum struct {
@@ -25,13 +31,15 @@ type ElementType struct {
 	ParentID   ID
 	Name       string
 	Definition string
+	Singleton  bool
 }
 
 func (e *ElementType) GetType() string       { return "" }
 func (e *ElementType) GetID() ID             { return e.ID }
 func (e *ElementType) GetParentID() ID       { return e.ParentID }
 func (e *ElementType) GetName() string       { return e.Name }
 func (e *ElementType) GetDefinition() string { return e.Definition }
+func (e *ElementType) IsSingleton() bool     { return e.Singleton }
 
 type ElementScalarType[T comparable] struct {
 	ElementType
@@ -80,6 +88,29 @@ type Master struct {
 func (e *Master) GetType() string           { return "master" }
 func (e *Master) GetMaster() map[ID]Element { return e.Master }
 
+// Simplifies the resolution of id to an Element by looking into standard EBML elements
+// if id is not found in schema-specific types.
+func (e *Master) LookupByID(id ID) (Element, bool) {
+	if match, ok := e.Master[id]; ok {
+		return match, true
+	}
+	if match, ok := Global.Master[id]; ok {
+		return match, true
+	}
+	return nil, false
+}
+
+// Peek the next element and sets `out`
+func (e *Master) PeekNextElement(d *decode.D, out *Element) uint64 {
+	n := PeekRawVint(d)
+	var ok bool
+	*out, ok = e.LookupByID(ID(n))
+	if !ok {
+		*out = &Unknown{}
+	}
+	return n
+}
+
 const (
 	RootID = 0
 
@@ -89,12 +120,13 @@ const (
 
 var Global = &Master{
 	ElementType: ElementType{
-		ID:       -1,
-		ParentID: -1,
-		Name:     "",
+		ID:        -1,
+		ParentID:  -1,
+		Name:      "",
+		Singleton: true,
 	},
 	Master: map[ID]Element{
-		CRC32ID: &Binary{ElementType: ElementType{Name: "crc32"}},
+		CRC32ID: &Binary{ElementType: ElementType{Name: "crc32", Singleton: true}},
 		VoidID:  &Binary{ElementType: ElementType{Name: "void"}},
 	},
 }
@@ -112,18 +144,19 @@ const (
 
 var Header = &Master{
 	ElementType: ElementType{
-		ID:       HeaderID,
-		ParentID: RootID,
-		Name:     "ebml",
+		ID:        HeaderID,
+		ParentID:  RootID,
+		Name:      "ebml",
+		Singleton: true,
 	},
 	Master: map[ID]Element{
-		EBMLVersionID:        &Uinteger{ElementType: ElementType{Name: "ebml_version", Definition: "EBML Version"}},
-		EBMLReadVersionID:    &Uinteger{ElementType: ElementType{Name: "ebml_read_version", Definition: "Minimum EBML reader version"}},
-		EBMLMaxIDLengthID:    &Uinteger{ElementType: ElementType{Name: "ebml_max_id_length", Definition: "Maximum id length"}},
-		EBMLMaxSizeLengthID:  &Uinteger{ElementType: ElementType{Name: "ebml_max_size_length", Definition: "Maximum body length"}},
-		DocTypeID:            &String{ElementType: ElementType{Name: "doc_type", Definition: "Document content type"}},
-		DocTypeVersionID:     &Uinteger{ElementType: ElementType{Name: "doc_type_version", Definition: "Document type version"}},
-		DocTypeReadVersionID: &Uinteger{ElementType: ElementType{Name: "doc_type_read_version", Definition: "Minimum document reader version"}},
+		EBMLVersionID:        &Uinteger{ElementType: ElementType{Name: "ebml_version", Definition: "EBML Version", Singleton: true}},
+		EBMLReadVersionID:    &Uinteger{ElementType: ElementType{Name: "ebml_read_version", Definition: "Minimum EBML reader version", Singleton: true}},
+		EBMLMaxIDLengthID:    &Uinteger{ElementType: ElementType{Name: "ebml_max_id_length", Definition: "Maximum id length", Singleton: true}},
+		EBMLMaxSizeLengthID:  &Uinteger{ElementType: ElementType{Name: "ebml_max_size_length", Definition: "Maximum body length", Singleton: true}},
+		DocTypeID:            &String{ElementType: ElementType{Name: "doc_type", Definition: "Document content type", Singleton: true}},
+		DocTypeVersionID:     &Uinteger{ElementType: ElementType{Name: "doc_type_version", Definition: "Document type version", Singleton: true}},
+		DocTypeReadVersionID: &Uinteger{ElementType: ElementType{Name: "doc_type_read_version", Definition: "Minimum document reader version", Singleton: true}},
 	},
 }
 
@@ -142,3 +175,43 @@ func FindParentID(idToElement map[ID]Element, startID ID, id ID) (Element, bool)
 	}
 	return nil, false
 }
+
+// TODO: smarter?
+func DecodeRawVintWidth(d *decode.D) (uint64, int) {
+	n := d.U8()
+	w := 1
+	for i := 0; i <= 7 && (n&(1<<(7-i))) == 0; i++ {
+		w++
+	}
+	for i := 1; i < w; i++ {
+		n = n<<8 | d.U8()
+	}
+	return n, w
+}
+
+func DecodeRawVint(d *decode.D) uint64 {
+	n, _ := DecodeRawVintWidth(d)
+	return n
+}
+
+func PeekRawVint(d *decode.D) uint64 {
+	n, w := DecodeRawVintWidth(d)
+	d.SeekRel(int64(-w) * 8)
+	return n
+}
+
+func DecodeVint(d *decode.D) uint64 {
+	n, w := DecodeRawVintWidth(d)
+	m := (uint64(1<<((w-1)*8+(8-w))) - 1)
+	return n & m
+}
+
+// returns a scalar.UintMapper for EBML tags' display
+func ElementIDMapper(e Element) scalar.UintMapper {
+	return scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+		s.DisplayFormat = scalar.NumberHex
+		s.Sym = e.GetName()
+		s.Description = e.GetDefinition()
+		return s, nil
+	})
+}

format/ebml/ebml.md

@@ -0,0 +1,19 @@
+## EBML helpers
+
+This module does not implements a decoder, but helper functions to decode EBML-based
+formats (in particular, Matroska).
+
+### Elements generator
+
+`format/ebml/gen` is a program that can generate FQ elements from an EBML schema written
+as XML:
+
+    go run format/ebml/gen/main.go \
+        format/matroska/ebml_matroska/ebml_matroska.xml \
+        ebml_matroska \
+        github.com/wader/fq/format/ebml Segment \
+        | gofmt -s > format/matroska/ebml_matroska/ebml_matroska_gen.go
+
+### References
+
+- https://www.rfc-editor.org/info/rfc8794

format/matroska/ebml/gen/main.go → format/ebml/gen/main.go

@@ -81,6 +81,7 @@ func findDefintion(docs []Documentation) (string, bool) {
 			s = whitespaceRE.ReplaceAllLiteralString(s, " ")
 			s = quotesRE.ReplaceAllLiteralString(s, "")
 			s = strings.TrimRight(s, " .")
+			s = strings.TrimLeft(s, " ")
 
 			if i := strings.IndexAny(s, ".,;"); i != -1 {
 				s = s[0:i]
@@ -134,6 +135,7 @@ func main() {
 	fmt.Printf("    ID: RootID,\n")
 	fmt.Printf("    ParentID: -1,\n")
 	fmt.Printf("    Name: \"\",\n")
+	fmt.Printf("    Singleton: true,\n")
 	fmt.Printf("  },\n")
 	fmt.Printf("  Master: map[ebml.ID]ebml.Element{\n")
 	fmt.Printf("    ebml.HeaderID: ebml.Header,\n")
@@ -189,6 +191,7 @@ func main() {
 			fmt.Printf("    ParentID: RootID,\n")
 		}
 		fmt.Printf("    Name: %q,\n", camelToSnake(e.Name))
+		fmt.Printf("    Singleton: %t,\n", e.MaxOccurs == "1")
 		if def, defOk := findDefintion(e.Documentations); defOk {
 			fmt.Printf("    Definition: %q,\n", newLineRE.ReplaceAllString(def, " "))
 		}
@@ -227,6 +230,7 @@ func main() {
 			fmt.Printf("    ID: %sID,\n", c.Name)
 			fmt.Printf("    ParentID: %sID,\n", e.Name)
 			fmt.Printf("    Name: %q,\n", camelToSnake(c.Name))
+			fmt.Printf("    Singleton: %t,\n", c.MaxOccurs == "1")
 			def, defOk := findDefintion(c.Documentations)
 			if defOk {
 				fmt.Printf("  Definition: %q,\n", newLineRE.ReplaceAllString(def, " "))

format/format.go

@@ -135,6 +135,7 @@ var (
 	Matroska            = &decode.Group{Name: "matroska"}
 	MIDI                = &decode.Group{Name: "midi"}
 	MOC3                = &decode.Group{Name: "moc3"}
+	MOSAIC              = &decode.Group{Name: "mosaic"}
 	MP3                 = &decode.Group{Name: "mp3"}
 	MP3_Frame           = &decode.Group{Name: "mp3_frame"}
 	MP3_Frame_VBRI      = &decode.Group{Name: "mp3_frame_vbri"}

format/matroska/ebml_matroska/ebml_matroska.go

@@ -1,4 +1,4 @@
 package ebml_matroska
 
 // https://raw.githubusercontent.com/ietf-wg-cellar/matroska-specification/master/ebml_matroska.xml
-//go:generate sh -c "go run ../ebml/gen/main.go ebml_matroska.xml ebml_matroska github.com/wader/fq/format/matroska/ebml Segment | gofmt -s > ebml_matroska_gen.go"
+//go:generate sh -c "go run ../../ebml/gen/main.go ebml_matroska.xml ebml_matroska github.com/wader/fq/format/ebml Segment | gofmt -s > ebml_matroska_gen.go"

format/matroska/ebml_matroska/ebml_matroska_gen.go

@@ -1,8 +1,8 @@
-// Code below generated from ebml_matroska.xml
+// Code below generated from format/matroska/ebml_matroska/ebml_matroska.xml
 package ebml_matroska
 
 import (
-	"github.com/wader/fq/format/matroska/ebml"
+	"github.com/wader/fq/format/ebml"
 )
 
 var RootElement = &ebml.Master{

format/matroska/matroska.go

@@ -17,7 +17,7 @@ import (
 	"time"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/matroska/ebml"
+	"github.com/wader/fq/format/ebml"
 	"github.com/wader/fq/format/matroska/ebml_matroska"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/interp"
@@ -142,10 +142,10 @@ func decodeLacingFn(d *decode.D, lacingType int, fn func(d *decode.D)) {
 	case lacingTypeEBML:
 		numLaces := int(d.FieldU8("num_laces"))
 		d.FieldArray("lace_sizes", func(d *decode.D) {
-			s := int64(d.FieldUintFn("lace_size", decodeVint)) // first is unsigned, not ranged shifted
+			s := int64(d.FieldUintFn("lace_size", ebml.DecodeVint)) // first is unsigned, not ranged shifted
 			laceSizes = append(laceSizes, s)
 			for range numLaces - 1 {
-				d := int64(d.FieldUintFn("lace_size_delta", decodeRawVint))
+				d := int64(d.FieldUintFn("lace_size_delta", ebml.DecodeRawVint))
 				// range shifting
 				switch {
 				case d&0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1000_0000 == 0b0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_1000_0000:
@@ -196,36 +196,6 @@ func decodeLacingFn(d *decode.D, lacingType int, fn func(d *decode.D)) {
 	})
 }
 
-// TODO: smarter?
-func decodeRawVintWidth(d *decode.D) (uint64, int) {
-	n := d.U8()
-	w := 1
-	for i := 0; i <= 7 && (n&(1<<(7-i))) == 0; i++ {
-		w++
-	}
-	for i := 1; i < w; i++ {
-		n = n<<8 | d.U8()
-	}
-	return n, w
-}
-
-func decodeRawVint(d *decode.D) uint64 {
-	n, _ := decodeRawVintWidth(d)
-	return n
-}
-
-func peekRawVint(d *decode.D) uint64 {
-	n, w := decodeRawVintWidth(d)
-	d.SeekRel(int64(-w) * 8)
-	return n
-}
-
-func decodeVint(d *decode.D) uint64 {
-	n, w := decodeRawVintWidth(d)
-	m := (uint64(1<<((w-1)*8+(8-w))) - 1)
-	return n & m
-}
-
 func decodeXiphLaceSize(d *decode.D) uint64 {
 	var s uint64
 	for {
@@ -272,7 +242,7 @@ func decodeMaster(d *decode.D, bitsLimit int64, elm *ebml.Master, unknownSize bo
 			// TODO: What to do if peeked is unknown?
 			// TODO: Handle garbage between element
 			if unknownSize {
-				peekTagID := peekRawVint(d)
+				peekTagID := ebml.PeekRawVint(d)
 				_, validParent := ebml.FindParentID(ebml_matroska.IDToElement, elm.GetID(), ebml.ID(peekTagID))
 				if validParent {
 					break
@@ -283,7 +253,7 @@ func decodeMaster(d *decode.D, bitsLimit int64, elm *ebml.Master, unknownSize bo
 				var childElm ebml.Element
 				childElm = &ebml.Unknown{}
 
-				tagID := d.FieldUintFn("id", decodeRawVint, scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+				tagID := d.FieldUintFn("id", ebml.DecodeRawVint, scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
 					n := s.Actual
 					var ok bool
 					childElm, ok = elm.Master[ebml.ID(n)]
@@ -309,7 +279,7 @@ func decodeMaster(d *decode.D, bitsLimit int64, elm *ebml.Master, unknownSize bo
 				}
 
 				const maxStringTagSize = 100 * 1024 * 1024
-				tagSize := d.FieldUintFn("size", decodeVint, scalar.UintMapDescription{
+				tagSize := d.FieldUintFn("size", ebml.DecodeVint, scalar.UintMapDescription{
 					0xffffffffffffff: "Unknown size",
 				})
 				unknownSize := tagSize == tagSizeUnknown
@@ -383,6 +353,9 @@ func decodeMaster(d *decode.D, bitsLimit int64, elm *ebml.Master, unknownSize bo
 					if dc.currentTrack != nil && tagID == ebml_matroska.CodecIDID {
 						dc.currentTrack.codec = v
 					}
+					if tagID == ebml.DocTypeID && v != "matroska" {
+						d.Errorf("EBML doctype is not matroska")
+					}
 				case *ebml.UTF8:
 					d.FieldUTF8NullFixedLen("value", int(tagSize))
 				case *ebml.Date:
@@ -515,7 +488,7 @@ func matroskaDecode(d *decode.D) any {
 	for _, b := range dc.blocks {
 		b.d.RangeFn(b.r.Start, b.r.Len, func(d *decode.D) {
 			var lacing uint64
-			trackNumber := d.FieldUintFn("track_number", decodeVint)
+			trackNumber := d.FieldUintFn("track_number", ebml.DecodeVint)
 			d.FieldU16("timestamp")
 			if b.simple {
 				d.FieldStruct("flags", func(d *decode.D) {

format/mosaic/README.md

@@ -0,0 +1,3 @@
+Generated code is based from https://gitlab.softwareheritage.org/swh/devel/swh-mosaic/-/raw/main/schema.ebml.xml - modified such that enum-restricted values are typed "string" instead of UTF8.
+
+It is licensed under the GNU GENERAL PUBLIC LICENSE Version 3, see https://gitlab.softwareheritage.org/swh/devel/swh-mosaic/-/blob/main/LICENSE

format/mosaic/ebml_mosaic/ebml_mosaic.go

@@ -0,0 +1,4 @@
+package ebml_mosaic
+
+// https://gitlab.softwareheritage.org/swh/devel/swh-mosaic/-/raw/main/schema.ebml.xml
+//go:generate sh -c "go run ../../ebml/gen/main.go ebml_mosaic.xml ebml_mosaic github.com/wader/fq/format/ebml Mosaic | gofmt -s > ebml_mosaic_gen.go"