From e95d5ba23dcd5628fe9969a9a92d88e0a98d67bd Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Wed, 2 Sep 2026 23:12:40 +0200 Subject: [PATCH] Project the temporal types out of the MEOS-API catalog tools/codegen reads meos-idl.json#/temporalTypes, the registry MEOS-API states per Temporal, rather than parsing meos_catalog.c and type_out.c itself. The generated table holds the same 23 rows the C parse yields at the same MobilityDB commit, so this decides where the facts are read from and not which facts they are. Reading the C works, and it is the only way while the catalog states neither the MF-JSON type token nor every temporal type a base carries. It is the wrong place to read from once the catalog states them: it leaves this tier a parse of someone else's C to keep working, and puts it on a different footing from PyMEOS, JMEOS and the rest, which all project the catalog. What goes with the parse is the machinery it needs: the comment stripper, the string-aware brace walk, the predicate reader and the MF-JSON case-group reader. The generator is a JSON read and a render. CI derives the catalog with MobilityDB/MEOS-API/.github/actions/provision-meos@master, the action every catalog-consuming binding uses, and regenerates against the path it reports. The catalog is a derived artifact of one MobilityDB commit, so it is not committed; catalog_gen.go is, and a difference fails the job. --- .github/workflows/go.yml | 29 ++-- GENERATION.md | 42 ++--- catalog_gen.go | 13 +- tools/codegen/main.go | 352 +++++++-------------------------------- 4 files changed, 102 insertions(+), 334 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a33e645..19a9787 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,27 +43,30 @@ jobs: - name: Vet and build, spark backend run: go vet -tags spark ./... && go build -tags spark ./... - - name: Check out MobilityDB master - uses: actions/checkout@v4 + # The MEOS-API catalog, derived here from MobilityDB master by the action + # every binding in the ecosystem uses. It is never committed: it is a + # derived artifact of one MobilityDB commit, and a tracked copy is drift + # waiting to happen. + - name: Derive the MEOS-API catalog + id: provision + uses: MobilityDB/MEOS-API/.github/actions/provision-meos@master with: - repository: MobilityDB/MobilityDB - ref: master - path: mobilitydb + mobilitydb-ref: master - # catalog_gen.go is MEOS's own table of temporal types, so it is read from - # the MEOS sources rather than kept by hand, and this is what makes the - # copy in the tree the current one. A type MEOS gains arrives here as a - # difference to regenerate, which is the whole reason the table is + # catalog_gen.go is the catalog's temporal-type registry projected into + # Go, so it is generated rather than kept by hand, and this is what makes + # the copy in the tree the current one. A type MEOS gains arrives here as + # a difference to regenerate, which is the whole reason the table is # generated: a hand-kept copy goes stale in silence instead. - - name: The generated catalog is what MEOS declares + - name: The generated table is what the catalog declares run: | - go run ./tools/codegen -mobilitydb mobilitydb -o catalog_gen.go + go run ./tools/codegen -catalog "${{ steps.provision.outputs.catalog-path }}" -o catalog_gen.go if ! git diff --quiet -- catalog_gen.go; then - echo "::error::catalog_gen.go is not what MEOS declares. Regenerate it: go run ./tools/codegen -mobilitydb -o catalog_gen.go" + echo "::error::catalog_gen.go is not what the catalog declares. Regenerate it: go run ./tools/codegen -catalog -o catalog_gen.go" git diff -- catalog_gen.go exit 1 fi - echo "the committed catalog is what MobilityDB master declares" + echo "the committed table is what the catalog of MobilityDB master declares" # EVERY TEST THIS REPOSITORY CARRIES RUNS HERE, AND NONE OF THEM SKIPS. # A test skips when a precondition is missing from the job that collects it, diff --git a/GENERATION.md b/GENERATION.md index 9aa659e..f44db44 100644 --- a/GENERATION.md +++ b/GENERATION.md @@ -1,9 +1,9 @@ # Generation MobilityAPI is a tier over MEOS, so what it knows about MEOS is generated from -MEOS rather than kept by hand. The generator lives in `tools/codegen`, its -output is committed, and CI regenerates it against MobilityDB master and -refuses a difference. +the MEOS-API catalog rather than kept by hand. The generator lives in +`tools/codegen`, its output is committed, and CI derives the catalog from +MobilityDB master, regenerates against it, and refuses a difference. ## What is generated @@ -11,16 +11,19 @@ refuses a difference. its base type, its bounding box, the token `asMFJSON` writes for it, whether it is spatial, whether it is a number, and whether MEOS interpolates it linearly. -Run it against any MobilityDB checkout: +Run it against a catalog: ```bash -go run ./tools/codegen -mobilitydb -o catalog_gen.go +go run ./tools/codegen -catalog -o catalog_gen.go ``` ## Where the facts come from -The generator reads the MEOS sources that define them, in the checkout it is -pointed at: +`meos-idl.json` is the machine-readable description of the MEOS C library that +[MEOS-API](https://github.com/MobilityDB/MEOS-API) derives from the MEOS +headers and sources, and that every binding in the ecosystem generates from. +This tier reads one registry of it, `temporalTypes`, which MEOS-API in turn +reads from: | source | what is read | | --- | --- | @@ -32,10 +35,11 @@ and the empty token in the table says so. That is how `tdouble2`, `tdouble3` and `tdouble4`, which exist for temporal aggregation, stay off every surface without a list here naming them. -The sources are read by anchoring on a definition and counting braces, never by -matching a pattern across a span, and string literals are stepped over so the -`{` inside `{\"type\":\"MovingFloat\",` is read as text rather than as -structure. +The catalog is a derived artifact of one MobilityDB commit, so it is never +committed. CI derives it with +`MobilityDB/MEOS-API/.github/actions/provision-meos@master`, the action every +catalog-consuming binding uses, and passes the path it reports to the +generator. ## What the tier adds, and what holds it complete @@ -58,13 +62,11 @@ store's value columns are one per scalar type. A temporal type MEOS adds over one of the standard's bases therefore arrives as a failing test rather than as a request the tier rejects at run time. -## The catalog this could read instead +## Why the catalog and not the C -`MobilityDB/MEOS-API` publishes `meos-idl.json`, the machine-readable catalog -every language binding generates from, and reading it here rather than the C -sources would put this tier on the same footing as the bindings. Two of the -facts above are not in that catalog today: the MF-JSON type token has no field, -and `typeRelations.byBase` holds one temporal type per base, so it carries -`trgeometry` for the base `pose` and drops `tpose`, which shares that base. -Both are single-source facts in `meos_catalog.c` and `type_out.c`, which is -what the generator reads until the catalog states them. +Reading MEOS's C directly would work, and it is what this generator did before +the catalog stated the MF-JSON type token and named every temporal type a base +carries. It is the wrong place to read from: it gives this tier a parse of +someone else's C to keep working, and it puts the tier on a different footing +from PyMEOS, JMEOS and the rest, which all project the catalog. One source of +truth, read one way, is the whole point of the catalog existing. diff --git a/catalog_gen.go b/catalog_gen.go index b873a01..0b39f75 100644 --- a/catalog_gen.go +++ b/catalog_gen.go @@ -1,13 +1,14 @@ -// Code generated by tools/codegen from the MEOS sources. DO NOT EDIT. +// Code generated by tools/codegen from the MEOS-API catalog. DO NOT EDIT. // // Regenerate with: // -// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// go run ./tools/codegen -catalog -o catalog_gen.go // -// The sources read are meos/src/temporal/meos_catalog.c (the type names, each -// temporal type's base and bounding box, and the predicates temporal_type, -// tnumber_type, tspatial_type and temptype_supports_linear) and -// meos/src/temporal/type_out.c (temptype_as_mfjson_sb). +// The source is meos-idl.json#/temporalTypes, which MobilityDB/MEOS-API derives +// from meos/src/temporal/meos_catalog.c (the type names, each temporal type's +// base and bounding box, and the predicates temporal_type, tnumber_type, +// tspatial_type and temptype_supports_linear) and meos/src/temporal/type_out.c +// (temptype_as_mfjson_sb, the MF-JSON type token). package main diff --git a/tools/codegen/main.go b/tools/codegen/main.go index da66f59..d3b4dac 100644 --- a/tools/codegen/main.go +++ b/tools/codegen/main.go @@ -1,70 +1,78 @@ // Command codegen writes catalog_gen.go, the tier's table of MEOS temporal -// types, by reading the MEOS sources that define them. +// types, from the MEOS-API catalog. // -// The tier needs five facts about each temporal type: its name, its base type, -// the token asMFJSON writes for it, whether it is spatial, and whether MEOS -// interpolates it linearly. MEOS states all five in C, and every one of them -// moves when a family is added, so a copy of them here is a copy that goes -// stale in silence. The generator reads the definitions themselves: -// -// meos/src/temporal/meos_catalog.c the type names, the base type and bounding -// box of each temporal type, and the -// predicates temporal_type, tnumber_type, -// tspatial_type and temptype_supports_linear -// meos/src/temporal/type_out.c temptype_as_mfjson_sb, the switch that -// writes each type's MF-JSON type token -// -// A type MEOS carries that this switch does not name has no MF-JSON form, and -// the empty token says so rather than a list kept here. +// meos-idl.json is the machine-readable description of the MEOS C library that +// MobilityDB/MEOS-API derives from the MEOS headers and sources, and that every +// binding in the ecosystem generates from. Its `temporalTypes` registry states, +// per Temporal, the facts this tier needs: the base type, the bounding box, +// the MF-JSON type token asMFJSON writes, and whether the type is spatial, a +// number, and linearly interpolated. Reading them here is what makes this tier +// a projection of the same catalog as PyMEOS, JMEOS and the rest, rather than a +// reader of MEOS's C with a parse of its own to keep working. // // Usage: // -// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// go run ./tools/codegen -catalog -o catalog_gen.go // -// The output is committed, and CI regenerates it against the MobilityDB -// checkout it already has and refuses a difference, so a type MEOS gains -// reaches this tier as a failing check rather than as silence. +// The catalog is derived, never committed: CI generates it from a MobilityDB +// commit with MobilityDB/MEOS-API's provision-meos action, regenerates this +// file against it and refuses a difference, so a type MEOS gains reaches this +// tier as a failing check rather than as silence. package main import ( "bytes" + "encoding/json" "flag" "fmt" "go/format" "os" - "path/filepath" - "regexp" "sort" - "strings" ) +// catalog is the part of meos-idl.json this generator reads. +type catalog struct { + TemporalTypes map[string]temporalType `json:"temporalTypes"` +} + +// temporalType is one entry of the catalog's temporalTypes registry. MFJSON is +// absent for a type asMFJSON does not write, so an empty token is what says a +// type has no MF-JSON form. +type temporalType struct { + Base string `json:"base"` + Box string `json:"bbox"` + MFJSON string `json:"mfjson"` + Spatial bool `json:"spatial"` + Number bool `json:"number"` + Linear bool `json:"linear"` +} + func main() { - mdb := flag.String("mobilitydb", "mobilitydb", "a MobilityDB checkout") + path := flag.String("catalog", "meos-idl.json", "the MEOS-API catalog") out := flag.String("o", "catalog_gen.go", "file to write") flag.Parse() - catalog, err := readSource(filepath.Join(*mdb, "meos", "src", "temporal", "meos_catalog.c")) + raw, err := os.ReadFile(*path) if err != nil { fail(err) } - typeOut, err := readSource(filepath.Join(*mdb, "meos", "src", "temporal", "type_out.c")) - if err != nil { - fail(err) + var c catalog + if err := json.Unmarshal(raw, &c); err != nil { + fail(fmt.Errorf("%s: %w", *path, err)) } - - types, err := build(catalog, typeOut) - if err != nil { - fail(err) + if len(c.TemporalTypes) == 0 { + fail(fmt.Errorf("%s carries no temporalTypes registry: it is older than the "+ + "MEOS-API that states one, or it is not a MEOS-API catalog", *path)) } - src, err := render(types) + src, err := render(c.TemporalTypes) if err != nil { fail(err) } if err := os.WriteFile(*out, src, 0o644); err != nil { fail(err) } - fmt.Fprintf(os.Stderr, "codegen: %d temporal types written to %s\n", len(types), *out) + fmt.Fprintf(os.Stderr, "codegen: %d temporal types written to %s\n", len(c.TemporalTypes), *out) } func fail(err error) { @@ -72,272 +80,25 @@ func fail(err error) { os.Exit(1) } -// temporalType is one row of the table. -type temporalType struct { - Enum string // the MeosType enumerator, e.g. T_TGEOMPOINT - Name string // the MEOS type name, e.g. tgeompoint - Base string // its base type's name, e.g. geometry - Box string // its bounding box type's name, e.g. stbox - MFJSON string // the token asMFJSON writes, e.g. MovingPoint - Spatial bool - Number bool - Linear bool // MEOS interpolates it linearly between samples -} - -// readSource returns the file with its C comments blanked, so a type -// enumerator named in prose is never read as a member of a list. -func readSource(path string) (string, error) { - b, err := os.ReadFile(path) - if err != nil { - return "", err +func render(types map[string]temporalType) ([]byte, error) { + names := make([]string, 0, len(types)) + for name := range types { + names = append(names, name) } - return stripComments(string(b)), nil -} - -// stripComments replaces comment bodies with spaces, keeping every newline so -// the line structure survives. A comment opener inside a string literal is -// part of the string, so literals are stepped over rather than scanned. -func stripComments(s string) string { - var b strings.Builder - b.Grow(len(s)) - for i := 0; i < len(s); { - switch { - case strings.HasPrefix(s[i:], "/*"): - end := strings.Index(s[i+2:], "*/") - if end < 0 { - b.WriteString(blank(s[i:])) - i = len(s) - continue - } - seg := s[i : i+2+end+2] - b.WriteString(blank(seg)) - i += len(seg) - case strings.HasPrefix(s[i:], "//"): - end := strings.IndexByte(s[i:], '\n') - if end < 0 { - end = len(s) - i - } - b.WriteString(blank(s[i : i+end])) - i += end - case s[i] == '"' || s[i] == '\'': - n := literal(s, i) - b.WriteString(s[i:n]) - i = n - default: - b.WriteByte(s[i]) - i++ - } - } - return b.String() -} - -// literal returns the index just past the string or character literal starting -// at i, honouring backslash escapes. -func literal(s string, i int) int { - quote := s[i] - for j := i + 1; j < len(s); j++ { - switch s[j] { - case '\\': - j++ - case quote: - return j + 1 - } - } - return len(s) -} - -// blank returns s with every byte other than a newline as a space. -func blank(s string) string { - out := []byte(s) - for i, c := range out { - if c != '\n' { - out[i] = ' ' - } - } - return string(out) -} - -var ( - reName = regexp.MustCompile(`\[(T_\w+)\]\s*=\s*"([^"]+)"`) - reRelRow = regexp.MustCompile(`\[(T_\w+)\]\s*=\s*\{([^}]*)\}`) - reBase = regexp.MustCompile(`\.temptype_basetype\s*=\s*(T_\w+)`) - reBox = regexp.MustCompile(`\.type_bboxtype\s*=\s*(T_\w+)`) - reEnum = regexp.MustCompile(`\bT_[A-Z0-9_]+\b`) - reCase = regexp.MustCompile(`\bcase\s+(T_\w+)\s*:`) - reMFToken = regexp.MustCompile(`\{\\"type\\":\\"(\w+)\\"`) -) - -func build(catalog, typeOut string) ([]temporalType, error) { - names := map[string]string{} - for _, m := range reName.FindAllStringSubmatch(catalog, -1) { - names[m[1]] = m[2] - } - if len(names) == 0 { - return nil, fmt.Errorf(`meos_catalog.c names no type: the [T_X] = "x" table is not where it is read from`) - } - - base, box := map[string]string{}, map[string]string{} - for _, m := range reRelRow.FindAllStringSubmatch(catalog, -1) { - if b := reBase.FindStringSubmatch(m[2]); b != nil { - base[m[1]] = b[1] - } - if b := reBox.FindStringSubmatch(m[2]); b != nil { - box[m[1]] = b[1] - } - } - - temporal, err := predicate(catalog, "temporal_type") - if err != nil { - return nil, err - } - number, err := predicate(catalog, "tnumber_type") - if err != nil { - return nil, err - } - spatial, err := predicate(catalog, "tspatial_type") - if err != nil { - return nil, err - } - linear, err := predicate(catalog, "temptype_supports_linear") - if err != nil { - return nil, err - } - - mfjson, err := mfjsonTokens(typeOut) - if err != nil { - return nil, err - } - - var out []temporalType - for enum := range temporal { - name, ok := names[enum] - if !ok { - return nil, fmt.Errorf("%s is a temporal type the name table does not name", enum) - } - b, ok := base[enum] - if !ok { - return nil, fmt.Errorf("%s is a temporal type with no temptype_basetype", enum) - } - baseName, ok := names[b] - if !ok { - return nil, fmt.Errorf("%s has base type %s, which the name table does not name", enum, b) - } - boxName := "" - if x, ok := box[enum]; ok { - boxName = names[x] - } - out = append(out, temporalType{ - Enum: enum, - Name: name, - Base: baseName, - Box: boxName, - MFJSON: mfjson[enum], - Spatial: spatial[enum], - Number: number[enum], - Linear: linear[enum], - }) - } - sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) - return out, nil -} - -// predicate returns the type enumerators named in the body of a boolean -// predicate such as tspatial_type. The body is taken by counting braces from -// the function's own opening brace, never by matching a pattern across it: a -// nested block ends a pattern early and silently shortens the list. -func predicate(src, fn string) (map[string]bool, error) { - body, err := functionBody(src, fn) - if err != nil { - return nil, err - } - set := map[string]bool{} - for _, m := range reEnum.FindAllString(body, -1) { - set[m] = true - } - if len(set) == 0 { - return nil, fmt.Errorf("%s names no type enumerator", fn) - } - return set, nil -} - -// functionBody returns the braced body of the named function definition, which -// is the occurrence of the name at the start of a line followed by its -// parameter list: MEOS writes a definition's return type on the line above, so -// a definition is the only occurrence in that column. -func functionBody(src, fn string) (string, error) { - re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(fn) + `\s*\(`) - loc := re.FindStringIndex(src) - if loc == nil { - return "", fmt.Errorf("no definition of %s", fn) - } - open := strings.IndexByte(src[loc[1]:], '{') - if open < 0 { - return "", fmt.Errorf("%s has no body", fn) - } - start := loc[1] + open - depth := 0 - for i := start; i < len(src); { - switch src[i] { - case '"', '\'': - // A brace inside a string literal is text, not structure: the - // MF-JSON switch writes `{\"type\":\"MovingFloat\",` for every - // type, and counting those opens a depth that never closes. - i = literal(src, i) - continue - case '{': - depth++ - case '}': - depth-- - if depth == 0 { - return src[start+1 : i], nil - } - } - i++ - } - return "", fmt.Errorf("%s has an unterminated body", fn) -} - -// mfjsonTokens reads temptype_as_mfjson_sb, whose case labels come in groups: -// several types share one token, since a geometry point and a geography point -// are both a MovingPoint. Labels accumulate until a token is written and are -// assigned together. -func mfjsonTokens(typeOut string) (map[string]string, error) { - body, err := functionBody(typeOut, "temptype_as_mfjson_sb") - if err != nil { - return nil, err - } - out := map[string]string{} - var pending []string - for _, line := range strings.Split(body, "\n") { - if m := reCase.FindStringSubmatch(line); m != nil { - pending = append(pending, m[1]) - continue - } - if m := reMFToken.FindStringSubmatch(line); m != nil { - for _, enum := range pending { - out[enum] = m[1] - } - pending = nil - } - } - if len(out) == 0 { - return nil, fmt.Errorf("temptype_as_mfjson_sb names no type token") - } - return out, nil -} + sort.Strings(names) -func render(types []temporalType) ([]byte, error) { var b bytes.Buffer - b.WriteString(`// Code generated by tools/codegen from the MEOS sources. DO NOT EDIT. + b.WriteString(`// Code generated by tools/codegen from the MEOS-API catalog. DO NOT EDIT. // // Regenerate with: // -// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// go run ./tools/codegen -catalog -o catalog_gen.go // -// The sources read are meos/src/temporal/meos_catalog.c (the type names, each -// temporal type's base and bounding box, and the predicates temporal_type, -// tnumber_type, tspatial_type and temptype_supports_linear) and -// meos/src/temporal/type_out.c (temptype_as_mfjson_sb). +// The source is meos-idl.json#/temporalTypes, which MobilityDB/MEOS-API derives +// from meos/src/temporal/meos_catalog.c (the type names, each temporal type's +// base and bounding box, and the predicates temporal_type, tnumber_type, +// tspatial_type and temptype_supports_linear) and meos/src/temporal/type_out.c +// (temptype_as_mfjson_sb, the MF-JSON type token). package main @@ -357,9 +118,10 @@ type TemporalType struct { // temporalTypes is every temporal type MEOS declares, ordered by name. var temporalTypes = []TemporalType{ `) - for _, t := range types { + for _, name := range names { + t := types[name] fmt.Fprintf(&b, "\t{Name: %q, Base: %q, Box: %q, MFJSON: %q, Spatial: %t, Number: %t, Linear: %t},\n", - t.Name, t.Base, t.Box, t.MFJSON, t.Spatial, t.Number, t.Linear) + name, t.Base, t.Box, t.MFJSON, t.Spatial, t.Number, t.Linear) } b.WriteString(`}