diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f9566db..a33e645 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,6 +43,28 @@ jobs: - name: Vet and build, spark backend run: go vet -tags spark ./... && go build -tags spark ./... + - name: Check out MobilityDB master + uses: actions/checkout@v4 + with: + repository: MobilityDB/MobilityDB + ref: master + path: mobilitydb + + # catalog_gen.go is MEOS's own table of temporal types, so it is read from + # the MEOS sources rather than kept by hand, and this is what makes the + # copy in the tree the current one. A type MEOS gains arrives here as a + # difference to regenerate, which is the whole reason the table is + # generated: a hand-kept copy goes stale in silence instead. + - name: The generated catalog is what MEOS declares + run: | + go run ./tools/codegen -mobilitydb mobilitydb -o catalog_gen.go + if ! git diff --quiet -- catalog_gen.go; then + echo "::error::catalog_gen.go is not what MEOS declares. Regenerate it: go run ./tools/codegen -mobilitydb -o catalog_gen.go" + git diff -- catalog_gen.go + exit 1 + fi + echo "the committed catalog is what MobilityDB master declares" + # EVERY TEST THIS REPOSITORY CARRIES RUNS HERE, AND NONE OF THEM SKIPS. # A test skips when a precondition is missing from the job that collects it, # so this job supplies all three: the fixture database the Annex A data tests @@ -77,12 +99,12 @@ jobs: # The vendored OGC schema bundle is compared against what OGC publishes. MFAPI_SCHEMA_FRESHNESS: "1" # The floor is every result line `go test -v` writes, subtests included: - # 77 top-level tests and 64 subtests. Subtests belong in it because a + # 84 top-level tests and 64 subtests. Subtests belong in it because a # t.Run whose body skips is as unrun as its parent would be, and deleting # a case from a table-driven test moves nothing else. Raise it when the # suite grows. Lowering it belongs in the same commit as the removal it # accounts for, because deleting a test is what a skip refusal cannot see. - MFAPI_TEST_FLOOR: "141" + MFAPI_TEST_FLOOR: "148" steps: - uses: actions/checkout@v4 diff --git a/GENERATION.md b/GENERATION.md new file mode 100644 index 0000000..9aa659e --- /dev/null +++ b/GENERATION.md @@ -0,0 +1,70 @@ +# Generation + +MobilityAPI is a tier over MEOS, so what it knows about MEOS is generated from +MEOS rather than kept by hand. The generator lives in `tools/codegen`, its +output is committed, and CI regenerates it against MobilityDB master and +refuses a difference. + +## What is generated + +`catalog_gen.go` is the table of MEOS temporal types: for each one, its name, +its base type, its bounding box, the token `asMFJSON` writes for it, whether it +is spatial, whether it is a number, and whether MEOS interpolates it linearly. + +Run it against any MobilityDB checkout: + +```bash +go run ./tools/codegen -mobilitydb -o catalog_gen.go +``` + +## Where the facts come from + +The generator reads the MEOS sources that define them, in the checkout it is +pointed at: + +| source | what is read | +| --- | --- | +| `meos/src/temporal/meos_catalog.c` | the `[T_X] = "x"` type names; each temporal type's `temptype_basetype` and `type_bboxtype`; and the predicates `temporal_type`, `tnumber_type`, `tspatial_type` and `temptype_supports_linear` | +| `meos/src/temporal/type_out.c` | `temptype_as_mfjson_sb`, the switch that writes each type's MF-JSON type token | + +A type MEOS carries that the MF-JSON switch does not name has no MF-JSON form, +and the empty token in the table says so. That is how `tdouble2`, `tdouble3` +and `tdouble4`, which exist for temporal aggregation, stay off every surface +without a list here naming them. + +The sources are read by anchoring on a definition and counting braces, never by +matching a pattern across a span, and string literals are stepped over so the +`{` inside `{\"type\":\"MovingFloat\",` is read as text rather than as +structure. + +## What the tier adds, and what holds it complete + +Two things are the standard's rather than MEOS's, so they are stated in +`main.go` next to the code that uses them: + +- `ogcScalar` maps a base type to the OGC temporal-property token Part 1 + defines for it, and to the interpolation this tier assumes when a request + body names none. Part 1 defines four tokens and no more, so this is keyed by + base type: every temporal type MEOS builds over one of those bases is carried + automatically, and one over a fifth base is a question for the standard. +- `tPropAlias` holds the further spellings a client may send for a type. + +`catalog_test.go` is what keeps the two halves in step. It asserts that every +non-spatial type MEOS writes MF-JSON for, over a base the standard names, +resolves through `tPropType` with the MEOS type and the MF-JSON token the +generated table carries; that no spatial type resolves, since a spatial value +is a moving feature's geometry rather than a temporal property; and that the +store's value columns are one per scalar type. A temporal type MEOS adds over +one of the standard's bases therefore arrives as a failing test rather than as +a request the tier rejects at run time. + +## The catalog this could read instead + +`MobilityDB/MEOS-API` publishes `meos-idl.json`, the machine-readable catalog +every language binding generates from, and reading it here rather than the C +sources would put this tier on the same footing as the bindings. Two of the +facts above are not in that catalog today: the MF-JSON type token has no field, +and `typeRelations.byBase` holds one temporal type per base, so it carries +`trgeometry` for the base `pose` and drops `tpose`, which shares that base. +Both are single-source facts in `meos_catalog.c` and `type_out.c`, which is +what the generator reads until the catalog states them. diff --git a/catalog_gen.go b/catalog_gen.go new file mode 100644 index 0000000..b873a01 --- /dev/null +++ b/catalog_gen.go @@ -0,0 +1,61 @@ +// Code generated by tools/codegen from the MEOS sources. DO NOT EDIT. +// +// Regenerate with: +// +// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// +// The sources read are meos/src/temporal/meos_catalog.c (the type names, each +// temporal type's base and bounding box, and the predicates temporal_type, +// tnumber_type, tspatial_type and temptype_supports_linear) and +// meos/src/temporal/type_out.c (temptype_as_mfjson_sb). + +package main + +// TemporalType is one MEOS temporal type, as MEOS's own catalog describes it. +// MFJSON is empty for a type asMFJSON does not write, so an empty token is what +// says a type has no MF-JSON form. +type TemporalType struct { + Name string + Base string + Box string + MFJSON string + Spatial bool + Number bool + Linear bool +} + +// temporalTypes is every temporal type MEOS declares, ordered by name. +var temporalTypes = []TemporalType{ + {Name: "tbigint", Base: "int8", Box: "tbox", MFJSON: "MovingBigInteger", Spatial: false, Number: true, Linear: false}, + {Name: "tbool", Base: "bool", Box: "tstzspan", MFJSON: "MovingBoolean", Spatial: false, Number: false, Linear: false}, + {Name: "tcbuffer", Base: "cbuffer", Box: "stbox", MFJSON: "MovingCircularBuffer", Spatial: true, Number: false, Linear: true}, + {Name: "tdouble2", Base: "double2", Box: "tstzspan", MFJSON: "", Spatial: false, Number: false, Linear: true}, + {Name: "tdouble3", Base: "double3", Box: "tstzspan", MFJSON: "", Spatial: false, Number: false, Linear: true}, + {Name: "tdouble4", Base: "double4", Box: "tstzspan", MFJSON: "", Spatial: false, Number: false, Linear: true}, + {Name: "tfloat", Base: "float8", Box: "tbox", MFJSON: "MovingFloat", Spatial: false, Number: true, Linear: true}, + {Name: "tgeogpoint", Base: "geography", Box: "stbox", MFJSON: "MovingPoint", Spatial: true, Number: false, Linear: true}, + {Name: "tgeography", Base: "geography", Box: "stbox", MFJSON: "MovingGeometry", Spatial: true, Number: false, Linear: false}, + {Name: "tgeometry", Base: "geometry", Box: "stbox", MFJSON: "MovingGeometry", Spatial: true, Number: false, Linear: false}, + {Name: "tgeompoint", Base: "geometry", Box: "stbox", MFJSON: "MovingPoint", Spatial: true, Number: false, Linear: true}, + {Name: "th3index", Base: "h3index", Box: "stbox", MFJSON: "MovingH3Index", Spatial: true, Number: false, Linear: false}, + {Name: "tint", Base: "int4", Box: "tbox", MFJSON: "MovingInteger", Spatial: false, Number: true, Linear: false}, + {Name: "tjsonb", Base: "jsonb", Box: "tstzspan", MFJSON: "MovingJsonb", Spatial: false, Number: false, Linear: false}, + {Name: "tnpoint", Base: "npoint", Box: "stbox", MFJSON: "MovingNetworkPoint", Spatial: true, Number: false, Linear: true}, + {Name: "tpcpatch", Base: "pcpatch", Box: "tpcbox", MFJSON: "MovingPCPatch", Spatial: true, Number: false, Linear: false}, + {Name: "tpcpoint", Base: "pcpoint", Box: "tpcbox", MFJSON: "MovingPCPoint", Spatial: true, Number: false, Linear: false}, + {Name: "tpose", Base: "pose", Box: "stbox", MFJSON: "MovingPose", Spatial: true, Number: false, Linear: true}, + {Name: "tposechain", Base: "posechain", Box: "stbox", MFJSON: "MovingPoseChain", Spatial: true, Number: false, Linear: true}, + {Name: "tquadbin", Base: "quadbin", Box: "stbox", MFJSON: "MovingQuadbin", Spatial: true, Number: false, Linear: false}, + {Name: "trgeometry", Base: "pose", Box: "stbox", MFJSON: "MovingRigidGeometry", Spatial: true, Number: false, Linear: true}, + {Name: "ts2cell", Base: "s2cell", Box: "stbox", MFJSON: "", Spatial: true, Number: false, Linear: false}, + {Name: "ttext", Base: "text", Box: "tstzspan", MFJSON: "MovingText", Spatial: false, Number: false, Linear: false}, +} + +// temporalTypeByName indexes temporalTypes by the MEOS type name. +var temporalTypeByName = func() map[string]*TemporalType { + m := make(map[string]*TemporalType, len(temporalTypes)) + for i := range temporalTypes { + m[temporalTypes[i].Name] = &temporalTypes[i] + } + return m +}() diff --git a/catalog_test.go b/catalog_test.go new file mode 100644 index 0000000..7ab96e0 --- /dev/null +++ b/catalog_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "sort" + "strings" + "testing" +) + +// The generated table is what the tier knows about MEOS's temporal types, so +// these assert the shape the rest of the code relies on rather than a list of +// names, which would be the copy the generator exists to remove. + +func TestCatalogIsPopulated(t *testing.T) { + if len(temporalTypes) < 10 { + t.Fatalf("the generated catalog holds %d temporal types, which is too few to have been read from MEOS; regenerate with go run ./tools/codegen", len(temporalTypes)) + } + for _, tt := range temporalTypes { + if tt.Name == "" || tt.Base == "" { + t.Errorf("a catalog row carries no name or no base type: %+v", tt) + } + if got := temporalTypeByName[tt.Name]; got == nil || got.Name != tt.Name { + t.Errorf("temporalTypeByName does not index %q", tt.Name) + } + } +} + +// A temporal property is a non-spatial value, so every non-spatial type MEOS +// writes MF-JSON for and gives a base the standard has a token for must be one +// the store can carry. A type MEOS adds over such a base arrives here as a +// failure rather than as a request the tier rejects at runtime. +func TestEveryScalarTypeTheStandardNamesIsCarried(t *testing.T) { + for _, tt := range temporalTypes { + if tt.Spatial || tt.MFJSON == "" { + continue + } + if _, named := ogcScalar[tt.Base]; !named { + continue + } + got, ok := tPropType(tt.Name) + if !ok { + t.Errorf("%s is a scalar temporal type over base %s, which OGC Part 1 names, and tPropType does not resolve it", tt.Name, tt.Base) + continue + } + if got.cast != tt.Name { + t.Errorf("tPropType(%q).cast = %q, want %q", tt.Name, got.cast, tt.Name) + } + if got.mf != tt.MFJSON { + t.Errorf("tPropType(%q).mf = %q, want the token MEOS writes, %q", tt.Name, got.mf, tt.MFJSON) + } + } +} + +// A spatial type is a moving feature's geometry, never a temporal property, so +// the store must not offer to hold one. +func TestNoSpatialTypeIsATemporalProperty(t *testing.T) { + for _, tt := range temporalTypes { + if !tt.Spatial { + continue + } + if _, ok := tPropType(tt.Name); ok { + t.Errorf("tPropType resolves %q, which is a spatial type and belongs under tgsequence", tt.Name) + } + } +} + +// The tokens a client may send resolve, in the case the standard writes them +// and in lower case, and an unknown one does not. +func TestTPropTypeResolvesTheTokensClientsSend(t *testing.T) { + for _, tok := range []string{"", "TReal", "treal", "tfloat", "measure", "TInteger", "tint", "integer", "TText", "ttext", "string", "TBoolean", "tbool", "boolean"} { + if _, ok := tPropType(tok); !ok { + t.Errorf("tPropType(%q) does not resolve", tok) + } + } + for _, tok := range []string{"tgeompoint", "tpose", "nonsense", "tjsonb"} { + if _, ok := tPropType(tok); ok { + t.Errorf("tPropType(%q) resolves, and it names no OGC temporal property", tok) + } + } +} + +// Two MEOS types carry the OGC token TInteger, tint over int4 and tbigint over +// int8, so the token alone does not name a type. Resolving it by whichever the +// map yields last picks a different type from run to run, and a stored property +// is then read out of a column its table may not have. +func TestATokenSeveralTypesCarryResolvesToOne(t *testing.T) { + carriers := map[string][]string{} + for name, tt := range scalarTemporalTypes { + carriers[tt.ogc] = append(carriers[tt.ogc], name) + } + for token, names := range carriers { + got, ok := tPropType(token) + if !ok { + t.Errorf("the OGC token %q resolves to no type, and %v carry it", token, names) + continue + } + if got.ogc != token { + t.Errorf("tPropType(%q) resolves to a type whose token is %q", token, got.ogc) + } + if len(names) > 1 { + want := ogcCanonical[token] + if want == "" { + t.Errorf("%v all carry the token %q and ogcCanonical names none of them", names, token) + } else if got.cast != want { + t.Errorf("tPropType(%q).cast = %q, want the canonical %q", token, got.cast, want) + } + } + } + // The resolution the conformance fixture depends on: a TInteger property is + // stored in vint, which is the column a store created before tbigint has. + got, ok := tPropType("TInteger") + if !ok || got.cast != "tint" || got.col != "vint" { + t.Errorf("tPropType(\"TInteger\") = %+v, want tint in vint", got) + } +} + +// The OGC token and the default interpolation are what the stored documents +// carry, so they are asserted by value: a change to either rewrites what the +// service answers for properties already in a store. +func TestScalarBindingsAreTheStandardsOwn(t *testing.T) { + want := map[string][2]string{ + "tbool": {"TBoolean", "Step"}, + "tint": {"TInteger", "Step"}, + "tbigint": {"TInteger", "Step"}, + "tfloat": {"TReal", "Linear"}, + "ttext": {"TText", "Discrete"}, + } + for name, w := range want { + got, ok := tPropType(name) + if !ok { + t.Errorf("%s does not resolve", name) + continue + } + if got.ogc != w[0] || got.defInterp != w[1] { + t.Errorf("tPropType(%q) = ogc %q interp %q, want %q and %q", name, got.ogc, got.defInterp, w[0], w[1]) + } + } +} + +// The store's value columns are one per scalar type, named for it, and the DDL +// fragment names each with its type. +func TestValueColumnsAreOnePerScalarType(t *testing.T) { + cols := tPropValueColumnNames() + if len(cols) != len(scalarTemporalTypes) { + t.Fatalf("%d value columns for %d scalar types", len(cols), len(scalarTemporalTypes)) + } + var names []string + for _, c := range cols { + names = append(names, c.name) + if c.name != "v"+strings.TrimPrefix(c.typ, "t") { + t.Errorf("column %q does not name its type %q", c.name, c.typ) + } + if !strings.Contains(tPropValueColumns(), c.name+" "+c.typ) { + t.Errorf("the DDL fragment does not declare %s %s", c.name, c.typ) + } + } + if !sort.StringsAreSorted(names) { + t.Errorf("the value columns are not in a stable order: %v", names) + } +} diff --git a/main.go b/main.go index 3484d02..c6fdbcf 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "os" "os/signal" "regexp" + "sort" "strconv" "strings" "syscall" @@ -765,22 +766,120 @@ func conformantForm(uom string) bool { // assumes when the body omits one. type tType struct{ mf, col, cast, ogc, defInterp string } -// tPropType resolves an OGC temporal property type token to the four scalar -// temporal types MobilityDB carries as time-varying attribute values. +// ogcScalar is the OGC vocabulary for a base type: the temporal property type +// token Part 1 defines for it, and the interpolation this tier assumes when a +// request body names none. Part 1 defines four tokens and no more, so a +// temporal type over any other base carries no OGC temporal-property form, +// which is why this is keyed by base type rather than listing temporal types: +// every temporal type MEOS builds over one of these bases is carried, and one +// it adds over a fifth base is a question for the standard, not a line here. +var ogcScalar = map[string]struct{ token, defInterp string }{ + "bool": {"TBoolean", "Step"}, + "int4": {"TInteger", "Step"}, + "int8": {"TInteger", "Step"}, + "float8": {"TReal", "Linear"}, + "text": {"TText", "Discrete"}, +} + +// tPropAlias resolves the spellings a request may use for a temporal property +// type to the MEOS type name. The OGC token and the MEOS name are accepted for +// every type through the table below; these are the further spellings the tier +// answers to, and the empty string is the default a body without a type gets. +var tPropAlias = map[string]string{ + "": "tfloat", + "measure": "tfloat", + "real": "tfloat", + "float": "tfloat", + "double": "tfloat", + "number": "tfloat", + "integer": "tint", + "int": "tint", + "bigint": "tbigint", + "long": "tbigint", + "tstring": "ttext", + "text": "ttext", + "string": "ttext", + "boolean": "tbool", + "bool": "tbool", +} + +// tPropType resolves a temporal property type token to the scalar temporal +// type MobilityDB carries it as. A token is the OGC name, the MEOS name, or +// one of the aliases above, in any case. func tPropType(t string) (tType, bool) { - switch strings.ToLower(strings.TrimSpace(t)) { - case "", "treal", "tfloat", "measure", "real", "float", "double", "number": - return tType{"MovingFloat", "vfloat", "tfloat", "TReal", "Linear"}, true - case "tint", "tinteger", "integer", "int": - return tType{"MovingInteger", "vint", "tint", "TInteger", "Step"}, true - case "ttext", "tstring", "text", "string": - return tType{"MovingText", "vtext", "ttext", "TText", "Discrete"}, true - case "tbool", "tboolean", "boolean", "bool": - return tType{"MovingBoolean", "vbool", "tbool", "TBoolean", "Step"}, true - } - return tType{}, false + tt, ok := tPropTokens[strings.ToLower(strings.TrimSpace(t))] + return tt, ok +} + +// ogcCanonical names the type an OGC token resolves to where several carry it. +// Part 1's TInteger is a 32-bit integer, so it names tint; tbigint is the wider +// type MobilityDB adds over the same token, and a request naming TInteger asks +// for the narrower one. A token exactly one type carries needs no entry. +var ogcCanonical = map[string]string{ + "TInteger": "tint", } +// tPropTokens is every token tPropType answers to. Each scalar type answers to +// its MEOS name and to the OGC token, both lowercased, and the aliases add the +// spellings a client may send instead. A token two types carry resolves through +// ogcCanonical, never through whichever the map happened to yield last. +var tPropTokens = func() map[string]tType { + byToken := map[string][]string{} + m := map[string]tType{} + for name, tt := range scalarTemporalTypes { + m[name] = tt + byToken[tt.ogc] = append(byToken[tt.ogc], name) + } + for token, carriers := range byToken { + owner := "" + switch { + case len(carriers) == 1: + owner = carriers[0] + default: + owner = ogcCanonical[token] + } + if tt, ok := scalarTemporalTypes[owner]; ok { + m[strings.ToLower(token)] = tt + } + } + for alias, name := range tPropAlias { + if tt, ok := scalarTemporalTypes[name]; ok { + m[alias] = tt + } + } + return m +}() + +// scalarTemporalTypes is every MEOS temporal type this tier can carry as an +// OGC temporal property, projected from the generated catalog: not spatial, an +// MF-JSON form to write, and a base type Part 1 gives a token to. The storage +// column is the type's own name with the leading t as a v, so a type MEOS adds +// needs a column of that name and nothing else. +var scalarTemporalTypes = func() map[string]tType { + m := map[string]tType{} + for _, t := range temporalTypes { + if t.Spatial || t.MFJSON == "" { + continue + } + b, ok := ogcScalar[t.Base] + if !ok { + continue + } + m[t.Name] = tType{ + mf: t.MFJSON, + col: tPropColumn(t.Name), + cast: t.Name, + ogc: b.token, + defInterp: b.defInterp, + } + } + return m +}() + +// tPropColumn is the column mf_tproperty holds a temporal type in: its name +// with a v where the leading t stands, so tfloat is vfloat. +func tPropColumn(name string) string { return "v" + strings.TrimPrefix(name, "t") } + // orTrue returns the JSON boolean v, or true when v is absent — MF-JSON sequence // bounds default to inclusive. func orTrue(v any) bool { @@ -846,12 +945,47 @@ func tPropMFJSON(mfType, defInterp string, body map[string]any) (string, error) func ensureTPropTable(ctx context.Context, q interface { Exec(context.Context, string, ...any) (int64, error) }) error { - _, err := q.Exec(ctx, `CREATE TABLE IF NOT EXISTS mf_tproperty ( + if _, err := q.Exec(ctx, `CREATE TABLE IF NOT EXISTS mf_tproperty ( cid text NOT NULL, fid bigint NOT NULL, name text NOT NULL, ptype text NOT NULL, uom text, description text, - vfloat tfloat, vint tint, vtext ttext, vbool tbool, - PRIMARY KEY (cid, fid, name))`) - return err + `+tPropValueColumns()+`, + PRIMARY KEY (cid, fid, name))`); err != nil { + return err + } + // A store created before MEOS carried one of these types has the other + // columns and not that one, and CREATE TABLE IF NOT EXISTS leaves such a + // table as it stands. + for _, col := range tPropValueColumnNames() { + if _, err := q.Exec(ctx, "ALTER TABLE mf_tproperty ADD COLUMN IF NOT EXISTS "+col.name+" "+col.typ); err != nil { + return err + } + } + return nil +} + +// tPropValueColumnNames is one column per scalar temporal type, ordered by +// type name so the DDL a run writes does not depend on map iteration. +func tPropValueColumnNames() []struct{ name, typ string } { + names := make([]string, 0, len(scalarTemporalTypes)) + for name := range scalarTemporalTypes { + names = append(names, name) + } + sort.Strings(names) + cols := make([]struct{ name, typ string }, 0, len(names)) + for _, name := range names { + cols = append(cols, struct{ name, typ string }{scalarTemporalTypes[name].col, name}) + } + return cols +} + +// tPropValueColumns is those columns as one DDL fragment. +func tPropValueColumns() string { + cols := tPropValueColumnNames() + parts := make([]string, len(cols)) + for i, c := range cols { + parts[i] = c.name + " " + c.typ + } + return strings.Join(parts, ", ") } // clip wraps a temporal expression with atTime for the OGC leaf (instant set) diff --git a/tools/codegen/main.go b/tools/codegen/main.go new file mode 100644 index 0000000..da66f59 --- /dev/null +++ b/tools/codegen/main.go @@ -0,0 +1,376 @@ +// Command codegen writes catalog_gen.go, the tier's table of MEOS temporal +// types, by reading the MEOS sources that define them. +// +// The tier needs five facts about each temporal type: its name, its base type, +// the token asMFJSON writes for it, whether it is spatial, and whether MEOS +// interpolates it linearly. MEOS states all five in C, and every one of them +// moves when a family is added, so a copy of them here is a copy that goes +// stale in silence. The generator reads the definitions themselves: +// +// meos/src/temporal/meos_catalog.c the type names, the base type and bounding +// box of each temporal type, and the +// predicates temporal_type, tnumber_type, +// tspatial_type and temptype_supports_linear +// meos/src/temporal/type_out.c temptype_as_mfjson_sb, the switch that +// writes each type's MF-JSON type token +// +// A type MEOS carries that this switch does not name has no MF-JSON form, and +// the empty token says so rather than a list kept here. +// +// Usage: +// +// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// +// The output is committed, and CI regenerates it against the MobilityDB +// checkout it already has and refuses a difference, so a type MEOS gains +// reaches this tier as a failing check rather than as silence. +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +func main() { + mdb := flag.String("mobilitydb", "mobilitydb", "a MobilityDB checkout") + out := flag.String("o", "catalog_gen.go", "file to write") + flag.Parse() + + catalog, err := readSource(filepath.Join(*mdb, "meos", "src", "temporal", "meos_catalog.c")) + if err != nil { + fail(err) + } + typeOut, err := readSource(filepath.Join(*mdb, "meos", "src", "temporal", "type_out.c")) + if err != nil { + fail(err) + } + + types, err := build(catalog, typeOut) + if err != nil { + fail(err) + } + + src, err := render(types) + if err != nil { + fail(err) + } + if err := os.WriteFile(*out, src, 0o644); err != nil { + fail(err) + } + fmt.Fprintf(os.Stderr, "codegen: %d temporal types written to %s\n", len(types), *out) +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "codegen:", err) + os.Exit(1) +} + +// temporalType is one row of the table. +type temporalType struct { + Enum string // the MeosType enumerator, e.g. T_TGEOMPOINT + Name string // the MEOS type name, e.g. tgeompoint + Base string // its base type's name, e.g. geometry + Box string // its bounding box type's name, e.g. stbox + MFJSON string // the token asMFJSON writes, e.g. MovingPoint + Spatial bool + Number bool + Linear bool // MEOS interpolates it linearly between samples +} + +// readSource returns the file with its C comments blanked, so a type +// enumerator named in prose is never read as a member of a list. +func readSource(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return stripComments(string(b)), nil +} + +// stripComments replaces comment bodies with spaces, keeping every newline so +// the line structure survives. A comment opener inside a string literal is +// part of the string, so literals are stepped over rather than scanned. +func stripComments(s string) string { + var b strings.Builder + b.Grow(len(s)) + for i := 0; i < len(s); { + switch { + case strings.HasPrefix(s[i:], "/*"): + end := strings.Index(s[i+2:], "*/") + if end < 0 { + b.WriteString(blank(s[i:])) + i = len(s) + continue + } + seg := s[i : i+2+end+2] + b.WriteString(blank(seg)) + i += len(seg) + case strings.HasPrefix(s[i:], "//"): + end := strings.IndexByte(s[i:], '\n') + if end < 0 { + end = len(s) - i + } + b.WriteString(blank(s[i : i+end])) + i += end + case s[i] == '"' || s[i] == '\'': + n := literal(s, i) + b.WriteString(s[i:n]) + i = n + default: + b.WriteByte(s[i]) + i++ + } + } + return b.String() +} + +// literal returns the index just past the string or character literal starting +// at i, honouring backslash escapes. +func literal(s string, i int) int { + quote := s[i] + for j := i + 1; j < len(s); j++ { + switch s[j] { + case '\\': + j++ + case quote: + return j + 1 + } + } + return len(s) +} + +// blank returns s with every byte other than a newline as a space. +func blank(s string) string { + out := []byte(s) + for i, c := range out { + if c != '\n' { + out[i] = ' ' + } + } + return string(out) +} + +var ( + reName = regexp.MustCompile(`\[(T_\w+)\]\s*=\s*"([^"]+)"`) + reRelRow = regexp.MustCompile(`\[(T_\w+)\]\s*=\s*\{([^}]*)\}`) + reBase = regexp.MustCompile(`\.temptype_basetype\s*=\s*(T_\w+)`) + reBox = regexp.MustCompile(`\.type_bboxtype\s*=\s*(T_\w+)`) + reEnum = regexp.MustCompile(`\bT_[A-Z0-9_]+\b`) + reCase = regexp.MustCompile(`\bcase\s+(T_\w+)\s*:`) + reMFToken = regexp.MustCompile(`\{\\"type\\":\\"(\w+)\\"`) +) + +func build(catalog, typeOut string) ([]temporalType, error) { + names := map[string]string{} + for _, m := range reName.FindAllStringSubmatch(catalog, -1) { + names[m[1]] = m[2] + } + if len(names) == 0 { + return nil, fmt.Errorf(`meos_catalog.c names no type: the [T_X] = "x" table is not where it is read from`) + } + + base, box := map[string]string{}, map[string]string{} + for _, m := range reRelRow.FindAllStringSubmatch(catalog, -1) { + if b := reBase.FindStringSubmatch(m[2]); b != nil { + base[m[1]] = b[1] + } + if b := reBox.FindStringSubmatch(m[2]); b != nil { + box[m[1]] = b[1] + } + } + + temporal, err := predicate(catalog, "temporal_type") + if err != nil { + return nil, err + } + number, err := predicate(catalog, "tnumber_type") + if err != nil { + return nil, err + } + spatial, err := predicate(catalog, "tspatial_type") + if err != nil { + return nil, err + } + linear, err := predicate(catalog, "temptype_supports_linear") + if err != nil { + return nil, err + } + + mfjson, err := mfjsonTokens(typeOut) + if err != nil { + return nil, err + } + + var out []temporalType + for enum := range temporal { + name, ok := names[enum] + if !ok { + return nil, fmt.Errorf("%s is a temporal type the name table does not name", enum) + } + b, ok := base[enum] + if !ok { + return nil, fmt.Errorf("%s is a temporal type with no temptype_basetype", enum) + } + baseName, ok := names[b] + if !ok { + return nil, fmt.Errorf("%s has base type %s, which the name table does not name", enum, b) + } + boxName := "" + if x, ok := box[enum]; ok { + boxName = names[x] + } + out = append(out, temporalType{ + Enum: enum, + Name: name, + Base: baseName, + Box: boxName, + MFJSON: mfjson[enum], + Spatial: spatial[enum], + Number: number[enum], + Linear: linear[enum], + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// predicate returns the type enumerators named in the body of a boolean +// predicate such as tspatial_type. The body is taken by counting braces from +// the function's own opening brace, never by matching a pattern across it: a +// nested block ends a pattern early and silently shortens the list. +func predicate(src, fn string) (map[string]bool, error) { + body, err := functionBody(src, fn) + if err != nil { + return nil, err + } + set := map[string]bool{} + for _, m := range reEnum.FindAllString(body, -1) { + set[m] = true + } + if len(set) == 0 { + return nil, fmt.Errorf("%s names no type enumerator", fn) + } + return set, nil +} + +// functionBody returns the braced body of the named function definition, which +// is the occurrence of the name at the start of a line followed by its +// parameter list: MEOS writes a definition's return type on the line above, so +// a definition is the only occurrence in that column. +func functionBody(src, fn string) (string, error) { + re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(fn) + `\s*\(`) + loc := re.FindStringIndex(src) + if loc == nil { + return "", fmt.Errorf("no definition of %s", fn) + } + open := strings.IndexByte(src[loc[1]:], '{') + if open < 0 { + return "", fmt.Errorf("%s has no body", fn) + } + start := loc[1] + open + depth := 0 + for i := start; i < len(src); { + switch src[i] { + case '"', '\'': + // A brace inside a string literal is text, not structure: the + // MF-JSON switch writes `{\"type\":\"MovingFloat\",` for every + // type, and counting those opens a depth that never closes. + i = literal(src, i) + continue + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return src[start+1 : i], nil + } + } + i++ + } + return "", fmt.Errorf("%s has an unterminated body", fn) +} + +// mfjsonTokens reads temptype_as_mfjson_sb, whose case labels come in groups: +// several types share one token, since a geometry point and a geography point +// are both a MovingPoint. Labels accumulate until a token is written and are +// assigned together. +func mfjsonTokens(typeOut string) (map[string]string, error) { + body, err := functionBody(typeOut, "temptype_as_mfjson_sb") + if err != nil { + return nil, err + } + out := map[string]string{} + var pending []string + for _, line := range strings.Split(body, "\n") { + if m := reCase.FindStringSubmatch(line); m != nil { + pending = append(pending, m[1]) + continue + } + if m := reMFToken.FindStringSubmatch(line); m != nil { + for _, enum := range pending { + out[enum] = m[1] + } + pending = nil + } + } + if len(out) == 0 { + return nil, fmt.Errorf("temptype_as_mfjson_sb names no type token") + } + return out, nil +} + +func render(types []temporalType) ([]byte, error) { + var b bytes.Buffer + b.WriteString(`// Code generated by tools/codegen from the MEOS sources. DO NOT EDIT. +// +// Regenerate with: +// +// go run ./tools/codegen -mobilitydb -o catalog_gen.go +// +// The sources read are meos/src/temporal/meos_catalog.c (the type names, each +// temporal type's base and bounding box, and the predicates temporal_type, +// tnumber_type, tspatial_type and temptype_supports_linear) and +// meos/src/temporal/type_out.c (temptype_as_mfjson_sb). + +package main + +// TemporalType is one MEOS temporal type, as MEOS's own catalog describes it. +// MFJSON is empty for a type asMFJSON does not write, so an empty token is what +// says a type has no MF-JSON form. +type TemporalType struct { + Name string + Base string + Box string + MFJSON string + Spatial bool + Number bool + Linear bool +} + +// temporalTypes is every temporal type MEOS declares, ordered by name. +var temporalTypes = []TemporalType{ +`) + for _, t := range types { + fmt.Fprintf(&b, "\t{Name: %q, Base: %q, Box: %q, MFJSON: %q, Spatial: %t, Number: %t, Linear: %t},\n", + t.Name, t.Base, t.Box, t.MFJSON, t.Spatial, t.Number, t.Linear) + } + b.WriteString(`} + +// temporalTypeByName indexes temporalTypes by the MEOS type name. +var temporalTypeByName = func() map[string]*TemporalType { + m := make(map[string]*TemporalType, len(temporalTypes)) + for i := range temporalTypes { + m[temporalTypes[i].Name] = &temporalTypes[i] + } + return m +}() +`) + return format.Source(b.Bytes()) +}