From 98cfa56cd6ed3032785833c52d068343352bae8d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 10:08:04 +0000 Subject: [PATCH 01/29] fix(widgets): sync per-property definition attributes from the installed .mpk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NOTE: this does NOT fix mendixlabs/mxcli#716. It is a real gap found while investigating it, committed on its own so the finding is not lost. Measured against Data Widgets 3.10: still 6 CE0463, unchanged by this commit. AugmentTemplate reconciled which properties EXIST between the embedded 11.6 template and the installed .mpk — adding new ones, removing stale ones — but left a surviving property's own attributes at whatever the template captured. Those attributes are part of what Mendix compares when deciding whether a widget's definition has changed, so a widget package that merely flipped one produces CE0463 on every instance with nothing in the property SET to explain it. Confirmed by diffing mxcli's BSON against `mx update-widgets` output and re-running `mx check` after patching each difference in isolation: Gallery OnChangeProperty "onConfigurationChange" -> "" (x4) DatagridDropdownFilter Required false -> true (x2) Both are declared by the .mpk XML (onChange=, required=), so both parsers now read onChange, and syncDefinitionAttrs copies Required/OnChangeProperty onto matching PropertyTypes, recursing into object-typed properties. Only attributes the XML actually declares are synced — overwriting a Mendix-derived one with a zero value would trade one definition mismatch for another. Why it changes nothing for #716: `widget init` never augments the six failing widgets. It reports "9 skipped (built-in or unparseable)", and the skipped set is exactly gallery, dropdownsort and the four datagrid filters — which then fall back to the embedded template verbatim. That skip is the actual bug and lives in the extraction path, not here. Two further findings, recorded but NOT addressed: - modelsdk/widgets/templates/mendix-11.6/*.json is missing AllowUpload in all 29 files (~580 fields); the 11.9 CE0463 fix landed only in sdk/. modelsdk is the default engine. Syncing them had no effect on #716. - The engines disagree on absent required=: sdk parses it false (== "true"), modelsdk true (!= "false"). One of them is wrong. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- modelsdk/widgets/augment.go | 67 +++++++++++++++++++++++++++++++++++++ modelsdk/widgets/mpk/mpk.go | 19 +++++++---- sdk/widgets/augment.go | 67 +++++++++++++++++++++++++++++++++++++ sdk/widgets/mpk/mpk.go | 21 ++++++++---- 4 files changed, 162 insertions(+), 12 deletions(-) diff --git a/modelsdk/widgets/augment.go b/modelsdk/widgets/augment.go index 61842fdde..bb165d728 100644 --- a/modelsdk/widgets/augment.go +++ b/modelsdk/widgets/augment.go @@ -214,6 +214,10 @@ func AugmentTemplate(tmpl *WidgetTemplate, def *mpk.WidgetDefinition) error { // $ID, so reordering the Type list is safe. The .mpk is authoritative. reorderPropertyTypes(tmpl.Type, def) + // A surviving property's own definition attributes are version-specific too; + // reconciling only the property SET leaves CE0463 unexplained (#716). + syncDefinitionAttrs(propTypes, def.Properties) + return nil } @@ -1216,3 +1220,66 @@ func remapObjectTypePointers(objProps []any, idRemap map[string]string) { } } } + +// syncDefinitionAttrs copies the scalar attributes that belong to the widget's +// DEFINITION from the installed .mpk onto the template's PropertyTypes. +// +// AugmentTemplate reconciles which properties exist (adds new ones, removes +// stale ones) but, before mendixlabs/mxcli#716, left a surviving property's own +// attributes at whatever the embedded 11.6-era template captured. Those +// attributes are part of what Mendix compares when deciding whether a widget's +// definition has changed, so a widget package that merely flipped one of them +// produced CE0463 on every instance — with nothing in the property SET to +// explain it. +// +// Observed on Data Widgets 3.10 against the 11.6 templates: +// +// Gallery OnChangeProperty "onConfigurationChange" -> "" (x4) +// DatagridDropdownFilter Required false -> true (x2) +// +// Only attributes the .mpk actually declares are synced. Anything Mendix derives +// but the XML does not carry is left alone — overwriting it with a zero value +// would trade one definition mismatch for another. +func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { + byKey := make(map[string]*mpk.PropertyDef, len(props)) + var index func([]mpk.PropertyDef) + index = func(ps []mpk.PropertyDef) { + for i := range ps { + byKey[ps[i].Key] = &ps[i] + if len(ps[i].Children) > 0 { + index(ps[i].Children) + } + } + } + index(props) + + var walk func([]any) + walk = func(pts []any) { + for _, pt := range pts { + ptMap, ok := pt.(map[string]any) + if !ok { + continue + } + if key, _ := ptMap["PropertyKey"].(string); key != "" { + if p := byKey[key]; p != nil { + ptMap["Required"] = p.Required + ptMap["OnChangeProperty"] = p.OnChange + } + } + // Object-typed properties nest their own PropertyTypes. + if vt, ok := getMapField(ptMap, "ValueType"); ok { + if ot, ok := getMapField(vt, "ObjectType"); ok { + if nested, ok := getArrayField(ot, "PropertyTypes"); ok { + walk(nested) + } + } + } + if ot, ok := getMapField(ptMap, "ObjectType"); ok { + if nested, ok := getArrayField(ot, "PropertyTypes"); ok { + walk(nested) + } + } + } + } + walk(propTypes) +} diff --git a/modelsdk/widgets/mpk/mpk.go b/modelsdk/widgets/mpk/mpk.go index 1d6430b4e..a4bf15a4b 100644 --- a/modelsdk/widgets/mpk/mpk.go +++ b/modelsdk/widgets/mpk/mpk.go @@ -17,12 +17,16 @@ import ( // PropertyDef describes a single property from a widget XML definition. type PropertyDef struct { - Key string // e.g. "staticDataSourceCaption" - Type string // XML type: "attribute", "expression", "textTemplate", "widgets", etc. - Caption string - Description string - Category string // from enclosing propertyGroup captions, joined with "::" - Required bool + Key string // e.g. "staticDataSourceCaption" + Type string // XML type: "attribute", "expression", "textTemplate", "widgets", etc. + Caption string + Description string + Category string // from enclosing propertyGroup captions, joined with "::" + Required bool + // OnChange names the sibling action property Studio Pro runs when this + // property changes. Part of the widget DEFINITION, so a stale value makes + // Mendix report CE0463 "definition of this widget has changed" (#716). + OnChange string DefaultValue string // for enumeration/boolean/integer types IsList bool Multiline bool // for string/textTemplate: multiline="true" @@ -201,6 +205,7 @@ type xmlProperty struct { Type string `xml:"type,attr"` DefaultValue string `xml:"defaultValue,attr"` Required string `xml:"required,attr"` + OnChange string `xml:"onChange,attr"` IsList string `xml:"isList,attr"` Multiline string `xml:"multiline,attr"` DataSource string `xml:"dataSource,attr"` @@ -391,6 +396,7 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini // an explicit required="false" is optional. Defaulting missing→false here // caused within-key CE0463 drift on augment-added keys (issue #600). Required: p.Required != "false", + OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", Multiline: p.Multiline == "true", @@ -463,6 +469,7 @@ func collectNestedProperties(pg xmlPropGroup, parent *PropertyDef, parentCategor // an explicit required="false" is optional. Defaulting missing→false here // caused within-key CE0463 drift on augment-added keys (issue #600). Required: p.Required != "false", + OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", Multiline: p.Multiline == "true", diff --git a/sdk/widgets/augment.go b/sdk/widgets/augment.go index ba148a54c..97c618520 100644 --- a/sdk/widgets/augment.go +++ b/sdk/widgets/augment.go @@ -157,6 +157,10 @@ func AugmentTemplate(tmpl *WidgetTemplate, def *mpk.WidgetDefinition) error { } } + // A surviving property's own definition attributes are version-specific too; + // reconciling only the property SET leaves CE0463 unexplained (#716). + syncDefinitionAttrs(propTypes, def.Properties) + return nil } @@ -1009,3 +1013,66 @@ func remapObjectTypePointers(objProps []any, idRemap map[string]string) { } } } + +// syncDefinitionAttrs copies the scalar attributes that belong to the widget's +// DEFINITION from the installed .mpk onto the template's PropertyTypes. +// +// AugmentTemplate reconciles which properties exist (adds new ones, removes +// stale ones) but, before mendixlabs/mxcli#716, left a surviving property's own +// attributes at whatever the embedded 11.6-era template captured. Those +// attributes are part of what Mendix compares when deciding whether a widget's +// definition has changed, so a widget package that merely flipped one of them +// produced CE0463 on every instance — with nothing in the property SET to +// explain it. +// +// Observed on Data Widgets 3.10 against the 11.6 templates: +// +// Gallery OnChangeProperty "onConfigurationChange" -> "" (x4) +// DatagridDropdownFilter Required false -> true (x2) +// +// Only attributes the .mpk actually declares are synced. Anything Mendix derives +// but the XML does not carry is left alone — overwriting it with a zero value +// would trade one definition mismatch for another. +func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { + byKey := make(map[string]*mpk.PropertyDef, len(props)) + var index func([]mpk.PropertyDef) + index = func(ps []mpk.PropertyDef) { + for i := range ps { + byKey[ps[i].Key] = &ps[i] + if len(ps[i].Children) > 0 { + index(ps[i].Children) + } + } + } + index(props) + + var walk func([]any) + walk = func(pts []any) { + for _, pt := range pts { + ptMap, ok := pt.(map[string]any) + if !ok { + continue + } + if key, _ := ptMap["PropertyKey"].(string); key != "" { + if p := byKey[key]; p != nil { + ptMap["Required"] = p.Required + ptMap["OnChangeProperty"] = p.OnChange + } + } + // Object-typed properties nest their own PropertyTypes. + if vt, ok := getMapField(ptMap, "ValueType"); ok { + if ot, ok := getMapField(vt, "ObjectType"); ok { + if nested, ok := getArrayField(ot, "PropertyTypes"); ok { + walk(nested) + } + } + } + if ot, ok := getMapField(ptMap, "ObjectType"); ok { + if nested, ok := getArrayField(ot, "PropertyTypes"); ok { + walk(nested) + } + } + } + } + walk(propTypes) +} diff --git a/sdk/widgets/mpk/mpk.go b/sdk/widgets/mpk/mpk.go index ba3f2a889..c1f1e50fc 100644 --- a/sdk/widgets/mpk/mpk.go +++ b/sdk/widgets/mpk/mpk.go @@ -17,12 +17,17 @@ import ( // PropertyDef describes a single property from a widget XML definition. type PropertyDef struct { - Key string // e.g. "staticDataSourceCaption" - Type string // XML type: "attribute", "expression", "textTemplate", "widgets", etc. - Caption string - Description string - Category string // from enclosing propertyGroup captions, joined with "::" - Required bool + Key string // e.g. "staticDataSourceCaption" + Type string // XML type: "attribute", "expression", "textTemplate", "widgets", etc. + Caption string + Description string + Category string // from enclosing propertyGroup captions, joined with "::" + Required bool + // OnChange names the sibling action property Studio Pro runs when this + // property changes. It is part of the widget DEFINITION, so a stale value + // makes Mendix report CE0463 "the definition of this widget has changed" + // (mendixlabs/mxcli#716). + OnChange string DefaultValue string // for enumeration/boolean/integer types IsList bool IsSystem bool // true for elements @@ -82,6 +87,7 @@ type xmlProperty struct { Type string `xml:"type,attr"` DefaultValue string `xml:"defaultValue,attr"` Required string `xml:"required,attr"` + OnChange string `xml:"onChange,attr"` IsList string `xml:"isList,attr"` DataSource string `xml:"dataSource,attr"` Caption string `xml:"caption"` @@ -274,6 +280,7 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini Description: p.Description, Category: category, Required: p.Required == "true", + OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", DataSource: p.DataSource, @@ -297,6 +304,7 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini Caption: np.Caption, Description: np.Description, Required: np.Required == "true", + OnChange: np.OnChange, DefaultValue: np.DefaultValue, IsList: np.IsList == "true", DataSource: np.DataSource, @@ -333,6 +341,7 @@ func collectNestedProperties(pg xmlPropGroup, parent *PropertyDef) { Caption: p.Caption, Description: p.Description, Required: p.Required == "true", + OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", DataSource: p.DataSource, From e54a4cc2eac65fe9ea199eb9f20210e0755ef9ff Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 14:16:43 +0000 Subject: [PATCH 02/29] fix(widgets): syncDefinitionAttrs must update keys, never invent them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit assigned Required/OnChangeProperty into every matching PropertyType, which ADDS the key when the map doesn't already have it. That is the mendixlabs/mxcli#759 failure shape — writing a property the target version's metamodel may not define — and it is the opposite of what the platform does: `mx update-widgets` output for the Gallery omits both keys on those PropertyTypes entirely, while mxcli was emitting them. Caught by diffing mxcli's Type node against update-widgets' with the ids normalised by graph position rather than stripped: the diff was 286 lines of mxcli-only "OnChangeProperty"/"Required" entries with no counterpart. Now the sync updates a key only when the PropertyType already carries it. Verified: Data Widgets 3.4 (shipped with 11.12) stays at 0 CE0463, and 3.10 is unchanged at 6 — this commit removes a hazard, it does not move #716. #716 remains open and unfixed. The cause is upstream of this code: widget init deliberately skips any widget with a hand-crafted definition in sdk/widgets/definitions/ ("the built-in def overrides any .mpk-derived one" — widget_defs.go), which is exactly gallery, dropdownsort and the four datagrid filters. Those definitions are embedded because older Mendix versions do not ship the widgets as marketplace mpks, so the override cannot simply be removed: it needs to become a fallback, used when the project has no .mpk and stepped aside when it does. Tested that inversion in isolation — all 42 defs generated, CE0463 still 6 — so the def is not the only input; the embedded TEMPLATE feeding AugmentTemplate is the other half. Also confirmed while probing: augmentFromMPK does reach Gallery and the filters and resolves the right .mpk, so the gap is in what augmentation reconciles, not in whether it runs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- modelsdk/widgets/augment.go | 13 +++++++++++-- sdk/widgets/augment.go | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/modelsdk/widgets/augment.go b/modelsdk/widgets/augment.go index bb165d728..0a42a7395 100644 --- a/modelsdk/widgets/augment.go +++ b/modelsdk/widgets/augment.go @@ -1262,8 +1262,17 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } if key, _ := ptMap["PropertyKey"].(string); key != "" { if p := byKey[key]; p != nil { - ptMap["Required"] = p.Required - ptMap["OnChangeProperty"] = p.OnChange + // Update in place only. Adding a key the PropertyType does not + // already carry invents a property this Mendix version may not + // define — the mendixlabs/mxcli#759 failure shape, and the + // opposite of what `mx update-widgets` produces (its Gallery + // PropertyTypes omit both keys entirely). + if _, ok := ptMap["Required"]; ok { + ptMap["Required"] = p.Required + } + if _, ok := ptMap["OnChangeProperty"]; ok { + ptMap["OnChangeProperty"] = p.OnChange + } } } // Object-typed properties nest their own PropertyTypes. diff --git a/sdk/widgets/augment.go b/sdk/widgets/augment.go index 97c618520..a779256d3 100644 --- a/sdk/widgets/augment.go +++ b/sdk/widgets/augment.go @@ -1055,8 +1055,17 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } if key, _ := ptMap["PropertyKey"].(string); key != "" { if p := byKey[key]; p != nil { - ptMap["Required"] = p.Required - ptMap["OnChangeProperty"] = p.OnChange + // Update in place only. Adding a key the PropertyType does not + // already carry invents a property this Mendix version may not + // define — the mendixlabs/mxcli#759 failure shape, and the + // opposite of what `mx update-widgets` produces (its Gallery + // PropertyTypes omit both keys entirely). + if _, ok := ptMap["Required"]; ok { + ptMap["Required"] = p.Required + } + if _, ok := ptMap["OnChangeProperty"]; ok { + ptMap["OnChangeProperty"] = p.OnChange + } } } // Object-typed properties nest their own PropertyTypes. From 84a462c3bba9b1c34c8aa81b4ee5e37d7205ff6e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 16:52:46 +0000 Subject: [PATCH 03/29] docs: CE0463 after a widget-package upgrade is usually not an mxcli bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Testing mendixlabs/mxcli#716 against two real mxcli-built projects instead of the synthetic doctype fixture reversed the diagnosis. Ledger (Mendix 11.12), Data Widgets 3.4 as authored -> 3.11.3: 0 errors -> 36 CE0463 -> 0 again after `mx update-widgets` (7 mxcli-authored, 29 Studio Pro's own template widgets) TimeRegistration, same upgrade: 29 CE0463, ALL of them Studio Pro's own widgets, zero mxcli-authored. Cause, from diffing dgTransactions against the update-widgets output with ids normalised by graph position: a single dropped property. key="advanced" ("Enable advanced options") is in Datagrid.xml at 3.4 and absent at 3.10 and 3.11.3. update-widgets deletes both the WidgetPropertyType and its WidgetProperty; the rest of the 3213-line diff is index shift. So a package that drops a property leaves every stored instance carrying one the new definition lacks — exactly what CE0463 reports, and exactly what its message tells you to fix. mxcli wrote a correct definition for the version installed at authoring time. This corrects what I reported earlier from the fixture. I claimed mxcli-authored DataGrid2 was clean and that #716's title named the wrong widget; in Ledger six of ten authored DataGrid2s fail. Both statements were artifacts of measuring in a blank project whose own template widgets were already failing for the upgrade reason, which inflates the count and misattributes it. Documented as a diagnosis procedure with the two controls that separate the cases — do Studio Pro's own widgets fail too, and does update-widgets clear it — since both were needed to reach the answer and neither is obvious from the mx check output. The genuine mxcli defect is the residue after those controls: authored FRESH against 3.10/3.11, DataGrid2 is clean and only Gallery and DatagridDropdownFilter still produce CE0463. #716 stays open, scoped to that. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/fix-issue.md | 1 + .../WIDGET_BSON_VERSION_COMPATIBILITY.md | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 2cfedb935..5379942e9 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -321,6 +321,7 @@ cases for these three BSON types — they fell to `default: return nil`. | An import/export mapping over an entity created with `EXTENDS` maps only its **own** attributes; every inherited field shows unmapped in Studio Pro, and `mx check` reports CE1613 "The selected attribute 'Mod.Child.Attr' no longer exists". An inherited Boolean/DateTime element also gets `DataType=String` | The mapping builder prefixed the entity being mapped unconditionally (`attr = parentEntity + "." + attr`), but a member reference is qualified against the entity that **declares** it — the same rule as entity access rules (#758) and the change-object writer (#451). Separately `resolveAttributeType` scanned only the entity's own attributes and fell through to its `"String"` default | `mdl/executor/cmd_import_mappings.go` and `cmd_export_mappings.go` (both carry the same two lines), `mdl/executor/entity_hierarchy.go` (`ResolveMemberRef`, `ResolveMemberType`) | Route both sites through the generalization walk added for #758: `ResolveMemberRef` returns the declaring-entity reference and `ResolveMemberType` finds the type up the chain, each falling back to the old behaviour when the member cannot be resolved. **Watch for the sibling defect**: the old `resolveAttributeType` matched entities **by name across every domain model**, so a same-named entity in another module could win — resolve the module by name instead. **Generalisable**: when one rule has several call sites, a fix at one of them proves nothing about the others; grep for the *pattern* (`range entity.Attributes`, `parentEntity + "."`) rather than the reported symptom. Repro `mdl-examples/bug-tests/703-mapping-inherited-attributes.mdl`; A/B on the same project shows `Map703.Contract.DocName` (CE1613) become `Map703.DocumentBase.DocName`. Issue #703, umbrella #765 | | `alter settings model JavaVersion = 'Java21'` on Mendix 11.12+ produces a project mxbuild refuses to **load**: `mx check` reports `System.ArgumentOutOfRangeException ... (Parameter 'majorVersion is an unsupported value: Java21')` at `JavaVersionExtensions.fromString`. Every check downstream of the settings unit is lost with it | Mendix renamed the property between 11.6 (`JavaVersion` = `"Java21"`) and 11.12 (`JavaMajorVersion` = `"21"`) — and the rename changed the **value format** as well as the key. The #759 fix followed only the key, writing the caller's value through verbatim, so the 11.6 spelling landed in the 11.12 key | `mdl/settingsoverlay/settingsoverlay.go` (`JavaVersionValue`, `SetJavaVersion`) — shared by both engines; the dead third copy in `modelsdk/mpr/serialize_services.go` carried it too | Render the value in the dialect the stored key expects: strip/add the `Java` prefix per key, and pass an unrecognisable value through untouched so a typo surfaces as a Mendix error instead of a mangled setting. **Generalisable**: a renamed property is not only a renamed key — check whether the value encoding moved with it, and cover *both* directions (either spelling in, document's dialect out). Note the sharper failure mode: the original #759 shape was an unknown property, which mxbuild **tolerates**, so only Studio Pro broke; a wrong *value* for a known enum is a hard build failure, which is why this one surfaced as a red nightly rather than a user report. Repro `mdl-examples/bug-tests/759-java-version-value-dialect.mdl`. Issue #759 (follow-up) | | On **Mendix 11.13 only**, every microflow using `EXECUTE DATABASE QUERY` fails `mx check` with **CE5277** "Please re-run and save the query to fix the error", once per activity. The queries themselves report nothing — the error lands on the *activities* pointing at them, so it reads like a microflow defect. Both engines. 11.12 and below are clean | 11.13 replaced the integer `QueryType` (1 = custom SQL) on `DatabaseConnector$DatabaseQuery` with a `Type` **string enum** (`Select` / `NonSelect` / `Unknown`), shipping a one-time conversion (`ExternalDatabaseConnectionQueryTypeConversion`) for old documents. mxcli wrote the legacy integer unconditionally, so on 11.13 the new property was simply **absent** — and an absent `Type` reads as Unknown, which is exactly what CE5277 reports | `mdl/dbconnector/querytype.go` (new, shared by both engines), `sdk/mpr/writer_dbconnection.go` + `parser_dbconnection.go`, `mdl/backend/modelsdk/db_write.go` + `integration_read.go`, `model/types.go` (`DatabaseQuery.QueryTypeName`) | Branch on the project's Mendix version (`ProjectVersion().IsAtLeast(11, 13)`) and write **exactly one** spelling. Writing both is not a safe hedge — a property the target's metamodel does not define is the #759 Studio-Pro-won't-open shape. Read side must accept either, or the next ALTER of an 11.13 project writes Unknown straight back. mxcli can't derive the type the way Studio Pro does (running the query and inspecting the result set), so it reads the leading SQL keyword — still better than Mendix's own converter, which marks every migrated query `Select` regardless of statement. **Diagnosis method**: `mx convert -p -s ` with the NEW mxbuild runs the version's own migration, then diff the BSON — that is what showed `QueryType: 1` → `Type: "Select"` without guessing. **Generalisable**: onboarding a new Mendix minor is not just adding it to the nightly matrix — run the doctype corpus against it first (`MX_BINARY=~/.mxcli/mxbuild//modeler/mx go test -tags integration -run TestMxCheck_DoctypeScripts`), because a renamed property surfaces as a red matrix job, not a compile error. Repro `mdl-examples/bug-tests/1113-database-query-type-enum.mdl`. Sibling drift found in the same sweep and deliberately NOT fixed: **CE5278** ("The JDBC driver is missing from the module settings"), a new 11.13 check about the module's Java dependencies, which mxcli has no way to author | +| A user reports CE0463 "the definition of this widget has changed" on DataGrid2 / Gallery / filters after upgrading the **Data Widgets** marketplace module. Reads like template drift; on real projects it is usually **not an mxcli bug at all** | A widget package that DROPS a property leaves every *stored* instance carrying a property the new definition lacks — which is precisely what CE0463 reports, and what its own message ("Update all widgets") tells you to fix. Ledger on 11.12: 0 errors at Data Widgets 3.4 (as authored) → 36 CE0463 at 3.11.3 → **0 again after `mx update-widgets`**. Single cause: `key="advanced"` is in `Datagrid.xml` at 3.4 and gone at 3.10/3.11 | no code change — diagnostic. See `docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md` "CE0463 after a widget-package upgrade" | **Two controls settle it, neither optional.** (1) Do **Studio Pro's own** widgets fail too? A blank project's `dataGrid2_*`/`gallery1,2`/`drop_downFilter1,2` are Mendix-authored — if they fail alongside mxcli's, the tool is not the variable (29 of Ledger's 36 were these). (2) Does **`mx update-widgets` clear it**? If yes, mxcli's BSON was structurally valid and correct for the version it was written against; genuine template bugs do NOT clear this way (the Image stale default and the number-filter markerless array both needed template fixes). **The real mxcli defect is the residue after those controls**: author FRESH against the new package (`widget init` + author + `mx check`) — on 3.10/3.11 that leaves DataGrid2 **clean** and only Gallery + DatagridDropdownFilter failing, i.e. far narrower than the issue as filed. **Trap that cost a full round-trip**: measuring with the doctype fixtures alone mixes both cases, because their pages live in a blank project whose own template widgets are already failing — subtract by widget NAME against a control project that ran no mxcli command. Issue #716 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md index 025e7b98c..b7698ee0c 100644 --- a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md +++ b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md @@ -108,6 +108,53 @@ regression guard `TestTemplates_NoMarkerlessEmptyArrays` (in both `sdk/widgets` and `modelsdk/widgets`) walks every embedded template and fails on any bare `[]`. When onboarding or re-extracting a template, never emit an empty markerless array. +## CE0463 after a widget-package upgrade is usually NOT an mxcli bug + +Before treating a CE0463 report as template drift, establish **when the widget was +authored relative to the installed package**. The two cases look identical in +`mx check` output and have completely different causes. + +**Case 1 — authored against version A, package upgraded to version B.** Expected +Mendix behaviour, not an mxcli defect. A widget package that drops a property +leaves every *stored* instance carrying a property the new definition no longer +has, which is exactly what CE0463 reports and exactly what its message +("Update this widget / Update all widgets") tells you to fix. + +Worked example (mendixlabs/mxcli#716, Ledger on Mendix 11.12): + +| | | +|---|---| +| Data Widgets 3.4 (as authored) | **0 errors** | +| upgraded to 3.11.3 | 36 CE0463 — 7 mxcli-authored, 29 Studio Pro's own template widgets | +| after `mx update-widgets` | **0 errors** | + +The cause was a single dropped property: `key="advanced"` ("Enable advanced +options") is present in `Datagrid.xml` at 3.4 and absent at 3.10 and 3.11.3. +`update-widgets` deletes both the `WidgetPropertyType` and its `WidgetProperty`; +everything else in the diff is index shift. + +**Two controls make this diagnosis, and neither is optional:** + +1. **Do Studio Pro's own widgets fail too?** A blank project's `dataGrid2_*`, + `gallery1/2`, `drop_downFilter1/2` are authored by Mendix. If they fail + alongside mxcli's, the tool is not the variable. (Here: 29 of the 36.) +2. **Does `mx update-widgets` clear it?** If yes, mxcli's BSON was structurally + valid — it was correct for the version it was written against. Genuine + template bugs do *not* clear this way; the Image stale-default and the + number-filter markerless array both needed template fixes. + +**Case 2 — authored fresh against the new package and still failing.** This is +the mxcli defect. Create a project with the new package installed, run +`widget init`, author the widgets, then `mx check`. On Data Widgets 3.10/3.11 +that isolates a much narrower failure than #716 as filed: freshly authored +DataGrid2 is **clean**, while Gallery and DatagridDropdownFilter still produce +CE0463 (6 instances across the v0.10 fixture). + +**Do not measure Case 2 with the doctype fixtures alone.** Their pages sit in a +blank project whose own template widgets are already failing from Case 1, so a +raw CE0463 count mixes the two. Subtract by widget *name* against a control +project that ran no mxcli command at all. + ## Onboarding a new Mendix minor (e.g. 11.10, 12.0) The CE0463 fix methodology used for 11.9 generalizes. Steps: From 26a04787415639becbedb3a9808843dbb865cce1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 19:30:55 +0000 Subject: [PATCH 04/29] docs: proposal for reconciling stored widget instances against installed mpks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mxcli authors pluggable widgets correctly for the package installed at authoring time and has nothing that reconciles instances already stored in the model when that package later changes. Studio Pro has "Update all widgets", mxbuild has `mx update-widgets` (which destroys mprcontents/ on MPR v2), mxcli has no equivalent — so a headless workflow breaks the moment a developer upgrades a widget module. Grounded in the #716 measurements: Ledger goes 0 -> 36 CE0463 on a Data Widgets 3.4 -> 3.11.3 upgrade and back to 0 after `mx update-widgets`, with the whole 3213-line dgTransactions diff reducing to one dropped property (key="advanced", present at 3.4, gone at 3.10+). That update-widgets clears it is the load-bearing fact: mxcli's BSON was valid for the version it was written against, so this is missing reconciliation, not template drift. Scoped against the existing widget proposals rather than overlapping them — multi_version_pluggable_widgets covers newly created instances and explicitly scopes stale stored ones out; update_builtin_widget_properties and bulk_widget_property_updates are about the MDL UPDATE WIDGETS SET statement, which sets values the author chooses rather than reconciling schema they do not. Proposes a `widget sync` CLI subcommand, not new MDL, for that reason. Records the measurement trap that produced a wrong diagnosis mid-investigation: the doctype fixtures sit in a blank project whose own Studio-Pro-authored widgets also fail after a package upgrade, so counts must be subtracted by widget name against an untouched control. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- ...PROPOSAL_widget_instance_reconciliation.md | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md diff --git a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md new file mode 100644 index 000000000..08d3988e5 --- /dev/null +++ b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md @@ -0,0 +1,214 @@ +# Proposal: `mxcli widget sync` — reconcile stored widget instances against installed .mpk packages + +**Status:** Draft +**Date:** 2026-08-03 + +## Problem Statement + +mxcli authors pluggable-widget instances correctly for the widget package installed +**at authoring time**. It has nothing that reconciles instances **already stored in +the model** when that package later changes. Studio Pro has "Update widget" / +"Update all widgets"; mxbuild has `mx update-widgets`; mxcli has no equivalent. + +So an mxcli-only workflow — the whole point of the tool — breaks the moment a +developer upgrades a widget module. The remedies today are both bad: + +- **Open Studio Pro** and click Update all widgets, which defeats headless use. +- **Run `mx update-widgets`**, which on MPR v2 **destroys `mprcontents/`**, collapsing + the project back to a single-file v1 layout. Documented as a data-loss trap in + `.claude/skills/fix-issue.md`; observed again while investigating #716. + +### Measured evidence (mendixlabs/mxcli#716) + +Two real mxcli-built projects, Mendix 11.12, upgrading Data Widgets 3.4 → 3.11.3: + +| Project | as authored (DW 3.4) | after upgrade to 3.11.3 | after `mx update-widgets` | +|---|---|---|---| +| Ledger | 0 errors | 36 CE0463 (7 mxcli-authored, 29 Studio Pro's own) | **0 errors** | +| TimeRegistration | 0 errors | 29 CE0463 (all 29 Studio Pro's own) | — | + +The cause is mechanical. Diffing `dgTransactions` against the `update-widgets` +output (ids normalised by graph position, not stripped) gives 3213 lines, of which +exactly one is a real change: mxcli's stored instance carries a property the new +package dropped. + +``` +key="advanced" ("Enable advanced options") + Datagrid.xml @ 3.4.0 -> present + Datagrid.xml @ 3.10.0 -> absent + Datagrid.xml @ 3.11.3 -> absent +``` + +`update-widgets` deletes the `CustomWidgets$WidgetPropertyType` from the Type and +its paired `CustomWidgets$WidgetProperty` from the Object. Everything else in the +diff is `$ID` index shift. + +**That `update-widgets` clears it to 0 is the key fact**: mxcli's BSON was +structurally valid and correct for the version it was written against. This is not +template drift — it is missing reconciliation. Genuine template bugs do *not* clear +this way (the Image stale default and the number-filter markerless array both +needed template fixes). + +### Why this is not covered by existing proposals + +[`PROPOSAL_multi_version_pluggable_widgets.md`](PROPOSAL_multi_version_pluggable_widgets.md) +covers the **creation** path and explicitly scopes *out* the case here: + +> They move independently (update Mendix without widgets, or a widget `.mpk` without +> the project's existing instances → Studio Pro shows `CE0463` on those stale +> instances = the normal "Update widget" prompt). The authoritative schema for a +> *newly created* instance is the currently installed `.mpk`. + +This proposal is the other half: the authoritative schema for an **already stored** +instance, when the `.mpk` moves under it. The two are complementary and share +`AugmentTemplate`'s reconciliation logic. + +[`PROPOSAL_update_builtin_widget_properties.md`](PROPOSAL_update_builtin_widget_properties.md) +and [`PROPOSAL_bulk_widget_property_updates.md`](PROPOSAL_bulk_widget_property_updates.md) +are about the MDL `UPDATE WIDGETS SET … WHERE …` statement, which *sets property +values* the author chooses. This proposal *reconciles schema* the author does not +choose. Different operation, and the naming must not collide — see Open Questions. + +## BSON Structure + +No new document type. The operation rewrites two paired arrays inside every stored +`CustomWidgets$CustomWidget` node on every page, snippet and building block. + +| Node | Path | Role | +|---|---|---| +| `CustomWidgets$WidgetType` | `.Type.ObjectType.PropertyTypes[]` | the widget's schema, one `CustomWidgets$WidgetPropertyType` per property, keyed by `PropertyKey` | +| `CustomWidgets$WidgetObject` | `.Object.Properties[]` | the instance's values, one `CustomWidgets$WidgetProperty` per property, bound to its type by `TypePointer` | + +The three reconciliation operations, all already implemented in +`sdk/widgets/augment.go` for the *template*: + +1. **Remove stale** — a `PropertyKey` present in the stored instance but absent from + the installed `.mpk`. This is the #716 case (`advanced`). `removeProperties` + deletes the `PropertyType` and its paired `WidgetProperty` together. +2. **Add missing** — a property the `.mpk` declares that the instance lacks. + `clonePropertyPair` / `createPropertyPair` emit both halves with the `.mpk` + default value. +3. **Update definition attributes** — a surviving property whose own attributes + changed (`Required`, `OnChangeProperty`). `syncDefinitionAttrs`, added on + `claude/gallery-dropdownfilter-ce0463-716`; **update-in-place only, never add a + key the node does not already carry** (adding one is the #759 failure shape, and + `update-widgets` output omits both keys on Gallery PropertyTypes). + +**Pairing invariant.** A `WidgetProperty` is bound to its `WidgetPropertyType` by +`TypePointer`. Removing or adding one half without the other yields the +`StreamingBsonUnitReader` "does not contain a constructor with a parameter of type +WidgetValue" class of load failure. Every mutation must move both halves together. + +**Nested object types.** `DataGrid2` columns, `Gallery` items and chart series nest +their own `PropertyTypes` inside an `IsList` object property. `augmentNestedObjectType` +already walks these; instance reconciliation must recurse the same way, per stored +list entry rather than once per widget. + +## Proposed CLI + +```bash +# Reconcile every stored widget instance against the installed .mpk set. +mxcli widget sync -p app.mpr + +# Preview without writing — the default posture for a destructive-ish operation. +mxcli widget sync -p app.mpr --dry-run + +# Narrow the blast radius. +mxcli widget sync -p app.mpr --widget com.mendix.widget.web.datagrid.Datagrid +mxcli widget sync -p app.mpr --page MyModule.Overview +``` + +Dry-run output names every instance and every property, because silently deleting a +stored property is exactly what a user needs to see before it happens: + +``` +MyModule.Transactions_Overview + dgTransactions (com.mendix.widget.web.datagrid.Datagrid 3.4.0 -> 3.11.3) + - remove advanced (dropped in 3.10.0) + + add loadingType (added in 3.7.0, default "spinner") + ~ update refreshInterval Required false -> true + +3 pages, 7 widget instances, 12 property changes. Re-run without --dry-run to apply. +``` + +**No new MDL statement.** `UPDATE WIDGETS SET … WHERE …` already exists and means +"set a property value I chose". Reconciliation is a maintenance operation on schema +the author does not choose; overloading that verb would conflate the two. A CLI +subcommand next to `widget init` is the honest home. + +## Implementation Plan + +The reconciliation logic exists; what is missing is applying it to **stored +instances** rather than to a freshly loaded template. + +### Files to modify/create + +| File | Change | +|------|--------| +| `cmd/mxcli/cmd_widget.go` | New `sync` subcommand: flags, dry-run rendering, summary | +| `mdl/executor/widget_sync.go` *(new)* | Orchestration: enumerate documents → locate CustomWidget nodes → diff vs `.mpk` → apply → write back | +| `sdk/widgets/augment.go` | Extract the add/remove/attr-sync core so it operates on an arbitrary `(PropertyTypes, Properties)` pair, not only on a `WidgetTemplate`. No behaviour change to the existing call. | +| `modelsdk/widgets/augment.go` | Same extraction — the engines carry independent copies | +| `mdl/backend/*/widget_*.go` | Backend method to enumerate + mutate widget subtrees in place (mutator pattern, per ADR-0002; no BSON in the executor) | +| `mdl/backend/mock/` | `Func`-field stub with the standard "not configured" default | + +### Order of operations + +1. **Read-only inventory first.** `--dry-run` reporting with zero mutation, validated + against the #716 projects: it must name exactly the 7 authored Ledger widgets and + the one `advanced` removal per DataGrid2. +2. **Extract the reconciliation core** from `AugmentTemplate` with the existing + template tests still green — a pure refactor, committed separately. +3. **Apply to stored instances**, one document type at a time (pages, then snippets, + then building blocks). +4. **Nested object types** last; they are the part most likely to break the pairing + invariant. + +## Version Compatibility + +No version gate. The operation is driven by the installed `.mpk` set, not by the +Mendix version, and it is a no-op on a project whose widgets already match. + +It must, however, be **correct on MPR v1 and v2**. `mx update-widgets` collapsing v2 +`mprcontents/` into a v1 `Unit` table is the specific failure this command exists to +avoid; an integration test must assert `mprcontents/` survives. + +## Test Plan + +| Tier | Test | +|---|---| +| Unit | Reconciliation core: remove-stale, add-missing, attr-sync, and the pairing invariant (removing a `PropertyType` removes its `WidgetProperty`) | +| Unit | Nested object-type recursion — a DataGrid2 column list where the column schema changed | +| Integration | Author the v0.10 fixture against Data Widgets 3.4, upgrade the `.mpk` set to 3.11.3, run `widget sync`, assert `mx check` = 0 CE0463 | +| Integration | `mprcontents/` still present and the project still opens after a sync on MPR v2 | +| Integration | Idempotence — a second sync reports zero changes | +| Regression | A project already matching its `.mpk` is byte-identical after a sync | +| Fixture | `mdl-examples/bug-tests/716-widget-package-upgrade.mdl` | + +**Measure against a control.** The doctype fixtures live in a blank project whose own +Studio-Pro-authored widgets (`dataGrid2_*`, `gallery1/2`, `drop_downFilter1/2`) also +fail after a package upgrade. A raw CE0463 count mixes them with mxcli's; subtract by +widget **name** against a project that ran no mxcli command. Getting this wrong +produced a confident wrong diagnosis during the #716 investigation. + +## Open Questions + +1. **Naming.** `widget sync` vs `widget update` vs `widget reconcile`. `update` reads + best but sits one word away from the MDL `UPDATE WIDGETS SET`, which does something + different. Leaning `sync` for that reason — worth a second opinion. +2. **Value preservation on a retyped property.** If a property survives but changes + type (`string` → `enumeration`), is the stored value coerced, reset to the `.mpk` + default, or does the sync refuse and report? Refusing is safest and matches + ADR-0005 guard-don't-drop, but may be too strict to be useful. Needs a real + example — none was observed in the 3.4 → 3.11.3 upgrade. +3. **Should `run --local` / `docker build` sync automatically?** Convenient, but an + implicit model mutation during a build is exactly the kind of surprise this repo + avoids elsewhere. Recommend explicit only, with the build *reporting* drift. +4. **Interaction with the built-in definition override.** `widget_defs.go` skips any + widget with a hand-crafted def (gallery, dropdownsort, four filters), so those + never see the project's `.mpk`. Reconciliation must not inherit that skip, or it + will no-op on exactly the widgets #716 is about. Fixing the override to be a + fallback is arguably a prerequisite — see the remaining #716 residue. +5. **Does this subsume `mx update-widgets` in CI?** If reconciliation is faithful, the + doctype harness could drop its "we deliberately do NOT run update-widgets" note. + Not a goal, but a good confidence signal if it turns out true. From b531fbac881402d6ead7b964ffb9fc4bc46fd2eb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 21:22:52 +0000 Subject: [PATCH 05/29] docs: record why widget init + catalog refresh does not fix stale instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on the upgraded Ledger project: `mxcli widget init` (33 defs regenerated from the 3.11.3 mpks) followed by `refresh catalog full force` leaves CE0463 at 36, unchanged. Both write derived artifacts. The regenerated datagrid.def.json is current and no longer mentions `advanced`; the stored page BSON for dgTransactions still carries PropertyKey "advanced". Extraction succeeded, nothing rewrote the pages — and pages are what CE0463 reads. Recorded as a rejected alternative because it is the obvious cheaper answer and deserves an explicit measurement rather than an assertion. Re-running widget init after an upgrade remains necessary for newly authored widgets; it is just not sufficient for existing ones. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- ...PROPOSAL_widget_instance_reconciliation.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md index 08d3988e5..02691c74a 100644 --- a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md +++ b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md @@ -136,6 +136,38 @@ MyModule.Transactions_Overview the author does not choose; overloading that verb would conflate the two. A CLI subcommand next to `widget init` is the honest home. +## Rejected alternative: just re-run `widget init` + `refresh catalog` + +The obvious cheaper answer is that after installing an updated widget package you +re-run the normal extraction and refresh the catalog. **Measured on the upgraded +Ledger project — it changes nothing:** + +``` +before: 36 CE0463 + mxcli widget init -> Extracted: 33 new, 0 refreshed, 9 skipped + refresh catalog full force -> Catalog cached +after: 36 CE0463 +``` + +Both commands operate on **derived artifacts**, not on the model: + +| Artifact | Written by | Consumed by | +|---|---|---| +| `.mxcli/widgets/*.def.json` | `widget init` | **future** authoring — which MDL keyword routes to which property | +| `.mxcli/catalog.db` | `refresh catalog` | queries, lint rules, `show`/`select` | +| `mprcontents/.mxunit` | page writers only | **mxbuild and Studio Pro — this is what CE0463 reads** | + +Confirmed directly after running both commands against Data Widgets 3.11.3: + +- the regenerated `datagrid.def.json` is **current** — it no longer mentions `advanced` +- the stored page BSON for `dgTransactions` **still carries** `PropertyKey: "advanced"` + +Extraction succeeded; nothing rewrote the pages. Re-running `widget init` after an +upgrade is still *necessary* — it is what makes newly authored widgets match the new +package — but it is not *sufficient*, and the two halves should not be conflated. +That split is precisely why this proposal exists as a separate operation rather than +as a flag on `widget init`. + ## Implementation Plan The reconciliation logic exists; what is missing is applying it to **stored From 0f89f30060973845d95db9ac4ee7561769e89472 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 21:56:13 +0000 Subject: [PATCH 06/29] docs: the built-in def override is not the #716 lever (measured, reverted) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attempted the fallback change: when a project ships an .mpk for a widget that also has a hand-crafted definition, generate the def from the .mpk and overlay only the built-in's identity and routing (MDL keyword, template file, child slots, mappings). The naive inversion is a trap on its own — DeriveMDLName turns DROPDOWNFILTER into DATAGRIDDROPDOWNFILTER, renaming the keyword every script uses and pointing templateFile at a file that does not exist — so the merge preserved those. It still fails on both counts: - No upside. Fresh-authoring CE0463 on Data Widgets 3.10 stayed at 6, the same result as the earlier blind inversion. `.def.json` carries ROUTING, not schema: GenerateDefJSON emits propertyMappings/childSlots/objectLists. The schema CE0463 compares comes from the embedded template plus augmentFromMPK — a different pipeline the def never touches. - Real downside. The doctype gate caught a regression in 17-custom-widget-examples: CE7006 "Selected value is not valid for attribute 'title'" and CE7247 "Move this widget into a data container" on the TEXTFILTER, because hand-authored routing encodes behaviour the generator cannot derive (the attrChoice="linked" rule from #605). Reverted. Recorded in the proposal as a resolved-negative open question, with the consequence for the design: instance reconciliation must read the installed .mpk directly the way augmentFromMPK does, not via the definition registry. Data Widgets 3.4 stayed at 0 CE0463 throughout, so the regression was specifically in routing, not in the schema path. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- ...PROPOSAL_widget_instance_reconciliation.md | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md index 02691c74a..974f99bf7 100644 --- a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md +++ b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md @@ -236,11 +236,28 @@ produced a confident wrong diagnosis during the #716 investigation. 3. **Should `run --local` / `docker build` sync automatically?** Convenient, but an implicit model mutation during a build is exactly the kind of surprise this repo avoids elsewhere. Recommend explicit only, with the build *reporting* drift. -4. **Interaction with the built-in definition override.** `widget_defs.go` skips any - widget with a hand-crafted def (gallery, dropdownsort, four filters), so those - never see the project's `.mpk`. Reconciliation must not inherit that skip, or it - will no-op on exactly the widgets #716 is about. Fixing the override to be a - fallback is arguably a prerequisite — see the remaining #716 residue. +4. **The built-in definition override is NOT the lever — resolved, negatively.** + `widget_defs.go` skips any widget with a hand-crafted def (gallery, + dropdownsort, four filters), so those never see the project's `.mpk`. It looks + like a prerequisite for this work. It is not, because **`.def.json` carries + routing, not schema**: `GenerateDefJSON` emits `propertyMappings`, + `childSlots` and `objectLists` — which MDL keyword feeds which property key. + The schema CE0463 compares comes from the embedded template plus + `augmentFromMPK`, a different pipeline. + + Measured twice: making extraction generate all 42 defs from the project's + `.mpk` left the fresh-authoring CE0463 count at **6, unchanged**. Attempted + properly (built-in keeping identity and routing, generated def supplying the + rest) it **regressed** `17-custom-widget-examples` — CE7006 "Selected value is + not valid for attribute 'title'" and CE7247 "Move this widget into a data + container" on the `TEXTFILTER`, because the hand-authored routing encodes + behaviour the generator cannot derive (e.g. the `attrChoice="linked"` rule + from #605). Reverted. + + The override should probably still become a fallback for its own sake, but it + is a separate concern with its own risk, and it buys this proposal nothing. + Instance reconciliation must read the installed `.mpk` **directly**, the way + `augmentFromMPK` does — not via the definition registry. 5. **Does this subsume `mx update-widgets` in CI?** If reconciliation is faithful, the doctype harness could drop its "we deliberately do NOT run update-widgets" note. Not a goal, but a good confidence signal if it turns out true. From 016a7a152f257850507ea44e0e199da326e9339d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 00:18:40 +0000 Subject: [PATCH 07/29] docs: template drift does not explain the residual #716 failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hypotheses disproven, both mine. 1. "The embedded template is behind the package, so the most-drifted widgets fail." Measured against Data Widgets 3.10: datagrid ADD 19 REMOVE 1 -> passes gallery ADD 11 REMOVE 0 -> fails (4) datagrid-dropdown-filter ADD 0 REMOVE 0 -> fails (2) datagrid-text-filter ADD 0 REMOVE 0 -> passes Drift predicts nothing in either direction. datagrid has the most churn and is clean; the two filters are byte-for-byte in sync with the package and disagree with each other. 2. "Prune the PropertyType fields the update-widgets reference omits." Takes fresh-authoring CE0463 from 6 to 4 on 3.10 — and the shipped 3.4 from 0 to 139. Those fields are required on the version the project ships with. Not landed; recorded as a trap, because "the reference omits it" is not "never emit it" until tested against the version in use. The four Gallery failures are open. Not the property set, not OnChangeProperty/Required values, not Appearance.DesignProperties, LabelTemplate, the GridSortBar marker, SortDirection/SortOrder, or AttributeRef.EntityRef — each patched in isolation and re-checked, none moved the count. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .../WIDGET_BSON_VERSION_COMPATIBILITY.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md index b7698ee0c..a5e68385c 100644 --- a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md +++ b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md @@ -155,6 +155,39 @@ blank project whose own template widgets are already failing from Case 1, so a raw CE0463 count mixes the two. Subtract by widget *name* against a control project that ran no mxcli command at all. +### The residual #716 failures are NOT explained by template drift + +The obvious model — "the embedded template is behind the installed package, so +widgets whose property set drifted the most fail" — is **wrong**. Measured on Data +Widgets 3.10, comparing each embedded template's `PropertyKey` set against the +installed `.mpk` XML, alongside whether freshly authored instances pass `mx check`: + +| Template | must ADD | must REMOVE | in sync | fresh authoring | +|---|---|---|---|---| +| `datagrid` | 19 | 1 | 60 | **passes** | +| `gallery` | 11 | 0 | 33 | **fails** (4 instances) | +| `datagrid-dropdown-filter` | 0 | 0 | 27 | **fails** (2 instances) | +| `datagrid-text-filter` | 0 | 0 | 13 | **passes** | + +Drift does not predict failure in either direction. `datagrid` has by far the most +churn and is clean; `dropdown-filter` and `text-filter` are byte-for-byte in sync +with the package and disagree with each other. Whatever distinguishes them is in the +*content* of specific properties, not in which properties exist. + +**A field-level prune is also disproven, and dangerously so.** Deleting +`OnChangeProperty` / `Required` from every `CustomWidgets$WidgetValueType` — the +fields `mx update-widgets` omits on Gallery — takes fresh-authoring CE0463 from 6 to +4 on Data Widgets 3.10, but takes the **shipped 3.4 from 0 to 139**. Those fields are +required on the version the project ships with; removing them unconditionally breaks +every widget. Do not treat "the reference output omits it" as "we should never emit +it" without testing the version the project actually uses. + +The cause of the four Gallery failures is **open**. It is not the property set, not +`OnChangeProperty`/`Required` values, not `Appearance.DesignProperties`, +`LabelTemplate`, the `GridSortBar` list marker, `SortDirection`/`SortOrder`, or +`AttributeRef.EntityRef` — each was patched in isolation and re-checked, none moved +the count. + ## Onboarding a new Mendix minor (e.g. 11.10, 12.0) The CE0463 fix methodology used for 11.9 generalizes. Steps: From b1a7ffd339c374be7e5447fa47ceb251551f2ef4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 00:32:45 +0000 Subject: [PATCH 08/29] docs: generic MPK-derived templates exist, don't fix Gallery; the real lead is values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections and a lead, from re-reading the past week's CE0463 fixes. 1. I previously told the user there is no template-free generic path. Wrong. modelsdk/widgets/loader.go:getOrGenerateTemplate falls back to GenerateFromMPK, which builds a complete Type+Object from the project's .mpk with no embedded snapshot. Charts are authored entirely that way (91b054b). Gallery never reaches it only because an embedded template exists and wins. 2. Forcing Gallery down that path on Data Widgets 3.10 (temporary switch, since reverted) demonstrably changed the output — 17 lines of Type diff, with OnChangeProperty moving to the value mx update-widgets produces — and all four galleries still failed CE0463. Together with the SynthesizeNeutralObject spike, two independent generic-construction approaches have now failed on the same widget, which is evidence the missing information is not in the .mpk. 3. The lead I have not tested: every CE0463 fix landed this past week was VALUE-shaped, not schema-shaped, while reporting the widget version — an empty TextTemplate header (3cb8ab6), an empty ClientTemplate where Studio Pro stores null (455c43a), placeholder " " ClientTemplates in object-list items (4ea402c2, which hit Accordion/AreaChart/Maps), an unset String as " " (abba773). The #716 investigation went schema-first and ruled the whole schema axis out; the empty-vs-null-vs-placeholder axis inside Gallery's Object has never been looked at, and that is where four of the last five CE0463 fixes actually were. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .../WIDGET_BSON_VERSION_COMPATIBILITY.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md index a5e68385c..f4724d56c 100644 --- a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md +++ b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md @@ -188,6 +188,47 @@ The cause of the four Gallery failures is **open**. It is not the property set, `AttributeRef.EntityRef` — each was patched in isolation and re-checked, none moved the count. +### There IS a template-free generic path — and it does not fix Gallery either + +`modelsdk/widgets/loader.go:getOrGenerateTemplate` resolves a widget template in +three steps: embedded template, session cache, then **`GenerateFromMPK`** — a +complete Type+Object built from the project's `.mpk` with no embedded snapshot at +all. It is not theoretical: Charts (Pie/Column/Line/Bar/Area) ship no template and +are authored entirely this way (`91b054b`). + +Because step 1 wins whenever an embedded template exists, Gallery never reaches it. +Forcing it to (temporary env switch, since reverted) on Data Widgets 3.10: + +- The output genuinely changed — 17 lines of Type diff, and `OnChangeProperty` + moved from the embedded `"onConfigurationChange"` to `""`, which is what + `mx update-widgets` produces. +- **All four galleries still failed CE0463.** + +So "derive the template from the package instead of the frozen snapshot" is +available today, demonstrably takes effect, and is still not sufficient. Combined +with the `SynthesizeNeutralObject` spike in +[`PROPOSAL_multi_version_pluggable_widgets.md`](../11-proposals/PROPOSAL_multi_version_pluggable_widgets.md), +two independent generic-construction approaches have now failed on the same class +of widget, which is evidence the missing information is genuinely not in the `.mpk`. + +### The untested lead: CE0463 from a VALUE, not a schema + +Every CE0463 fix landed in the past week was value-shaped, not schema-shaped, and +the error message named the widget version in each case: + +| Fix | Cause | +|---|---| +| `3cb8ab6` (ledger #54) | a column header serialized as an **empty** `TextTemplate` where Studio Pro wants the attribute name filled in | +| `455c43a` | a hidden chart-series `markerColor` serialized as an **empty** `Forms$ClientTemplate`; Studio Pro stores **null** | +| `4ea402c2` (#548) | object-list item TextTemplate slots emitting a placeholder `" "` ClientTemplate instead of null — CE0463 on Accordion, AreaChart, Maps | +| `abba773` | an unset chart-series String emitted as `" "` instead of `""` | + +The Gallery investigation for #716 went the other way — Type/schema first — and ruled +out the whole schema axis. The empty-vs-null-vs-placeholder axis inside the Gallery's +`Object` (its content slots, item templates, and the `Forms$ClientTemplate` nodes +underneath) has **not** been examined, and it is where four of the last five CE0463 +fixes actually lived. + ## Onboarding a new Mendix minor (e.g. 11.10, 12.0) The CE0463 fix methodology used for 11.9 generalizes. Steps: From 3b6dc80bac511a901f38ffb2737cbdaea7af808c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 00:48:04 +0000 Subject: [PATCH 09/29] =?UTF-8?q?docs:=20#716=20Gallery=20=E2=80=94=20exha?= =?UTF-8?q?ustive=20elimination,=20still=20unresolved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recording the full ruled-out list so the next attempt does not repeat it. The constraining fact: swapping mxcli's whole galCustomers widget node for the update-widgets reference node clears its CE0463 (35 -> 34). Swapping only Type or only Object CRASHES the load — mx check then prints "0 errors" because it never loads, which is an artifact I nearly reported as a fix. So the cause is inside the widget node and needs Type and Object consistently paired. Ruled out by patch-and-recheck: property-set drift (does not predict failure in either direction); all 16 differing paths applied to the failing widget alone; OnChangeProperty/Required values and a field prune (the prune takes shipped DW 3.4 from 0 to 139); PrimitiveValue below->bottom; GridSortBar marker, SortDirection->SortOrder, AttributeRef.EntityRef; Appearance.DesignProperties; LabelTemplate; pointer integrity; pointer->PropertyKey mapping at all depths in document order; property ordering; BSON key order (mxcli non-alphabetical, reference alphabetical — but tfSearch PASSES with mxcli's order, so it is not the discriminator); GenerateFromMPK; definition-registry precedence. By every measure computable from the decoded BSON the failing widget is identical to a reference that passes. The difference must be in what a Python BSON round-trip normalises — binary field values or an encoding detail below the document model. Next attempt should work at the byte level on the encoded unit, or get a Studio-Pro-authored Gallery on Data Widgets 3.10 as a third reference. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/lsp_completions_gen.go | 1 - .../WIDGET_BSON_VERSION_COMPATIBILITY.md | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 67e583e29..40b294377 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -525,7 +525,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "DRY", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "RUN", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "WIDGETTYPE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, - {Label: "V3", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, // Keyword {Label: "BUSINESS", Kind: protocol.CompletionItemKindKeyword, Detail: "Keyword"}, diff --git a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md index f4724d56c..41eb1405a 100644 --- a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md +++ b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md @@ -229,6 +229,43 @@ out the whole schema axis. The empty-vs-null-vs-placeholder axis inside the Gall underneath) has **not** been examined, and it is where four of the last five CE0463 fixes actually lived. +### #716 Gallery: what is ruled out, and the one fact that constrains the answer + +Investigated exhaustively on Mendix 11.12.2 + Data Widgets 3.10, fixture 31, against +an `mx update-widgets` reference of the same project. **Unresolved** — recorded so the +next attempt starts from the eliminations rather than repeating them. + +**The constraining fact.** Replacing mxcli's whole `galCustomers` widget node with the +reference node clears its CE0463 (35 → 34 errors). Replacing only its `Type`, or only +its `Object`, **crashes the project load** — mx check reports "0 errors" because it +never loads, which is an artifact, not a fix. So the cause is inside the widget node +and requires Type and Object to stay consistently paired. + +**Ruled out, each by patch-and-recheck:** + +| Axis | Method | Result | +|---|---|---| +| Property set drift | template `PropertyKey` set vs `.mpk` XML | does not predict failure — `datagrid` ADD 19/REMOVE 1 passes, `datagrid-dropdown-filter` 0/0 fails | +| All value differences | full path-level diff → **16** differing paths, applied to the failing widget alone | still fails | +| `OnChangeProperty`, `Required` | value sync, then field prune | prune fixes 2 filters but takes shipped DW 3.4 from 0 → **139** | +| `PrimitiveValue` `below`→`bottom` | patched | no change | +| `GridSortBar` marker 3→2, `SortDirection`→`SortOrder`, `AttributeRef.EntityRef` | patched | no change | +| `Appearance.DesignProperties`, `LabelTemplate` | patched | no change | +| Pointer integrity | every `TypePointer` resolves; no orphan `PropertyType` | identical to reference | +| Pointer semantics | each property mapped to the `PropertyKey` it points at, all depths, document order | **identical** to reference | +| Property ordering | Object order vs Type order | identical to reference | +| BSON key order | raw key sequence of the widget node | mxcli is non-alphabetical, reference is — but **`tfSearch` passes with the identical non-alphabetical order**, so key order is not the discriminator | +| Generic MPK-derived template | forced Gallery through `GenerateFromMPK` | output changed (17 Type lines), still fails | +| Definition-registry precedence | built-in as fallback instead of override | no change; regressed `17-custom-widget-examples` | + +**Where that leaves it.** By every measure computable from the decoded BSON — values, +keys, ordering, pointer topology — mxcli's failing Gallery is identical to a reference +that passes. The difference is therefore in something a Python BSON round-trip +normalises: binary field values, or an encoding detail below the document model. The +next attempt should work at the **byte level** (compare the encoded unit ranges +directly) rather than on decoded documents, or obtain a Studio-Pro-authored Gallery on +Data Widgets 3.10 for a third reference point. + ## Onboarding a new Mendix minor (e.g. 11.10, 12.0) The CE0463 fix methodology used for 11.9 generalizes. Steps: From 2abf2bec9df64562a2edef3d1910d680e135eb66 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:57:58 +0000 Subject: [PATCH 10/29] docs: CE0463 diagnosis skill, and correct an overclaim in the widgets guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things learned during #716 that were not written down anywhere. 1. The docs-site guide promised more than holds. It said widgets created by mxcli "open cleanly ... against Marketplace-updated packages, without manual 'Update widgets' fix-ups" and "no mx update-widgets step required". Measured: on the bundled Data Widgets 3.4 that is true (0 CE0463), but upgrading to 3.11.3 flags every stored instance — mxcli-authored and Studio Pro's alike — and freshly authored gallery/dropdownfilter fail on 3.10+. The guide now states both limits with the measurements, and points at #716. 2. The guide's resolution order omitted a step: a widget with no embedded template is generated whole from the package (GenerateFromMPK), which is how Marketplace widgets and Charts are supported. I asserted the opposite twice during this investigation because the page and sdk/widgets/loader.go both suggest the embedded template is mandatory. 3. New skill .claude/skills/diagnose-ce0463.md — the procedure, which is the part that would have saved the most time: - establish which of two bugs you have (package upgraded vs authored fresh), via two controls: do Studio Pro's OWN widgets fail too, and does mx update-widgets clear it - measure against an untouched control, subtracting by widget name - exhaustive path diff BEFORE any hypothesis - patch each difference in isolation, then in combination, on the failing widget only - the traps: a crashed load reports "0 errors" so read the tail not the count; every normalisation (dropping $ID, bytes->'', sorted keys) is a place the answer hides; test any candidate fix against the BUNDLED package too — the field prune that fixes 2 widgets on 3.10 takes 3.4 from 0 to 139 Registered in CLAUDE.md and cross-linked from debug-bson.md. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/debug-bson.md | 5 + .claude/skills/diagnose-ce0463.md | 145 ++++++++++++++++++++++ CLAUDE.md | 1 + docs-site/src/guides/pluggable-widgets.md | 34 ++++- 4 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 .claude/skills/diagnose-ce0463.md diff --git a/.claude/skills/debug-bson.md b/.claude/skills/debug-bson.md index 76bb2f7db..05e821e52 100644 --- a/.claude/skills/debug-bson.md +++ b/.claude/skills/debug-bson.md @@ -1,5 +1,10 @@ # Debug BSON Serialization Issues +> **CE0463 specifically**: start with [`diagnose-ce0463.md`](diagnose-ce0463.md). +> It carries the elimination order, the two controls that separate a package +> upgrade from an mxcli defect, and the measurement traps (a crashed load reports +> "0 errors"; normalised diffs hide the answer). + This skill provides a systematic workflow for debugging BSON serialization errors when programmatically creating Mendix pages and widgets. ## When to Use This Skill diff --git a/.claude/skills/diagnose-ce0463.md b/.claude/skills/diagnose-ce0463.md new file mode 100644 index 000000000..464f59bbc --- /dev/null +++ b/.claude/skills/diagnose-ce0463.md @@ -0,0 +1,145 @@ +# Diagnosing CE0463 "the definition of this widget has changed" + +CE0463 names the *widget version* but is caused by almost anything in the widget's +stored BSON. That mismatch between what the error says and what it means is why this +class of bug burns time. Work in this order — each step is cheap and eliminates a +whole family of causes. + +Related: [`debug-bson.md`](debug-bson.md) for the general BSON diff workflow; +[`WIDGET_BSON_VERSION_COMPATIBILITY.md`](../../docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md) +for what is version-fragile and the per-minor onboarding record. + +## Step 0 — Establish which of two bugs you have + +These look identical in `mx check` output and have unrelated causes. + +| | Case A: package upgraded | Case B: authored fresh and still wrong | +|---|---|---| +| Trigger | widget package changed *after* the widgets were created | widgets created *against the current* package | +| Cause | a stored instance carries a property the new definition dropped | mxcli emits something the package does not accept | +| Owner | **not an mxcli bug** — this is what "Update all widgets" is for | mxcli | + +**Two controls settle it. Neither is optional.** + +1. **Do Studio Pro's own widgets fail too?** A blank project ships `dataGrid2_*`, + `gallery1/2`, `drop_downFilter1/2` on its template pages. If those fail alongside + yours, the tool is not the variable. +2. **Does `mx update-widgets` clear it?** If yes, the BSON was structurally valid and + correct for the version it was written against → Case A. Genuine template bugs do + *not* clear this way (the Image stale default and the number-filter markerless + array both needed template fixes). + +```bash +cp -r proj proj-ref && mx update-widgets proj-ref/App.mpr # ALWAYS on a copy +mx check proj-ref/App.mpr | grep -c CE0463 +``` + +`mx update-widgets` **destroys `mprcontents/`** on MPR v2, collapsing the project to +the v1 single-file layout. Only ever run it on a throwaway copy. + +## Step 1 — Measure against an untouched control + +The doctype fixtures live in a *blank project whose own widgets are already failing* +for Case A reasons. A raw CE0463 count therefore mixes both bugs. + +```bash +# control: same project, same package, mxcli never ran +mx check control/App.mpr | grep CE0463 | sed 's/.*at //' | sort -u > /tmp/base.txt +mx check proj/App.mpr | grep CE0463 | sed 's/.*at //' | sort -u > /tmp/mine.txt +comm -13 /tmp/base.txt /tmp/mine.txt # ← the only failures that are yours +``` + +Skipping this produces a confident wrong answer. It did during #716: counting raw +totals in a blank project attributed template widgets to mxcli and led to +"DataGrid2 is clean" — which a real project immediately disproved. + +## Step 2 — Exhaustive path diff FIRST, before any hypothesis + +Do not pattern-match against previous CE0463 fixes until you know the diff. Flatten +both widget subtrees to `path → value` maps and list every difference. This takes +minutes and usually bounds the problem to a handful of paths. + +```python +def flat(o, path, out): + if isinstance(o, dict): + for k, v in sorted(o.items()): + if k == '$ID': continue # regenerated UUIDs, never meaningful + flat(v, path + '/' + k, out) + elif isinstance(o, list): + for i, v in enumerate(o): flat(v, path + '[%d]' % i, out) + else: + out[path] = '' if isinstance(o, bytes) else o +``` + +Diff **the whole widget node**, not just `Type` and `Object` — `Appearance`, +`LabelTemplate` and the other sibling fields live above them and have differed in +practice. + +## Step 3 — Patch each difference in isolation, then in combination + +Apply one candidate to the stored BSON, re-run `mx check`, revert. A difference that +does not move the count is not the cause, however plausible it looks. Then apply the +survivors together — CE0463 compares the whole widget, so a combination can matter +where no single item does. + +Apply to the **failing widget only**. A project-wide patch changes other widgets and +makes the count unreadable. + +## Step 4 — Bisect by splice when the diff comes up empty + +Replace mxcli's node with the reference node and confirm the error clears; then +narrow. This proves *where* the cause is even when you cannot see *what* it is. + +**Trap that produces a false positive:** swapping only `Type` or only `Object` +desynchronises `TypePointer` and makes the project fail to **load**. `mx check` then +prints `0 errors` because it never got far enough to check anything. Always read the +output tail, never just the error count: + +```bash +mx check proj/App.mpr | tail -3 # "The app contains: N errors." or a stack trace? +``` + +## What normalising hides + +Every convenience in a comparison script is a place the answer can hide. During #716 +the following were all confirmed identical to the reference, and the cause was still +somewhere the tooling flattened: + +| Normalisation | What it masks | +|---|---| +| dropping `$ID` | pointer *identity* — check `TypePointer` → `PropertyKey` mapping separately | +| `bytes → ''` | every binary field value (attribute/entity refs, pointers) | +| `sorted(o.items())` | BSON key order — a documented CE0463 cause (`b1f4de3a`) | +| comparing sets not sequences | property ordering within `PropertyTypes` / `Properties` | + +When a structural comparison says "identical" but the behaviour differs, the answer +is in what you normalised. Go to bytes. + +## Known cause families + +Ordered by how often they have actually been the answer. + +1. **A value, not the schema.** Four of the five CE0463 fixes in the v0.13 cycle were + value-shaped while the error named the widget version: an empty `TextTemplate` + header where Studio Pro wants the attribute name (`3cb8ab6`), an empty + `Forms$ClientTemplate` where Studio Pro stores `null` (`455c43a`), placeholder + `" "` ClientTemplates in object-list items (`4ea402c2`), an unset String as `" "` + (`abba773`). +2. **A stale default in the embedded template** — the Image widget's + `Atlas_Core.Content.Mendix` (`549c44f`). +3. **A markerless empty array** — `"Items": []` instead of `"Items": [3]`. Mendix + ≤ 11.11 tolerates it, 11.12 fails the whole project load. +4. **Property-set drift against the package.** Real, but a weak predictor: `datagrid` + needs 19 additions and 1 removal against Data Widgets 3.10 and passes, while + `datagrid-dropdown-filter` is byte-for-byte in sync and fails. + +## Rules of thumb + +- **Read the tail, not the count.** `0 errors` can mean "did not load". +- **Establish the control before the first measurement**, not after the tenth. +- **A hypothesis that survives only because you have not tested it is not evidence.** + Patch it, measure, and write down the negative result — the elimination list is + worth as much as the fix. +- **Test any candidate fix against the bundled package too.** Pruning the fields the + `update-widgets` reference omits fixes 2 widgets on Data Widgets 3.10 and takes the + bundled 3.4 from **0 → 139**. diff --git a/CLAUDE.md b/CLAUDE.md index e4a512c89..87e7f895f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -616,6 +616,7 @@ Full syntax tables for all MDL statements (microflows, pages, security, navigati - `docs/03-development/PAGE_BSON_SERIALIZATION.md` - Page/widget BSON format, type mappings, required defaults - `docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md` - What's version-resilient vs version-fragile in widget BSON output, and how to onboard a new Mendix minor (e.g. 11.10) - `.claude/skills/debug-bson.md` - Workflow for debugging BSON serialization issues with `mx` tool (includes the "Studio Pro Update Widget" diff methodology that closed CE0463) +- `.claude/skills/diagnose-ce0463.md` - **Read first for any CE0463 report**: the elimination order, the two controls that separate "the user upgraded a widget package" (not our bug) from a real mxcli defect, and the measurement traps that make CE0463 investigations go wrong - `.claude/skills/verify-in-runtime.md` - Proving a fix in a real app in a real browser (`run --local` + Playwright). For symptoms that only exist at render time, where valid-looking BSON and a clean `mx check` prove nothing — see #812 - `cmd/mxcli/lsp.go` - LSP server implementation (hover, definition, diagnostics, completion, symbols) - `cmd/mxcli/init.go` - `mxcli init` command (project initialization + VS Code extension install) diff --git a/docs-site/src/guides/pluggable-widgets.md b/docs-site/src/guides/pluggable-widgets.md index 6b3ce3f86..33775a94b 100644 --- a/docs-site/src/guides/pluggable-widgets.md +++ b/docs-site/src/guides/pluggable-widgets.md @@ -26,11 +26,14 @@ trip CE0463 the moment the project's widget version didn't match. mxcli never hard-codes one widget shape. It reconciles a known-good template against the widget package **installed in your project**, so the definition it writes matches the -version you actually have — no `mx update-widgets` step required. +version you actually have at the moment you create the widget. 1. **Embedded template.** mxcli ships a known-good template for each built-in widget (extracted from Studio Pro). This provides the correct nested BSON structure that is - hard to build from scratch. + hard to build from scratch. A widget with **no** embedded template — anything from + the Marketplace, and the Charts family — is instead generated whole from the + package (`GenerateFromMPK`), so an embedded template is not a prerequisite for + support. 2. **Reconcile against the project `.mpk`.** When you pass a project (`-p`), mxcli finds the widget's `.mpk` in the project's `widgets/` folder, parses its definition, and @@ -49,8 +52,31 @@ version you actually have — no `mx update-widgets` step required. Studio Pro would give it. The result: widgets created by mxcli open cleanly across Mendix 10.x and 11.x with the -bundled widget packages, and against Marketplace-updated packages, without manual -"Update widgets" fix-ups. +widget packages bundled with those releases — verified on Mendix 11.12 with Data +Widgets 3.4: a project authored entirely by mxcli reports **0** CE0463. + +### Two things this does *not* cover + +**Upgrading a widget package does not update widgets you already created.** A package +that drops a property leaves every *stored* instance carrying a property the new +definition no longer has — which is what CE0463 reports, and what its message +("Update all widgets") asks you to fix. This is normal Mendix behaviour and it is not +specific to mxcli: Studio Pro's own widgets are flagged identically. Measured on +Mendix 11.12, upgrading Data Widgets 3.4 → 3.11.3: + +| Project | as authored | after the upgrade | after `mx update-widgets` | +|---|---|---|---| +| a real mxcli-built app | 0 errors | 36 CE0463 (7 mxcli-authored, 29 Studio Pro's own) | 0 errors | + +mxcli has no "Update all widgets" equivalent yet. Until it does, run the update in +Studio Pro. Avoid `mx update-widgets` on an MPR v2 project — it collapses +`mprcontents/` back to the single-file v1 layout. + +**Two widgets do not yet author cleanly against Data Widgets 3.10+.** Freshly created +`gallery` and `dropdownfilter` widgets produce CE0463 on those packages; `datagrid2`, +`textfilter`, `datefilter` and `numberfilter` are clean. Tracked as +[mendixlabs/mxcli#716](https://github.com/mendixlabs/mxcli/issues/716). On the +bundled 3.4 package all of them are clean. > For the internal mechanics (template extraction, BSON cross-references, the augment > pipeline), see [Widget Template System](../internals/widget-templates.md) and From 8607620a0bdbd65e6a95c1ddb695ef45d0af162f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 04:04:27 +0000 Subject: [PATCH 11/29] chore: drop unrelated lsp_completions_gen.go drift from this branch The file is regenerated from MDLLexer.g4 by `make build`; a local regeneration dropped a stale V3 keyword and the diff rode along on an earlier commit. Not related to this branch's work, so it should land (or not) on its own. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/lsp_completions_gen.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 40b294377..67e583e29 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -525,6 +525,7 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "DRY", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "RUN", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "WIDGETTYPE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, + {Label: "V3", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, // Keyword {Label: "BUSINESS", Kind: protocol.CompletionItemKindKeyword, Detail: "Keyword"}, From 2387584d06fcf10ac70ed73b2f85bf823fe9b6a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 05:28:16 +0000 Subject: [PATCH 12/29] =?UTF-8?q?docs:=20#716=20is=20two=20bugs=20?= =?UTF-8?q?=E2=80=94=20augmentation=20breaks=20the=20filters,=20galleries?= =?UTF-8?q?=20are=20just=20stale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authoring the v0.10 fixture three ways on Mendix 11.13 splits the failing set: 3.4 then upgraded fresh on 3.10 fresh, no augment Galleries 4 fail 4 fail 4 fail Drop-down filters 0 fail 2 fail 0 fail Drop-down filters: augmentFromMPK INTRODUCES the fault. Clean against the package the template matches, clean when that project is upgraded, clean on 3.10 with augmentation disabled — and failing only when augmentation runs against the 3.10 mpk. The template needs 0 additions and 0 removals against 3.10, so what augmentation changes is at the attribute level. This is the real mxcli bug in #716 and it is actionable. Galleries: mxcli emits the same 3.4-shaped gallery whatever package is installed — correct on 3.4 (0 errors), stale on 3.10 — failing identically with augmentation, without it, and when merely upgraded. That is the same behaviour as the blank project's own Studio-Pro-authored gallery1/gallery2: Case A, the normal "Update all widgets" situation, fixed by instance reconciliation rather than a template patch. It explains why the earlier elimination pass found nothing — it was hunting an authoring bug that does not exist. Also corrects a claim I made here earlier: no Studio Pro is needed for a reference. A blank project ships Studio-Pro-authored galleries, and authoring the same fixture against two package versions gives the comparison directly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .../WIDGET_BSON_VERSION_COMPATIBILITY.md | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md index 41eb1405a..2908f21e6 100644 --- a/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md +++ b/docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md @@ -229,7 +229,42 @@ out the whole schema axis. The empty-vs-null-vs-placeholder axis inside the Gall underneath) has **not** been examined, and it is where four of the last five CE0463 fixes actually lived. -### #716 Gallery: what is ruled out, and the one fact that constrains the answer +### #716 is TWO bugs, separated by one experiment + +The failing set — 4 galleries + 2 drop-down filters on Data Widgets 3.10 — is not one +defect. Authoring the same fixture two ways on Mendix 11.13 splits it cleanly: + +| | authored on bundled 3.4, then package upgraded to 3.10 | authored fresh on 3.10 | fresh on 3.10, `augmentFromMPK` disabled | +|---|---|---|---| +| Galleries | **4 fail** | 4 fail | **4 fail** | +| Drop-down filters | **0 fail** | **2 fail** | **0 fail** | + +**Drop-down filters: `augmentFromMPK` introduces the fault.** They are clean when +authored against the package the template matches (3.4), clean when that project is +upgraded, and clean on 3.10 with augmentation switched off — but fail when +augmentation runs against the 3.10 `.mpk`. Augmentation is *making them worse*. This +is a real mxcli bug and the actionable half of #716. Note the template needs **0 +additions and 0 removals** against 3.10, so whatever augmentation changes is at the +attribute level, not the property set. + +**Galleries: the embedded template is simply 3.4-shaped.** They fail identically with +augmentation, without it, and when authored on 3.4 and merely upgraded — the same +behaviour as the blank project's own Studio-Pro-authored `gallery1`/`gallery2`. mxcli +emits the same gallery whatever package is installed: correct on 3.4 (0 errors), +stale on 3.10. That is **Case A**, the normal "Update all widgets" situation, not an +authoring defect — and it is fixed by instance reconciliation +([`PROPOSAL_widget_instance_reconciliation.md`](../11-proposals/PROPOSAL_widget_instance_reconciliation.md)), +not by patching the template. + +This also explains why the earlier elimination pass found nothing: it was hunting an +authoring bug in the gallery, and there isn't one. + +**Method note.** No Studio Pro required — a blank project ships Studio-Pro-authored +`gallery1`/`gallery2`, and authoring the same fixture against two package versions +gives the comparison. An earlier note here claiming a Studio Pro reference was needed +was wrong. + +### #716 Gallery: what was ruled out while hunting the wrong bug Investigated exhaustively on Mendix 11.12.2 + Data Widgets 3.10, fixture 31, against an `mx update-widgets` reference of the same project. **Unresolved** — recorded so the From 0ee37802a23b18ef30117de46605c2c3027c0974 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 06:00:06 +0000 Subject: [PATCH 13/29] =?UTF-8?q?fix(widgets):=20syncDefinitionAttrs=20wro?= =?UTF-8?q?te=20to=20the=20wrong=20node=20=E2=80=94=20closes=20the=20#716?= =?UTF-8?q?=20galleries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pass added earlier reconciled Required/OnChangeProperty from the installed .mpk onto the CustomWidgets$WidgetPropertyType. Those fields live one level down, on its ValueType. Combined with the "only update a key that already exists" guard, that made the whole pass a silent no-op — which is why two rounds of measurement showed it changing nothing and I concluded the hypothesis was wrong rather than that the code never ran. Found by diffing a failing widget against `mx update-widgets` output at the PATH level: the differing paths were Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty, and the path named the node to write. Fresh authoring on Data Widgets 3.10, v0.10 fixture: before: 6 CE0463 (4 galleries + 2 dropdown filters) after: 2 CE0463 (dropdown filters only) All four galleries now clean. Bundled Data Widgets 3.4 stays at 0 CE0463 on Mendix 11.13, unit suite green, doctype integration suite green (494s). Still open: the two dropdown filters. Their residual diff against update-widgets is ValueType/AllowUpload, absent from the modelsdk template copy and present in sdk's — the two engines' template sets have diverged in all 29 files. Syncing them does NOT clear the filters, so something further is involved; the divergence is recorded but not addressed here. Repro mdl-examples/bug-tests/716-widget-package-upgrade.mdl; symptom row added with the generalisable lesson (a guarded update aimed at the wrong node fails invisibly — verify a change executed before concluding it didn't help). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/fix-issue.md | 1 + .../bug-tests/716-widget-package-upgrade.mdl | 49 +++++++++++++++++++ modelsdk/widgets/augment.go | 26 ++++++---- sdk/widgets/augment.go | 26 ++++++---- 4 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 mdl-examples/bug-tests/716-widget-package-upgrade.mdl diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 5379942e9..77cb2aecd 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -322,6 +322,7 @@ cases for these three BSON types — they fell to `default: return nil`. | `alter settings model JavaVersion = 'Java21'` on Mendix 11.12+ produces a project mxbuild refuses to **load**: `mx check` reports `System.ArgumentOutOfRangeException ... (Parameter 'majorVersion is an unsupported value: Java21')` at `JavaVersionExtensions.fromString`. Every check downstream of the settings unit is lost with it | Mendix renamed the property between 11.6 (`JavaVersion` = `"Java21"`) and 11.12 (`JavaMajorVersion` = `"21"`) — and the rename changed the **value format** as well as the key. The #759 fix followed only the key, writing the caller's value through verbatim, so the 11.6 spelling landed in the 11.12 key | `mdl/settingsoverlay/settingsoverlay.go` (`JavaVersionValue`, `SetJavaVersion`) — shared by both engines; the dead third copy in `modelsdk/mpr/serialize_services.go` carried it too | Render the value in the dialect the stored key expects: strip/add the `Java` prefix per key, and pass an unrecognisable value through untouched so a typo surfaces as a Mendix error instead of a mangled setting. **Generalisable**: a renamed property is not only a renamed key — check whether the value encoding moved with it, and cover *both* directions (either spelling in, document's dialect out). Note the sharper failure mode: the original #759 shape was an unknown property, which mxbuild **tolerates**, so only Studio Pro broke; a wrong *value* for a known enum is a hard build failure, which is why this one surfaced as a red nightly rather than a user report. Repro `mdl-examples/bug-tests/759-java-version-value-dialect.mdl`. Issue #759 (follow-up) | | On **Mendix 11.13 only**, every microflow using `EXECUTE DATABASE QUERY` fails `mx check` with **CE5277** "Please re-run and save the query to fix the error", once per activity. The queries themselves report nothing — the error lands on the *activities* pointing at them, so it reads like a microflow defect. Both engines. 11.12 and below are clean | 11.13 replaced the integer `QueryType` (1 = custom SQL) on `DatabaseConnector$DatabaseQuery` with a `Type` **string enum** (`Select` / `NonSelect` / `Unknown`), shipping a one-time conversion (`ExternalDatabaseConnectionQueryTypeConversion`) for old documents. mxcli wrote the legacy integer unconditionally, so on 11.13 the new property was simply **absent** — and an absent `Type` reads as Unknown, which is exactly what CE5277 reports | `mdl/dbconnector/querytype.go` (new, shared by both engines), `sdk/mpr/writer_dbconnection.go` + `parser_dbconnection.go`, `mdl/backend/modelsdk/db_write.go` + `integration_read.go`, `model/types.go` (`DatabaseQuery.QueryTypeName`) | Branch on the project's Mendix version (`ProjectVersion().IsAtLeast(11, 13)`) and write **exactly one** spelling. Writing both is not a safe hedge — a property the target's metamodel does not define is the #759 Studio-Pro-won't-open shape. Read side must accept either, or the next ALTER of an 11.13 project writes Unknown straight back. mxcli can't derive the type the way Studio Pro does (running the query and inspecting the result set), so it reads the leading SQL keyword — still better than Mendix's own converter, which marks every migrated query `Select` regardless of statement. **Diagnosis method**: `mx convert -p -s ` with the NEW mxbuild runs the version's own migration, then diff the BSON — that is what showed `QueryType: 1` → `Type: "Select"` without guessing. **Generalisable**: onboarding a new Mendix minor is not just adding it to the nightly matrix — run the doctype corpus against it first (`MX_BINARY=~/.mxcli/mxbuild//modeler/mx go test -tags integration -run TestMxCheck_DoctypeScripts`), because a renamed property surfaces as a red matrix job, not a compile error. Repro `mdl-examples/bug-tests/1113-database-query-type-enum.mdl`. Sibling drift found in the same sweep and deliberately NOT fixed: **CE5278** ("The JDBC driver is missing from the module settings"), a new 11.13 check about the module's Java dependencies, which mxcli has no way to author | | A user reports CE0463 "the definition of this widget has changed" on DataGrid2 / Gallery / filters after upgrading the **Data Widgets** marketplace module. Reads like template drift; on real projects it is usually **not an mxcli bug at all** | A widget package that DROPS a property leaves every *stored* instance carrying a property the new definition lacks — which is precisely what CE0463 reports, and what its own message ("Update all widgets") tells you to fix. Ledger on 11.12: 0 errors at Data Widgets 3.4 (as authored) → 36 CE0463 at 3.11.3 → **0 again after `mx update-widgets`**. Single cause: `key="advanced"` is in `Datagrid.xml` at 3.4 and gone at 3.10/3.11 | no code change — diagnostic. See `docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md` "CE0463 after a widget-package upgrade" | **Two controls settle it, neither optional.** (1) Do **Studio Pro's own** widgets fail too? A blank project's `dataGrid2_*`/`gallery1,2`/`drop_downFilter1,2` are Mendix-authored — if they fail alongside mxcli's, the tool is not the variable (29 of Ledger's 36 were these). (2) Does **`mx update-widgets` clear it**? If yes, mxcli's BSON was structurally valid and correct for the version it was written against; genuine template bugs do NOT clear this way (the Image stale default and the number-filter markerless array both needed template fixes). **The real mxcli defect is the residue after those controls**: author FRESH against the new package (`widget init` + author + `mx check`) — on 3.10/3.11 that leaves DataGrid2 **clean** and only Gallery + DatagridDropdownFilter failing, i.e. far narrower than the issue as filed. **Trap that cost a full round-trip**: measuring with the doctype fixtures alone mixes both cases, because their pages live in a blank project whose own template widgets are already failing — subtract by widget NAME against a control project that ran no mxcli command. Issue #716 | +| Freshly authored **Gallery** widgets fail `mx check` with **CE0463** on Data Widgets 3.10+, while the 3.4 package bundled with Mendix 11.12/11.13 is clean — so it looks like ordinary post-upgrade staleness. Every schema-level explanation is disproven (property sets, list markers, ordering, pointer topology, `GenerateFromMPK`) | `syncDefinitionAttrs` reconciled a surviving property's definition attributes (`Required`, `OnChangeProperty`) from the installed `.mpk` onto the `CustomWidgets$WidgetPropertyType` node — but they live one level down, on its **`ValueType`**. The "only update a key that already exists" guard then never fired, making the entire pass a **silent no-op**, so the Gallery kept the embedded template's `OnChangeProperty = "onConfigurationChange"` where 3.10 expects `""` | `sdk/widgets/augment.go` + `modelsdk/widgets/augment.go` (`syncDefinitionAttrs`) | Descend to `ValueType` before the update (`getMapField(ptMap, "ValueType")`, falling back to the PropertyType). **Diagnosis method that found it**: diff against `mx update-widgets` output at the PATH level — the differing paths were `Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty`, and the path told me which node to write. **Generalisable — the trap that cost the most here**: a guarded update (`if _, ok := m[k]; ok`) aimed at the wrong node is *invisible*. It cannot fail loudly, so measurements read as "the fix didn't help" rather than "the fix never ran". When a change measurably does nothing, verify it executed before concluding the hypothesis was wrong. Result: fresh-authoring CE0463 on 3.10 went 6 → 2; bundled 3.4 stayed at 0. **Still open**: the two datagrid dropdown filters, whose residual diff is `ValueType/AllowUpload` (absent from the modelsdk template copy, present in sdk's — the engines' template sets have diverged); syncing them does not clear it. Repro `mdl-examples/bug-tests/716-widget-package-upgrade.mdl`. Issue #716 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/mdl-examples/bug-tests/716-widget-package-upgrade.mdl b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl new file mode 100644 index 000000000..cc46945ed --- /dev/null +++ b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl @@ -0,0 +1,49 @@ +-- Bug #716 (gallery half): CE0463 on freshly authored Gallery widgets +-- +-- Symptom: on a project with Data Widgets 3.10+ installed, every Gallery mxcli +-- authors fails `mx check` with CE0463 "The definition of this widget has +-- changed". On the 3.4 package bundled with Mendix 11.12/11.13 the same script +-- is clean, which is what made this look like ordinary staleness. +-- +-- Cause: `syncDefinitionAttrs` reconciled a surviving property's definition +-- attributes (Required, OnChangeProperty) from the installed .mpk — but wrote +-- them onto the CustomWidgets$WidgetPropertyType node. They live one level +-- down, on its ValueType. The guard "only update a key that already exists" +-- then never fired, so the whole pass was a silent no-op and the Gallery kept +-- the embedded template's OnChangeProperty = "onConfigurationChange" where +-- 3.10 expects "". +-- +-- Found by diffing against `mx update-widgets` output: the differing paths were +-- Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty, not +-- PropertyTypes[N]/OnChangeProperty. +-- +-- Verify on a project with Data Widgets 3.10+ installed: +-- mxcli widget init -p app.mpr +-- mxcli exec mdl-examples/doctype-tests/31-pluggable-datagrid-gallery-v010-examples.mdl -p app.mpr +-- mx check app.mpr -> no CE0463 on any Gallery +-- +-- STILL OPEN: the two DataGrid dropdown filters (ddfStatus / ddfActive) in that +-- fixture remain CE0463 on 3.10. Their residual diff against update-widgets is +-- ValueType/AllowUpload, absent in the modelsdk template copy (present in sdk's +-- — the two engines' template sets have diverged). Syncing the templates does +-- NOT clear it, so something further is involved. + +create module W716; +create persistent entity W716.Customer ( Name: string(100), City: string(100) ); + +create page W716.Overview +( + title: 'Customers', + layout: Atlas_Core.Atlas_Default +) +{ + gallery galCustomers ( + DataSource: database from W716.Customer sort by Name asc + ) { + template tpl1 { + container ctnCard (Class: 'card') { + dynamictext txtName (Content: '{1}', ContentParams: [{1} = Name]) + } + } + } +} diff --git a/modelsdk/widgets/augment.go b/modelsdk/widgets/augment.go index 0a42a7395..c1a61ce3e 100644 --- a/modelsdk/widgets/augment.go +++ b/modelsdk/widgets/augment.go @@ -1262,16 +1262,24 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } if key, _ := ptMap["PropertyKey"].(string); key != "" { if p := byKey[key]; p != nil { - // Update in place only. Adding a key the PropertyType does not - // already carry invents a property this Mendix version may not - // define — the mendixlabs/mxcli#759 failure shape, and the - // opposite of what `mx update-widgets` produces (its Gallery - // PropertyTypes omit both keys entirely). - if _, ok := ptMap["Required"]; ok { - ptMap["Required"] = p.Required + // These live on the PropertyType's ValueType, not on the + // PropertyType itself — targeting the wrong node made this a + // silent no-op. Verified against `mx update-widgets` output: + // the differing paths are + // PropertyTypes[N]/ValueType/{Required,OnChangeProperty}. + // + // Update in place only. Adding a key the node does not already + // carry invents a property this Mendix version may not define — + // the mendixlabs/mxcli#759 failure shape. + target := ptMap + if vt, ok := getMapField(ptMap, "ValueType"); ok { + target = vt } - if _, ok := ptMap["OnChangeProperty"]; ok { - ptMap["OnChangeProperty"] = p.OnChange + if _, ok := target["Required"]; ok { + target["Required"] = p.Required + } + if _, ok := target["OnChangeProperty"]; ok { + target["OnChangeProperty"] = p.OnChange } } } diff --git a/sdk/widgets/augment.go b/sdk/widgets/augment.go index a779256d3..6ed3a055a 100644 --- a/sdk/widgets/augment.go +++ b/sdk/widgets/augment.go @@ -1055,16 +1055,24 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } if key, _ := ptMap["PropertyKey"].(string); key != "" { if p := byKey[key]; p != nil { - // Update in place only. Adding a key the PropertyType does not - // already carry invents a property this Mendix version may not - // define — the mendixlabs/mxcli#759 failure shape, and the - // opposite of what `mx update-widgets` produces (its Gallery - // PropertyTypes omit both keys entirely). - if _, ok := ptMap["Required"]; ok { - ptMap["Required"] = p.Required + // These live on the PropertyType's ValueType, not on the + // PropertyType itself — targeting the wrong node made this a + // silent no-op. Verified against `mx update-widgets` output: + // the differing paths are + // PropertyTypes[N]/ValueType/{Required,OnChangeProperty}. + // + // Update in place only. Adding a key the node does not already + // carry invents a property this Mendix version may not define — + // the mendixlabs/mxcli#759 failure shape. + target := ptMap + if vt, ok := getMapField(ptMap, "ValueType"); ok { + target = vt } - if _, ok := ptMap["OnChangeProperty"]; ok { - ptMap["OnChangeProperty"] = p.OnChange + if _, ok := target["Required"]; ok { + target["Required"] = p.Required + } + if _, ok := target["OnChangeProperty"]; ok { + target["OnChangeProperty"] = p.OnChange } } } From 509c51c4949062761cb91459e35ad5570452f031 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 08:32:56 +0000 Subject: [PATCH 14/29] fix(widgets): absent `required=` in widget XML means true, not false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on PR 85 failed TestMxCheck_DataGridPage and TestMxCheck_DataGridNoColumns with CE0463 on Data grid 2 — a regression from the previous commit, and a real defect that commit merely made visible. The Mendix pluggable-widget XML schema defaults `required` to true; only an explicit required="false" is optional. DataGrid2 3.4 omits the attribute on 24 of its 40 properties. sdk/widgets/mpk read a missing attribute as false, so once syncDefinitionAttrs started actually executing (previous commit, the ValueType descent) it overwrote 24 correct `true`s with `false` on every authored Data grid 2. modelsdk/widgets/mpk has read it correctly since #600 — that is why only the legacy engine failed, and why the same script was clean under the default engine. The two parsers are parallel copies and had silently diverged; this brings sdk into line. Measured against the untouched control, fixture 31 on Mendix 11.13: engine package baseline ValueType only both fixes modelsdk DW 3.10 6 2 2 modelsdk DW 3.4 0 0 0 legacy DW 3.10 11 11 11 legacy DW 3.4 0 9 0 The legacy engine's 11 on DW 3.10 predates both commits and is untouched by them; recorded in the fixture, not addressed here. Causation established both ways: the two integration tests fail on the parent commit and pass with this one, and the new unit test fails when the parser is reverted to `== "true"`. Tests: sdk/widgets/mpk/required_default_test.go (absent / explicit true / explicit false, top-level and nested); TestMxCheck_DataGrid{Page,NoColumns} green on 11.12.2. Symptom row added, with the two traps this cost — a latent divergence in a parallel parser copy, and an out-of-disk run whose exec created no widgets and so scored a misleading clean `mx check`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/diagnose-ce0463.md | 12 ++ .claude/skills/fix-issue.md | 1 + .../bug-tests/716-widget-package-upgrade.mdl | 19 +++ sdk/widgets/mpk/mpk.go | 39 +++--- sdk/widgets/mpk/required_default_test.go | 125 ++++++++++++++++++ 5 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 sdk/widgets/mpk/required_default_test.go diff --git a/.claude/skills/diagnose-ce0463.md b/.claude/skills/diagnose-ce0463.md index 464f59bbc..cdfdbc6f9 100644 --- a/.claude/skills/diagnose-ce0463.md +++ b/.claude/skills/diagnose-ce0463.md @@ -132,10 +132,22 @@ Ordered by how often they have actually been the answer. 4. **Property-set drift against the package.** Real, but a weak predictor: `datagrid` needs 19 additions and 1 removal against Data Widgets 3.10 and passes, while `datagrid-dropdown-filter` is byte-for-byte in sync and fails. +5. **A mis-defaulted definition attribute in the `.mpk` parser.** The widget XML + schema defaults `required` to **true**; reading a missing attribute as `false` + put the wrong `Required` on 24 of DataGrid2 3.4's 40 properties. Two things make + this family easy to miss: the value is only wrong for properties that *omit* the + attribute (so the packages that spell it out look fine), and `sdk/widgets/mpk` + and `modelsdk/widgets/mpk` are parallel copies — a fix in one leaves the other + latent until something starts consuming the value. Grep the sibling package + before concluding a defect is engine-specific. ## Rules of thumb - **Read the tail, not the count.** `0 errors` can mean "did not load". +- **Assert the artifact exists before trusting a zero.** A run that exhausted disk + created no widgets at all and scored a clean `mx check` — read as "the pre-fix + binary is fine" and cost a wrong conclusion. `mxcli -p -c 'show widgets' + | grep ` before every measurement. - **Establish the control before the first measurement**, not after the tenth. - **A hypothesis that survives only because you have not tested it is not evidence.** Patch it, measure, and write down the negative result — the elimination list is diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 77cb2aecd..4347dbb68 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -323,6 +323,7 @@ cases for these three BSON types — they fell to `default: return nil`. | On **Mendix 11.13 only**, every microflow using `EXECUTE DATABASE QUERY` fails `mx check` with **CE5277** "Please re-run and save the query to fix the error", once per activity. The queries themselves report nothing — the error lands on the *activities* pointing at them, so it reads like a microflow defect. Both engines. 11.12 and below are clean | 11.13 replaced the integer `QueryType` (1 = custom SQL) on `DatabaseConnector$DatabaseQuery` with a `Type` **string enum** (`Select` / `NonSelect` / `Unknown`), shipping a one-time conversion (`ExternalDatabaseConnectionQueryTypeConversion`) for old documents. mxcli wrote the legacy integer unconditionally, so on 11.13 the new property was simply **absent** — and an absent `Type` reads as Unknown, which is exactly what CE5277 reports | `mdl/dbconnector/querytype.go` (new, shared by both engines), `sdk/mpr/writer_dbconnection.go` + `parser_dbconnection.go`, `mdl/backend/modelsdk/db_write.go` + `integration_read.go`, `model/types.go` (`DatabaseQuery.QueryTypeName`) | Branch on the project's Mendix version (`ProjectVersion().IsAtLeast(11, 13)`) and write **exactly one** spelling. Writing both is not a safe hedge — a property the target's metamodel does not define is the #759 Studio-Pro-won't-open shape. Read side must accept either, or the next ALTER of an 11.13 project writes Unknown straight back. mxcli can't derive the type the way Studio Pro does (running the query and inspecting the result set), so it reads the leading SQL keyword — still better than Mendix's own converter, which marks every migrated query `Select` regardless of statement. **Diagnosis method**: `mx convert -p -s ` with the NEW mxbuild runs the version's own migration, then diff the BSON — that is what showed `QueryType: 1` → `Type: "Select"` without guessing. **Generalisable**: onboarding a new Mendix minor is not just adding it to the nightly matrix — run the doctype corpus against it first (`MX_BINARY=~/.mxcli/mxbuild//modeler/mx go test -tags integration -run TestMxCheck_DoctypeScripts`), because a renamed property surfaces as a red matrix job, not a compile error. Repro `mdl-examples/bug-tests/1113-database-query-type-enum.mdl`. Sibling drift found in the same sweep and deliberately NOT fixed: **CE5278** ("The JDBC driver is missing from the module settings"), a new 11.13 check about the module's Java dependencies, which mxcli has no way to author | | A user reports CE0463 "the definition of this widget has changed" on DataGrid2 / Gallery / filters after upgrading the **Data Widgets** marketplace module. Reads like template drift; on real projects it is usually **not an mxcli bug at all** | A widget package that DROPS a property leaves every *stored* instance carrying a property the new definition lacks — which is precisely what CE0463 reports, and what its own message ("Update all widgets") tells you to fix. Ledger on 11.12: 0 errors at Data Widgets 3.4 (as authored) → 36 CE0463 at 3.11.3 → **0 again after `mx update-widgets`**. Single cause: `key="advanced"` is in `Datagrid.xml` at 3.4 and gone at 3.10/3.11 | no code change — diagnostic. See `docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md` "CE0463 after a widget-package upgrade" | **Two controls settle it, neither optional.** (1) Do **Studio Pro's own** widgets fail too? A blank project's `dataGrid2_*`/`gallery1,2`/`drop_downFilter1,2` are Mendix-authored — if they fail alongside mxcli's, the tool is not the variable (29 of Ledger's 36 were these). (2) Does **`mx update-widgets` clear it**? If yes, mxcli's BSON was structurally valid and correct for the version it was written against; genuine template bugs do NOT clear this way (the Image stale default and the number-filter markerless array both needed template fixes). **The real mxcli defect is the residue after those controls**: author FRESH against the new package (`widget init` + author + `mx check`) — on 3.10/3.11 that leaves DataGrid2 **clean** and only Gallery + DatagridDropdownFilter failing, i.e. far narrower than the issue as filed. **Trap that cost a full round-trip**: measuring with the doctype fixtures alone mixes both cases, because their pages live in a blank project whose own template widgets are already failing — subtract by widget NAME against a control project that ran no mxcli command. Issue #716 | | Freshly authored **Gallery** widgets fail `mx check` with **CE0463** on Data Widgets 3.10+, while the 3.4 package bundled with Mendix 11.12/11.13 is clean — so it looks like ordinary post-upgrade staleness. Every schema-level explanation is disproven (property sets, list markers, ordering, pointer topology, `GenerateFromMPK`) | `syncDefinitionAttrs` reconciled a surviving property's definition attributes (`Required`, `OnChangeProperty`) from the installed `.mpk` onto the `CustomWidgets$WidgetPropertyType` node — but they live one level down, on its **`ValueType`**. The "only update a key that already exists" guard then never fired, making the entire pass a **silent no-op**, so the Gallery kept the embedded template's `OnChangeProperty = "onConfigurationChange"` where 3.10 expects `""` | `sdk/widgets/augment.go` + `modelsdk/widgets/augment.go` (`syncDefinitionAttrs`) | Descend to `ValueType` before the update (`getMapField(ptMap, "ValueType")`, falling back to the PropertyType). **Diagnosis method that found it**: diff against `mx update-widgets` output at the PATH level — the differing paths were `Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty`, and the path told me which node to write. **Generalisable — the trap that cost the most here**: a guarded update (`if _, ok := m[k]; ok`) aimed at the wrong node is *invisible*. It cannot fail loudly, so measurements read as "the fix didn't help" rather than "the fix never ran". When a change measurably does nothing, verify it executed before concluding the hypothesis was wrong. Result: fresh-authoring CE0463 on 3.10 went 6 → 2; bundled 3.4 stayed at 0. **Still open**: the two datagrid dropdown filters, whose residual diff is `ValueType/AllowUpload` (absent from the modelsdk template copy, present in sdk's — the engines' template sets have diverged); syncing them does not clear it. Repro `mdl-examples/bug-tests/716-widget-package-upgrade.mdl`. Issue #716 | +| Every authored **Data grid 2** fails `mx check` with **CE0463** on the *bundled* Data Widgets 3.4 (Mendix 11.12/11.13) — but only on the **legacy** engine; `modelsdk` is clean on the same script. Appeared the moment `syncDefinitionAttrs` was corrected to write to `ValueType` (the row above), i.e. the moment that pass first actually executed | The Mendix pluggable-widget XML schema defaults `required` to **true**; only an explicit `required="false"` is optional. `sdk/widgets/mpk` read a missing attribute as `false` (`p.Required == "true"`), so the now-live sync overwrote 24 correct `true`s with `false` on DataGrid2 3.4 (which omits `required=` on 24 of its 40 properties). `modelsdk/widgets/mpk` already read it correctly (`p.Required != "false"`, fixed under #600) — the engines had silently diverged on the default | `sdk/widgets/mpk/mpk.go` (all three `PropertyDef` construction sites: `walkPropertyGroup` top-level + nested-direct, `collectNestedProperties`) | Read `p.Required != "false"` so absent means true, matching `modelsdk` and `mx update-widgets`. **Generalisable — the shape to look for**: when two engines carry parallel copies of a parser, a fix applied to one leaves a *latent* divergence in the other that stays invisible until some unrelated change starts consuming the value. Grep the sibling package for the same expression before assuming a defect is engine-specific. **Diagnosis trap hit here**: an A/B ran out of disk mid-run, the exec silently created no widgets, and `mx check` reported 0 CE0463 — reading as "the pre-fix binary is clean". Always assert the artifact exists (`show widgets | grep `) before trusting a zero. Unit repro `sdk/widgets/mpk/required_default_test.go`; integration `TestMxCheck_DataGridPage`/`TestMxCheck_DataGridNoColumns`. Issue #716 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/mdl-examples/bug-tests/716-widget-package-upgrade.mdl b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl index cc46945ed..cda228528 100644 --- a/mdl-examples/bug-tests/716-widget-package-upgrade.mdl +++ b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl @@ -17,16 +17,35 @@ -- Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty, not -- PropertyTypes[N]/OnChangeProperty. -- +-- Second half, exposed by the first: with the pass finally executing, every +-- authored Data grid 2 went CE0463 on the *bundled* Data Widgets 3.4 — on the +-- legacy engine only. The Mendix widget XML schema defaults `required` to true; +-- `sdk/widgets/mpk` read a missing attribute as false, so the sync overwrote 24 +-- correct `true`s with `false` on DataGrid2 3.4. `modelsdk/widgets/mpk` already +-- read it correctly (#600) — the two parsers had silently diverged. +-- -- Verify on a project with Data Widgets 3.10+ installed: -- mxcli widget init -p app.mpr -- mxcli exec mdl-examples/doctype-tests/31-pluggable-datagrid-gallery-v010-examples.mdl -p app.mpr -- mx check app.mpr -> no CE0463 on any Gallery -- +-- Measured, fixture 31 minus the untouched control (Mendix 11.13): +-- +-- engine package baseline with both fixes +-- modelsdk DW 3.10 6 2 +-- modelsdk DW 3.4 0 0 +-- legacy DW 3.10 11 11 (pre-existing, untouched) +-- legacy DW 3.4 0 0 (9 with the ValueType fix alone) +-- -- STILL OPEN: the two DataGrid dropdown filters (ddfStatus / ddfActive) in that -- fixture remain CE0463 on 3.10. Their residual diff against update-widgets is -- ValueType/AllowUpload, absent in the modelsdk template copy (present in sdk's -- — the two engines' template sets have diverged). Syncing the templates does -- NOT clear it, so something further is involved. +-- +-- ALSO STILL OPEN: the legacy engine's 11 CE0463 on Data Widgets 3.10. That +-- predates both fixes and is unchanged by them; the modelsdk engine (the +-- default) is the one this issue tracks. create module W716; create persistent entity W716.Customer ( Name: string(100), City: string(100) ); diff --git a/sdk/widgets/mpk/mpk.go b/sdk/widgets/mpk/mpk.go index c1f1e50fc..430f46df1 100644 --- a/sdk/widgets/mpk/mpk.go +++ b/sdk/widgets/mpk/mpk.go @@ -274,12 +274,17 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini // Collect regular properties for _, p := range pg.Properties { prop := PropertyDef{ - Key: p.Key, - Type: p.Type, - Caption: p.Caption, - Description: p.Description, - Category: category, - Required: p.Required == "true", + Key: p.Key, + Type: p.Type, + Caption: p.Caption, + Description: p.Description, + Category: category, + // Mendix pluggable-widget spec: `required` defaults to true when the + // attribute is absent. Only an explicit required="false" is optional. + // Defaulting missing→false made syncDefinitionAttrs flip 24 correct + // `true`s to `false` on DataGrid2, producing CE0463 (mendixlabs/mxcli#716). + // modelsdk/widgets/mpk already reads it this way (issue #600). + Required: p.Required != "false", OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", @@ -299,11 +304,12 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini } for _, np := range p.NestedDirectProps { prop.Children = append(prop.Children, PropertyDef{ - Key: np.Key, - Type: np.Type, - Caption: np.Caption, - Description: np.Description, - Required: np.Required == "true", + Key: np.Key, + Type: np.Type, + Caption: np.Caption, + Description: np.Description, + // Absent `required` means true — see the note above. + Required: np.Required != "false", OnChange: np.OnChange, DefaultValue: np.DefaultValue, IsList: np.IsList == "true", @@ -336,11 +342,12 @@ func walkPropertyGroup(pg xmlPropGroup, parentCategory string, def *WidgetDefini func collectNestedProperties(pg xmlPropGroup, parent *PropertyDef) { for _, p := range pg.Properties { child := PropertyDef{ - Key: p.Key, - Type: p.Type, - Caption: p.Caption, - Description: p.Description, - Required: p.Required == "true", + Key: p.Key, + Type: p.Type, + Caption: p.Caption, + Description: p.Description, + // Absent `required` means true — see the note in walkPropertyGroup. + Required: p.Required != "false", OnChange: p.OnChange, DefaultValue: p.DefaultValue, IsList: p.IsList == "true", diff --git a/sdk/widgets/mpk/required_default_test.go b/sdk/widgets/mpk/required_default_test.go new file mode 100644 index 000000000..3d1fad593 --- /dev/null +++ b/sdk/widgets/mpk/required_default_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpk + +import ( + "archive/zip" + "os" + "path/filepath" + "testing" +) + +// The Mendix pluggable-widget XML schema defaults `required` to true; only an +// explicit required="false" makes a property optional. Most properties in the +// shipped packages omit the attribute entirely — DataGrid2 3.4 omits it on 24 of +// its 40 properties. +// +// Reading missing→false made syncDefinitionAttrs (augment.go) overwrite 24 +// correct `true`s with `false` on every authored Data grid 2, which Mendix +// reports as CE0463 "the definition of this widget has changed" +// (mendixlabs/mxcli#716). modelsdk/widgets/mpk has read it correctly since #600; +// this is the sdk copy catching up, so the two engines agree. +func TestRequiredDefaultsToTrueWhenAttributeAbsent(t *testing.T) { + path := writeTestMPK(t, ` + + Req + + + + Absent + + + Explicit true + + + Explicit false + + + Parent + + + + Child absent + + + Child false + + + + + + +`) + + ClearCache() + def, err := ParseMPK(path) + if err != nil { + t.Fatalf("ParseMPK: %v", err) + } + + got := map[string]bool{} + var index func([]PropertyDef) + index = func(ps []PropertyDef) { + for _, p := range ps { + got[p.Key] = p.Required + index(p.Children) + } + } + index(def.Properties) + + want := map[string]bool{ + "absent": true, + "explicitTrue": true, + "explicitFalse": false, + "parent": true, + "childAbsent": true, + "childFalse": false, + } + for key, exp := range want { + act, ok := got[key] + if !ok { + t.Errorf("property %q not parsed", key) + continue + } + if act != exp { + t.Errorf("property %q: Required = %v, want %v", key, act, exp) + } + } +} + +// writeTestMPK builds a minimal single-widget .mpk (a ZIP holding package.xml +// plus the widget XML) and returns its path. +func writeTestMPK(t *testing.T, widgetXML string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "test.mpk") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create mpk: %v", err) + } + defer f.Close() + + zw := zip.NewWriter(f) + add := func(name, body string) { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("zip entry %s: %v", name, err) + } + if _, err := w.Write([]byte(body)); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + add("package.xml", ` + + + + +`) + add("Req.xml", widgetXML) + + if err := zw.Close(); err != nil { + t.Fatalf("close zip: %v", err) + } + return path +} From 9b133b6e94bb44b81135faa44b17bb5ed8ac8866 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 09:01:54 +0000 Subject: [PATCH 15/29] fix(widgets): run value reconciliation even when the property set matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Freshly authored drop-down filters were the last CE0463 on Data Widgets 3.10 (mendixlabs/mxcli#716). The visible symptom was a missing ValueType/AllowUpload and a stale Required on refCaption/refCaptionExp, both of which pointed at the embedded template — but replacing the template changed nothing, because AugmentTemplate never ran on this widget at all. Its opening guard ("nothing to add or remove at top level, and no nested children") was written when the function only synced the property SET. Six value-level passes were appended after it later — reconcileEnumValues, reconcilePropertyMetadata, reconcileValueTypesFromMPK, completeValueTypeEnvelope, reorderPropertyTypes, syncDefinitionAttrs — and the early return skips every one of them. Data Widgets 3.10's drop-down filter declares exactly the 25 keys the embedded 11.6-era template already has, so it took that exit every time. DataGrid2 never did, because `columns` has nested children; the hasNestedChildren clause in the guard is a patch around this same bug for one widget. The guard now wraps only the add/remove work. Localised by calling augmentFromMPK directly and counting ValueTypes carrying AllowUpload: 0/25 for the drop-down filter against 44/44 for Gallery. That ruled out every BSON-shape hypothesis in one step. Measured against the untouched control, fixture 31 on Mendix 11.13: engine package baseline ValueType +required +early-return modelsdk DW 3.10 6 2 2 0 modelsdk DW 3.4 0 0 0 0 legacy DW 3.10 11 11 11 11 legacy DW 3.4 0 9 0 0 Still open: the legacy engine on DW 3.10. Its AugmentTemplate carries only syncDefinitionAttrs — the five reconcile passes added to modelsdk under #600 were never ported to sdk/widgets. Recorded in the fixture; modelsdk is the default engine and is now clean on both packages. Tests: TestAugmentTemplate_MatchingKeysStillReconcilesValues in both engines, each mutation-checked (restoring the early return fails both with the reported symptom). Symptom row and CE0463 skill updated with the generalisable shape — an early return placed correctly for a function's original job silently disables everything appended after it, so check that a pass was REACHED before theorising about its input. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/diagnose-ce0463.md | 9 +- .claude/skills/fix-issue.md | 1 + .../bug-tests/716-widget-package-upgrade.mdl | 33 +++--- modelsdk/widgets/augment.go | 110 +++++++++--------- .../widgets/augment_matching_keys_test.go | 96 +++++++++++++++ sdk/widgets/augment.go | 104 +++++++++-------- sdk/widgets/augment_matching_keys_test.go | 84 +++++++++++++ 7 files changed, 320 insertions(+), 117 deletions(-) create mode 100644 modelsdk/widgets/augment_matching_keys_test.go create mode 100644 sdk/widgets/augment_matching_keys_test.go diff --git a/.claude/skills/diagnose-ce0463.md b/.claude/skills/diagnose-ce0463.md index cdfdbc6f9..76a396a14 100644 --- a/.claude/skills/diagnose-ce0463.md +++ b/.claude/skills/diagnose-ce0463.md @@ -132,7 +132,14 @@ Ordered by how often they have actually been the answer. 4. **Property-set drift against the package.** Real, but a weak predictor: `datagrid` needs 19 additions and 1 removal against Data Widgets 3.10 and passes, while `datagrid-dropdown-filter` is byte-for-byte in sync and fails. -5. **A mis-defaulted definition attribute in the `.mpk` parser.** The widget XML +5. **Augmentation never ran.** Before theorising about a widget's stored BSON, + confirm the reconciliation reached it. `AugmentTemplate`'s "nothing to add or + remove" guard returned before six value-level passes appended after it, so any + widget whose property set already matched its package was emitted unreconciled + (#716's drop-down filter). A one-line probe settles it — call `augmentFromMPK` + directly and count `ValueTypes with AllowUpload`: 0/25 for the broken widget + against 44/44 for a working one localised this in a single step. +6. **A mis-defaulted definition attribute in the `.mpk` parser.** The widget XML schema defaults `required` to **true**; reading a missing attribute as `false` put the wrong `Required` on 24 of DataGrid2 3.4's 40 properties. Two things make this family easy to miss: the value is only wrong for properties that *omit* the diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 4347dbb68..7cb416ca1 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -324,6 +324,7 @@ cases for these three BSON types — they fell to `default: return nil`. | A user reports CE0463 "the definition of this widget has changed" on DataGrid2 / Gallery / filters after upgrading the **Data Widgets** marketplace module. Reads like template drift; on real projects it is usually **not an mxcli bug at all** | A widget package that DROPS a property leaves every *stored* instance carrying a property the new definition lacks — which is precisely what CE0463 reports, and what its own message ("Update all widgets") tells you to fix. Ledger on 11.12: 0 errors at Data Widgets 3.4 (as authored) → 36 CE0463 at 3.11.3 → **0 again after `mx update-widgets`**. Single cause: `key="advanced"` is in `Datagrid.xml` at 3.4 and gone at 3.10/3.11 | no code change — diagnostic. See `docs/03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md` "CE0463 after a widget-package upgrade" | **Two controls settle it, neither optional.** (1) Do **Studio Pro's own** widgets fail too? A blank project's `dataGrid2_*`/`gallery1,2`/`drop_downFilter1,2` are Mendix-authored — if they fail alongside mxcli's, the tool is not the variable (29 of Ledger's 36 were these). (2) Does **`mx update-widgets` clear it**? If yes, mxcli's BSON was structurally valid and correct for the version it was written against; genuine template bugs do NOT clear this way (the Image stale default and the number-filter markerless array both needed template fixes). **The real mxcli defect is the residue after those controls**: author FRESH against the new package (`widget init` + author + `mx check`) — on 3.10/3.11 that leaves DataGrid2 **clean** and only Gallery + DatagridDropdownFilter failing, i.e. far narrower than the issue as filed. **Trap that cost a full round-trip**: measuring with the doctype fixtures alone mixes both cases, because their pages live in a blank project whose own template widgets are already failing — subtract by widget NAME against a control project that ran no mxcli command. Issue #716 | | Freshly authored **Gallery** widgets fail `mx check` with **CE0463** on Data Widgets 3.10+, while the 3.4 package bundled with Mendix 11.12/11.13 is clean — so it looks like ordinary post-upgrade staleness. Every schema-level explanation is disproven (property sets, list markers, ordering, pointer topology, `GenerateFromMPK`) | `syncDefinitionAttrs` reconciled a surviving property's definition attributes (`Required`, `OnChangeProperty`) from the installed `.mpk` onto the `CustomWidgets$WidgetPropertyType` node — but they live one level down, on its **`ValueType`**. The "only update a key that already exists" guard then never fired, making the entire pass a **silent no-op**, so the Gallery kept the embedded template's `OnChangeProperty = "onConfigurationChange"` where 3.10 expects `""` | `sdk/widgets/augment.go` + `modelsdk/widgets/augment.go` (`syncDefinitionAttrs`) | Descend to `ValueType` before the update (`getMapField(ptMap, "ValueType")`, falling back to the PropertyType). **Diagnosis method that found it**: diff against `mx update-widgets` output at the PATH level — the differing paths were `Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty`, and the path told me which node to write. **Generalisable — the trap that cost the most here**: a guarded update (`if _, ok := m[k]; ok`) aimed at the wrong node is *invisible*. It cannot fail loudly, so measurements read as "the fix didn't help" rather than "the fix never ran". When a change measurably does nothing, verify it executed before concluding the hypothesis was wrong. Result: fresh-authoring CE0463 on 3.10 went 6 → 2; bundled 3.4 stayed at 0. **Still open**: the two datagrid dropdown filters, whose residual diff is `ValueType/AllowUpload` (absent from the modelsdk template copy, present in sdk's — the engines' template sets have diverged); syncing them does not clear it. Repro `mdl-examples/bug-tests/716-widget-package-upgrade.mdl`. Issue #716 | | Every authored **Data grid 2** fails `mx check` with **CE0463** on the *bundled* Data Widgets 3.4 (Mendix 11.12/11.13) — but only on the **legacy** engine; `modelsdk` is clean on the same script. Appeared the moment `syncDefinitionAttrs` was corrected to write to `ValueType` (the row above), i.e. the moment that pass first actually executed | The Mendix pluggable-widget XML schema defaults `required` to **true**; only an explicit `required="false"` is optional. `sdk/widgets/mpk` read a missing attribute as `false` (`p.Required == "true"`), so the now-live sync overwrote 24 correct `true`s with `false` on DataGrid2 3.4 (which omits `required=` on 24 of its 40 properties). `modelsdk/widgets/mpk` already read it correctly (`p.Required != "false"`, fixed under #600) — the engines had silently diverged on the default | `sdk/widgets/mpk/mpk.go` (all three `PropertyDef` construction sites: `walkPropertyGroup` top-level + nested-direct, `collectNestedProperties`) | Read `p.Required != "false"` so absent means true, matching `modelsdk` and `mx update-widgets`. **Generalisable — the shape to look for**: when two engines carry parallel copies of a parser, a fix applied to one leaves a *latent* divergence in the other that stays invisible until some unrelated change starts consuming the value. Grep the sibling package for the same expression before assuming a defect is engine-specific. **Diagnosis trap hit here**: an A/B ran out of disk mid-run, the exec silently created no widgets, and `mx check` reported 0 CE0463 — reading as "the pre-fix binary is clean". Always assert the artifact exists (`show widgets | grep `) before trusting a zero. Unit repro `sdk/widgets/mpk/required_default_test.go`; integration `TestMxCheck_DataGridPage`/`TestMxCheck_DataGridNoColumns`. Issue #716 | +| Freshly authored **drop-down filter** widgets fail `mx check` with **CE0463** on Data Widgets 3.10 while every other widget in the same script is clean. The stored BSON is missing `ValueType/AllowUpload` on all 25 ValueTypes and carries `Required=false` on `refCaption`/`refCaptionExp`, which the 3.10 XML declares `required="true"` — so the embedded template looks like the culprit, but replacing it changes nothing | `AugmentTemplate` never ran on this widget. Its opening guard — "nothing to add or remove at top level, and no nested children" — was written when the function only synced the property SET, and returns before the **six value-level passes** bolted on later (`reconcileEnumValues`, `reconcilePropertyMetadata`, `reconcileValueTypesFromMPK`, `completeValueTypeEnvelope`, `reorderPropertyTypes`, `syncDefinitionAttrs`). DataGrid2 never hit it because `columns` has nested children; the drop-down filter declares exactly the 25 keys the 11.6-era template already has, so it took the exit every time | `modelsdk/widgets/augment.go` + `sdk/widgets/augment.go` (`AugmentTemplate`) | Wrap the add/remove block in `if len(missing) > 0 \|\| len(stale) > 0 \|\| hasNestedChildren { … }` instead of returning early, so the value-level passes always run. **Generalisable — the shape to look for**: an early return placed correctly for a function's original job silently disables everything appended after it. When a pass "does nothing" for one input and works for others, check whether it *reached* the pass before theorising about the data (same root shape as the `ValueType` no-op two rows up). The `hasNestedChildren` clause in that guard is the tell: someone had already patched around the same bug for one widget rather than fixing it. **Diagnosis method**: a probe calling `augmentFromMPK` directly and counting `ValueTypes with AllowUpload` — 0/25 for the drop-down filter vs 44/44 for Gallery — localised it to "augment did not run" in one step, before any BSON theorising. Result: modelsdk on DW 3.10 went 2 → **0**. Tests `TestAugmentTemplate_MatchingKeysStillReconcilesValues` in both engines. Issue #716 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/mdl-examples/bug-tests/716-widget-package-upgrade.mdl b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl index cda228528..76ed802bf 100644 --- a/mdl-examples/bug-tests/716-widget-package-upgrade.mdl +++ b/mdl-examples/bug-tests/716-widget-package-upgrade.mdl @@ -31,21 +31,28 @@ -- -- Measured, fixture 31 minus the untouched control (Mendix 11.13): -- --- engine package baseline with both fixes --- modelsdk DW 3.10 6 2 --- modelsdk DW 3.4 0 0 --- legacy DW 3.10 11 11 (pre-existing, untouched) --- legacy DW 3.4 0 0 (9 with the ValueType fix alone) +-- engine package baseline ValueType +required +early-return +-- modelsdk DW 3.10 6 2 2 0 +-- modelsdk DW 3.4 0 0 0 0 +-- legacy DW 3.10 11 11 11 11 +-- legacy DW 3.4 0 9 0 0 -- --- STILL OPEN: the two DataGrid dropdown filters (ddfStatus / ddfActive) in that --- fixture remain CE0463 on 3.10. Their residual diff against update-widgets is --- ValueType/AllowUpload, absent in the modelsdk template copy (present in sdk's --- — the two engines' template sets have diverged). Syncing the templates does --- NOT clear it, so something further is involved. +-- Third half: the two drop-down filters (ddfStatus / ddfActive). Missing +-- ValueType/AllowUpload was the visible symptom, but the template was not the +-- cause — augmentation never ran on this widget at all. AugmentTemplate returned +-- early whenever the property SET already matched the .mpk ("nothing to add or +-- remove"), which also skipped the six value-level passes that follow. Data +-- Widgets 3.10's drop-down filter declares exactly the 25 keys the embedded +-- 11.6-era template has, so it took the early exit and kept both a stale +-- Required (refCaption / refCaptionExp, explicitly required="true" in the 3.10 +-- XML) and no AllowUpload on any of its 25 ValueTypes. The guard now wraps only +-- the add/remove work. -- --- ALSO STILL OPEN: the legacy engine's 11 CE0463 on Data Widgets 3.10. That --- predates both fixes and is unchanged by them; the modelsdk engine (the --- default) is the one this issue tracks. +-- STILL OPEN: the legacy engine's 11 CE0463 on Data Widgets 3.10. Its +-- augmentation carries only syncDefinitionAttrs — the five reconcile passes +-- added to modelsdk under #600 (enum values, property metadata, ValueType +-- scalars, the AllowUpload envelope, PropertyType order) were never ported. +-- modelsdk is the default engine and is clean; porting them is a separate job. create module W716; create persistent entity W716.Customer ( Name: string(100), City: string(100) ); diff --git a/modelsdk/widgets/augment.go b/modelsdk/widgets/augment.go index c1a61ce3e..f8cc2144d 100644 --- a/modelsdk/widgets/augment.go +++ b/modelsdk/widgets/augment.go @@ -97,67 +97,71 @@ func AugmentTemplate(tmpl *WidgetTemplate, def *mpk.WidgetDefinition) error { } } - // Nothing to add/remove at top level, and no nested children to process - if len(missing) == 0 && len(stale) == 0 && !hasNestedChildren { - return nil - } - - // Remove stale properties - if len(stale) > 0 { - staleSet := make(map[string]bool, len(stale)) - for _, key := range stale { - staleSet[key] = true + // The add/remove work below is only needed when the property SET differs. + // The value-level reconciliation that FOLLOWS this block is not: a template + // whose keys already match the installed package can still carry stale values + // (Required, AllowUpload, enum options, defaults, order). Returning early here + // skipped all six of those passes and was the #716 dropdown-filter bug — its + // property set is byte-for-byte in sync with Data Widgets 3.10, so augmentation + // silently did nothing at all. + if len(missing) > 0 || len(stale) > 0 || hasNestedChildren { + // Remove stale properties + if len(stale) > 0 { + staleSet := make(map[string]bool, len(stale)) + for _, key := range stale { + staleSet[key] = true + } + propTypes, objProps = removeProperties(propTypes, objProps, staleSet) } - propTypes, objProps = removeProperties(propTypes, objProps, staleSet) - } - // Create a cloner for property pair deep-cloning - cloner := defaultCloner() + // Create a cloner for property pair deep-cloning + cloner := defaultCloner() - // Add missing properties - for _, p := range missing { - bsonType := xmlTypeToBSONType(p.Type) - if bsonType == "" { - continue // Unknown type, skip - } + // Add missing properties + for _, p := range missing { + bsonType := xmlTypeToBSONType(p.Type) + if bsonType == "" { + continue // Unknown type, skip + } - // Find an exemplar of the same type to clone - exemplarIdx, hasExemplar := typeExemplars[bsonType] - var newPropType, newProp map[string]any - if hasExemplar { - var err error - newPropType, newProp, err = cloner.ClonePair(propTypes, objProps, exemplarIdx, p) - if err != nil { - return fmt.Errorf("augment %s: %w", tmpl.WidgetID, err) + // Find an exemplar of the same type to clone + exemplarIdx, hasExemplar := typeExemplars[bsonType] + var newPropType, newProp map[string]any + if hasExemplar { + var err error + newPropType, newProp, err = cloner.ClonePair(propTypes, objProps, exemplarIdx, p) + if err != nil { + return fmt.Errorf("augment %s: %w", tmpl.WidgetID, err) + } + } + // Fall back to createPropertyPair if cloning failed (no exemplar or no matching property) + if newPropType == nil || newProp == nil { + newPropType, newProp = createPropertyPair(p, bsonType) } - } - // Fall back to createPropertyPair if cloning failed (no exemplar or no matching property) - if newPropType == nil || newProp == nil { - newPropType, newProp = createPropertyPair(p, bsonType) - } - if newPropType != nil { - propTypes = append(propTypes, newPropType) - } - if newProp != nil { - objProps = append(objProps, newProp) + if newPropType != nil { + propTypes = append(propTypes, newPropType) + } + if newProp != nil { + objProps = append(objProps, newProp) + } } - } - // Write back top-level - setArrayField(objType, "PropertyTypes", propTypes) - setArrayField(tmpl.Object, "Properties", objProps) - - // Augment nested ObjectType properties (e.g., DataGrid2 column properties). - // Top-level augmentation syncs the property list, but nested ObjectTypes inside - // IsList Object properties also need syncing when the .mpk version differs - // from the template version. - for _, mpkProp := range def.Properties { - if len(mpkProp.Children) == 0 { - continue - } - if err := augmentNestedObjectType(propTypes, objProps, mpkProp); err != nil { - return fmt.Errorf("augment nested %s: %w", mpkProp.Key, err) + // Write back top-level + setArrayField(objType, "PropertyTypes", propTypes) + setArrayField(tmpl.Object, "Properties", objProps) + + // Augment nested ObjectType properties (e.g., DataGrid2 column properties). + // Top-level augmentation syncs the property list, but nested ObjectTypes inside + // IsList Object properties also need syncing when the .mpk version differs + // from the template version. + for _, mpkProp := range def.Properties { + if len(mpkProp.Children) == 0 { + continue + } + if err := augmentNestedObjectType(propTypes, objProps, mpkProp); err != nil { + return fmt.Errorf("augment nested %s: %w", mpkProp.Key, err) + } } } diff --git a/modelsdk/widgets/augment_matching_keys_test.go b/modelsdk/widgets/augment_matching_keys_test.go new file mode 100644 index 000000000..b8fd53b2c --- /dev/null +++ b/modelsdk/widgets/augment_matching_keys_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package widgets + +import ( + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/widgets/mpk" +) + +// AugmentTemplate does two unrelated jobs: it syncs the property SET (add/remove +// keys), and it reconciles the VALUES of properties that already match. Only the +// first depends on the sets differing. +// +// A guard written for the add/remove work returned early whenever the sets were +// already in sync, which skipped every value-level pass. Data Widgets 3.10's +// drop-down filter has exactly the same 25 keys as the embedded 11.6-era +// template, so its augmentation silently did nothing and it kept a stale +// Required plus a missing AllowUpload envelope field — CE0463 on every freshly +// authored drop-down filter (mendixlabs/mxcli#716). +// +// The template here deliberately matches the definition key-for-key, so the test +// fails the moment that early return comes back. +func TestAugmentTemplate_MatchingKeysStillReconcilesValues(t *testing.T) { + ResetPlaceholderCounter() + + tmpl := &WidgetTemplate{ + WidgetID: "test.Widget", + Type: map[string]any{ + "ObjectType": map[string]any{ + "PropertyTypes": []any{ + float64(2), + map[string]any{ + "$ID": "pt0001", + "$Type": "CustomWidgets$WidgetPropertyType", + "PropertyKey": "caption", + "ValueType": map[string]any{ + "$ID": "vt0001", + "$Type": "CustomWidgets$WidgetValueType", + "Type": "Attribute", + // Stale: the installed package declares required="true". + "Required": false, + // AllowUpload absent: the template predates the field. + }, + }, + }, + }, + }, + Object: map[string]any{ + "Properties": []any{ + float64(2), + map[string]any{ + "$ID": "p0001", + "$Type": "CustomWidgets$WidgetValue", + "TypePointer": "pt0001", + }, + }, + }, + } + + // Same key set — nothing to add, nothing to remove, no nested children. + def := &mpk.WidgetDefinition{ + ID: "test.Widget", + Properties: []mpk.PropertyDef{ + {Key: "caption", Type: "attribute", Required: true}, + }, + } + + if err := AugmentTemplate(tmpl, def); err != nil { + t.Fatalf("AugmentTemplate: %v", err) + } + + objType := tmpl.Type["ObjectType"].(map[string]any) + propTypes := objType["PropertyTypes"].([]any) + + var vt map[string]any + for _, pt := range propTypes { + m, ok := pt.(map[string]any) + if !ok { + continue + } + if m["PropertyKey"] == "caption" { + vt, _ = m["ValueType"].(map[string]any) + } + } + if vt == nil { + t.Fatal("caption PropertyType or its ValueType went missing") + } + + if req, _ := vt["Required"].(bool); !req { + t.Errorf("Required = %v, want true — the value-level reconciliation did not run", vt["Required"]) + } + if _, ok := vt["AllowUpload"]; !ok { + t.Error("AllowUpload missing — completeValueTypeEnvelope did not run") + } +} diff --git a/sdk/widgets/augment.go b/sdk/widgets/augment.go index 6ed3a055a..461f44a7c 100644 --- a/sdk/widgets/augment.go +++ b/sdk/widgets/augment.go @@ -96,64 +96,68 @@ func AugmentTemplate(tmpl *WidgetTemplate, def *mpk.WidgetDefinition) error { } } - // Nothing to add/remove at top level, and no nested children to process - if len(missing) == 0 && len(stale) == 0 && !hasNestedChildren { - return nil - } - - // Remove stale properties - if len(stale) > 0 { - staleSet := make(map[string]bool, len(stale)) - for _, key := range stale { - staleSet[key] = true + // The add/remove work below is only needed when the property SET differs. + // The value-level reconciliation that FOLLOWS this block is not: a template + // whose keys already match the installed package can still carry stale values + // (Required, AllowUpload, enum options, defaults, order). Returning early here + // skipped all six of those passes and was the #716 dropdown-filter bug — its + // property set is byte-for-byte in sync with Data Widgets 3.10, so augmentation + // silently did nothing at all. + if len(missing) > 0 || len(stale) > 0 || hasNestedChildren { + // Remove stale properties + if len(stale) > 0 { + staleSet := make(map[string]bool, len(stale)) + for _, key := range stale { + staleSet[key] = true + } + propTypes, objProps = removeProperties(propTypes, objProps, staleSet) } - propTypes, objProps = removeProperties(propTypes, objProps, staleSet) - } - // Add missing properties - for _, p := range missing { - bsonType := xmlTypeToBSONType(p.Type) - if bsonType == "" { - continue // Unknown type, skip - } + // Add missing properties + for _, p := range missing { + bsonType := xmlTypeToBSONType(p.Type) + if bsonType == "" { + continue // Unknown type, skip + } - // Find an exemplar of the same type to clone - exemplarIdx, hasExemplar := typeExemplars[bsonType] - var newPropType, newProp map[string]any - if hasExemplar { - var err error - newPropType, newProp, err = clonePropertyPair(propTypes, objProps, exemplarIdx, p) - if err != nil { - return fmt.Errorf("augment %s: %w", tmpl.WidgetID, err) + // Find an exemplar of the same type to clone + exemplarIdx, hasExemplar := typeExemplars[bsonType] + var newPropType, newProp map[string]any + if hasExemplar { + var err error + newPropType, newProp, err = clonePropertyPair(propTypes, objProps, exemplarIdx, p) + if err != nil { + return fmt.Errorf("augment %s: %w", tmpl.WidgetID, err) + } + } + // Fall back to createPropertyPair if cloning failed (no exemplar or no matching property) + if newPropType == nil || newProp == nil { + newPropType, newProp = createPropertyPair(p, bsonType) } - } - // Fall back to createPropertyPair if cloning failed (no exemplar or no matching property) - if newPropType == nil || newProp == nil { - newPropType, newProp = createPropertyPair(p, bsonType) - } - if newPropType != nil { - propTypes = append(propTypes, newPropType) - } - if newProp != nil { - objProps = append(objProps, newProp) + if newPropType != nil { + propTypes = append(propTypes, newPropType) + } + if newProp != nil { + objProps = append(objProps, newProp) + } } - } - // Write back top-level - setArrayField(objType, "PropertyTypes", propTypes) - setArrayField(tmpl.Object, "Properties", objProps) + // Write back top-level + setArrayField(objType, "PropertyTypes", propTypes) + setArrayField(tmpl.Object, "Properties", objProps) - // Augment nested ObjectType properties (e.g., DataGrid2 column properties). - // Top-level augmentation syncs the property list, but nested ObjectTypes inside - // IsList Object properties also need syncing when the .mpk version differs - // from the template version. - for _, mpkProp := range def.Properties { - if len(mpkProp.Children) == 0 { - continue - } - if err := augmentNestedObjectType(propTypes, objProps, mpkProp); err != nil { - return fmt.Errorf("augment nested %s: %w", mpkProp.Key, err) + // Augment nested ObjectType properties (e.g., DataGrid2 column properties). + // Top-level augmentation syncs the property list, but nested ObjectTypes inside + // IsList Object properties also need syncing when the .mpk version differs + // from the template version. + for _, mpkProp := range def.Properties { + if len(mpkProp.Children) == 0 { + continue + } + if err := augmentNestedObjectType(propTypes, objProps, mpkProp); err != nil { + return fmt.Errorf("augment nested %s: %w", mpkProp.Key, err) + } } } diff --git a/sdk/widgets/augment_matching_keys_test.go b/sdk/widgets/augment_matching_keys_test.go new file mode 100644 index 000000000..eb03390e1 --- /dev/null +++ b/sdk/widgets/augment_matching_keys_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +package widgets + +import ( + "testing" + + "github.com/mendixlabs/mxcli/sdk/widgets/mpk" +) + +// Sibling of modelsdk/widgets' test of the same name. AugmentTemplate's early +// return was written for the add/remove work and skipped the value-level +// reconciliation that follows it, so a template whose keys already match the +// installed package was never reconciled at all (mendixlabs/mxcli#716). +// +// This copy carries only syncDefinitionAttrs — the five reconcile passes added +// to modelsdk under #600 (enum values, property metadata, ValueType scalars, +// the AllowUpload envelope, PropertyType order) were never ported here, which +// is why the legacy engine still reports CE0463 on Data Widgets 3.10. Asserting +// the one pass this copy does have keeps the guard honest in both engines. +func TestAugmentTemplate_MatchingKeysStillReconcilesValues(t *testing.T) { + tmpl := &WidgetTemplate{ + WidgetID: "test.Widget", + Type: map[string]any{ + "ObjectType": map[string]any{ + "PropertyTypes": []any{ + float64(2), + map[string]any{ + "$ID": "pt0001", + "$Type": "CustomWidgets$WidgetPropertyType", + "PropertyKey": "caption", + "ValueType": map[string]any{ + "$ID": "vt0001", + "$Type": "CustomWidgets$WidgetValueType", + "Type": "Attribute", + // Stale: the installed package declares required="true". + "Required": false, + }, + }, + }, + }, + }, + Object: map[string]any{ + "Properties": []any{ + float64(2), + map[string]any{ + "$ID": "p0001", + "$Type": "CustomWidgets$WidgetValue", + "TypePointer": "pt0001", + }, + }, + }, + } + + // Same key set — nothing to add, nothing to remove, no nested children. + def := &mpk.WidgetDefinition{ + ID: "test.Widget", + Properties: []mpk.PropertyDef{ + {Key: "caption", Type: "attribute", Required: true}, + }, + } + + if err := AugmentTemplate(tmpl, def); err != nil { + t.Fatalf("AugmentTemplate: %v", err) + } + + objType := tmpl.Type["ObjectType"].(map[string]any) + var vt map[string]any + for _, pt := range objType["PropertyTypes"].([]any) { + m, ok := pt.(map[string]any) + if !ok { + continue + } + if m["PropertyKey"] == "caption" { + vt, _ = m["ValueType"].(map[string]any) + } + } + if vt == nil { + t.Fatal("caption PropertyType or its ValueType went missing") + } + if req, _ := vt["Required"].(bool); !req { + t.Errorf("Required = %v, want true — syncDefinitionAttrs did not run", vt["Required"]) + } +} From 32ce82cd2ec1b0b63da7e850ffa31721fa20f681 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 14:05:25 +0000 Subject: [PATCH 16/29] docs: proposal for marketplace module upgrade, and why the obvious design fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picks up the work PROPOSAL_marketplace_modules.md parks ("a future ID-preserving merge is the remaining work"), prompted by field findings #32/#37 from an app authored end to end through mxcli: six of seven marketplace modules behind, and no route forward that does not involve Studio Pro. The investigation changed the design, so the proposal leads with the negative results rather than the feature. Measured, not assumed: - `mx` 11.13.0 has no module upgrade. module-import is positional-only (no --replace/--force) and error 3 on name collision is unconditional. `mx merge` is the right shape but is ID-keyed over projects sharing history. - Marketplace packages do NOT carry stable element IDs, including Mendix's own platform-supported modules: DataWidgets 3.10 -> 3.11.3 0 of 17 unit IDs shared Administration 4.3.2 -> 4.5.0 0 of 34 shared; Account and AccountPasswordData both renumbered So a literal replace is incorrect, not just risky: it renumbers entities the consuming app holds references to. An upgrade has to be a name-keyed merge that preserves the in-project IDs. - The installed module is not the package. A blank 11.13 project's untouched Administration 4.3.2, compared against the published 4.3.2 .mpk, differs in 15,066 paths across 10 of 27 elements. Converting the package to 11.13 first (mx convert accepts an .mpk) removes ~25 of them, so this is not version drift — the installed copy was transformed on the way in. That last result is the point of the document: a BSON-level `marketplace diff` would report every module as heavily modified. The proposal instead compares DESCRIBE output, which already discards IDs, storage envelopes and widget internals, and requires that a type with no DESCRIBE be reported "not comparable" rather than clean. Name+$Type is established as a sound join key (27 <-> 27, zero orphans both directions), which is what makes the eventual merge feasible. Scope is the read-only `mxcli marketplace diff`; the merge itself is named as phase 2 and deliberately not proposed here. Test plan leads with the negative control — untouched modules must report zero drift — since that is the check that catches the naive implementation. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .../PROPOSAL_marketplace_module_upgrade.md | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 docs/11-proposals/PROPOSAL_marketplace_module_upgrade.md diff --git a/docs/11-proposals/PROPOSAL_marketplace_module_upgrade.md b/docs/11-proposals/PROPOSAL_marketplace_module_upgrade.md new file mode 100644 index 000000000..7bb809e57 --- /dev/null +++ b/docs/11-proposals/PROPOSAL_marketplace_module_upgrade.md @@ -0,0 +1,273 @@ +--- +title: mxcli marketplace diff — detect local modification and plan an ID-preserving module upgrade +status: draft +date: 2026-08-04 +--- + +# Proposal: `mxcli marketplace diff` — detect local modification of an installed module + +**Status:** Draft +**Date:** 2026-08-04 + +Follows on from [`PROPOSAL_marketplace_modules.md`](PROPOSAL_marketplace_modules.md), +which ships discovery + download + install and explicitly parks the update path: +*"A future ID-preserving merge is the remaining work."* This proposal picks up that +remaining work and argues it should be approached back-to-front — ship the safety +question first, because it is separable, useful on its own, and a precondition for +the merge. + +## Problem Statement + +Updating a marketplace module is routine maintenance, and today there is no path +through it that does not involve Studio Pro. Field report from a Mendix app authored +end-to-end through mxcli ([`ako/mxcli-sudoku` FINDINGS #32/#37](https://github.com/ako/mxcli-sudoku)): +six of seven marketplace modules were behind — `DataWidgets` at 3.5.0 against 3.11.3 — +and every route closed: + +``` +$ mxcli marketplace install 116540 -p Sudoku.mpr +Module "DataWidgets" is already installed (version 3.5.0). Target version: 3.11.3. +In-place module updates are not applied automatically … Update via Studio Pro. + +$ mx module-import DataWidgets-3.11.3.mpk Sudoku.mpr +error 3: Project already contains a module with the name of an importing module. +``` + +The reporter's conclusion is the one that matters: *"An app buildable but not +maintainable through the CLI is only half-automatable."* + +### Why mxcli's refusal is right but unhelpful + +mxcli refuses because an in-place update can discard local edits and change +persistent-entity `$ID`s. That reasoning is sound but **unconditional** — there is no +way to tell mxcli a given case is safe, and no way to find out whether it is. + +The user cannot answer "is this safe?" because the question that actually decides it +— *has anyone modified this module since it was installed?* — has no tool. That is +the gap this proposal closes. + +## Investigation + +Everything below was measured, not assumed. Commands and results are reproducible +with the packages named. + +### 1. `mx` has no module upgrade + +`mx` 11.13.0's full command list contains `module-import`, `create-module-package`, +`show-module-version`, `set-module-version`, `merge`, `diff` — and no upgrade. +`module-import` takes positional arguments only; there is no `--replace`, `--force` +or `--overwrite`, and error 3 (name collision) is unconditional. + +`mx merge BASE MINE THEIRS` is the right *shape* — a three-way merge is exactly what +separates "the module author changed it" from "the user changed it" — but it is +ID-keyed and operates on whole projects that share history. Between two marketplace +packages it would see every element as deleted-plus-added (see §2), not modified. + +### 2. Marketplace packages do NOT carry stable element IDs + +A natural assumption is that a module author keeps element IDs stable across releases +so the module can be swapped. **Mendix's own platform-supported modules do not.** + +| Module | Versions compared | Shared unit IDs | Entity IDs | +|---|---|---|---| +| DataWidgets | 3.10 → 3.11.3 | **0 of 17** | *(module has no entities)* | +| Administration | 4.3.2 → 4.5.0 | **0 of 34** | `Account` **changed**, `AccountPasswordData` **changed** | + +Every element is regenerated per release — same name, same `$Type`, same folder, new +`$ID`. `Filter_Operators` moves `3eb4c31c…` → `6e70cf59…`; `Account` moves +`6dbb53a1…` → `815a92ab…`. + +**Consequence: a literal replace is not merely risky, it is incorrect.** Replacing the +installed module's units with the package's would renumber every entity in it, and +every reference *from the consuming app* — an association to `Administration.Account`, +a security rule, a microflow retrieve — points at the old `$ID`. The upgrade would +break the callers, not just the database mapping. + +An upgrade therefore has to be a **name-keyed merge into the existing module that +preserves the in-project `$ID`s**. This is consistent with what +`PROPOSAL_marketplace_modules.md` already records about Studio Pro's behaviour, and it +explains the failure mode Mendix users report: a name-keyed merge is well-defined +right up until the user has also edited the element, at which point it cannot decide +whose change wins. + +### 3. The installed module is not the package — so you cannot diff against it + +This is the finding that shapes the design, and it invalidates the obvious +implementation. + +Control: a **blank Mendix 11.13 project**, whose `Administration` module is untouched +by definition, compared against the **published 4.3.2 `.mpk`** it was built from. + +| Comparison | Elements matched by name | Orphans | Elements differing | Paths differing | +|---|---|---|---|---| +| installed 4.3.2 vs published 4.3.2 | 27 ↔ 27 | **0** | 10 of 27 | 15,066 | +| installed 4.3.2 vs published 4.3.2, **converted to 11.13 first** (`mx convert`) | 27 ↔ 27 | **0** | 7 of 27 | 15,041 | + +Two things follow. + +**Name+`$Type` is a sound join key.** Zero orphans in either direction — every element +in the installed module has exactly one counterpart in the package. This is what makes +a name-keyed merge feasible at all. + +**A path-level comparison against the package is not a drift signal.** An untouched +module differs in 15,041 paths. Running Mendix's own conversion first (`mx convert` +accepts an `.mpk` and emits a converted one) removes only ~25 of them, so this is not +version drift. The differences are whole subtrees present in the project and absent in +the package: + +``` +/FormCall/…/Object/Properties[25]/Value/TextTemplate/$Type + project = 'Forms$ClientTemplate' package = +/FormCall/…/Value/TextTemplate/Fallback/$Type + project = 'Texts$Text' package = +/Autofocus + project = 'DesktopOnly' package = (this one IS conversion) +``` + +The installed copy has been transformed on the way in — by import, by version +conversion, and (for the widget-bearing pages) by reconciliation against the widget +packages present in the consuming project. It is not, and never was, a copy of the +`.mpk` payload. + +A naive `mxcli marketplace diff` built on BSON comparison would therefore report every +module as heavily modified and be worse than useless. **This is the trap the proposal +exists to document.** + +## Design + +### Compare semantically, not structurally + +mxcli already has the normalisation this needs: `DESCRIBE` emits re-executable MDL for +a document, which by construction discards `$ID`s, storage envelopes, widget-internal +representation and the other artefacts §3 exposed. Two elements that describe to the +same MDL are the same element as far as an author is concerned. + +So drift detection compares **DESCRIBE output**, not BSON: + +1. Download the package for the version the project *claims* to have (`show modules` + reports `AppStoreVersion`). +2. Import it into a scratch project at the consuming project's Mendix version, so it + goes through the same conversion the installed copy did. +3. `DESCRIBE` every element of the module on both sides, key by name + `$Type`. +4. Report elements whose MDL differs — those are local modifications. + +This is bounded by DESCRIBE coverage: document types with no DESCRIBE (or a lossy one) +must be reported as **unknown**, never as clean. Silently treating an +un-describable element as unmodified is the one failure mode that would make the tool +dangerous rather than merely incomplete. + +### Then the upgrade becomes decidable + +With drift known, the three cases separate cleanly: + +| Installed module | Upgrade | +|---|---| +| unmodified | mechanical name-keyed merge, preserving in-project `$ID`s | +| modified, no collision with the new version | merge, reporting what was kept | +| modified, colliding with the new version | refuse, listing each conflicting element | + +Only the first is in scope for a first implementation; the others need the merge +engine and are deliberately deferred. + +## Proposed CLI + +```bash +# What has been changed locally in this module since it was installed? +mxcli marketplace diff -p app.mpr +mxcli marketplace diff DataWidgets -p app.mpr # resolve by module name + +# What would upgrading change? (adds the target-version comparison) +mxcli marketplace diff -p app.mpr --to 3.11.3 + +# Machine-readable for CI ("fail the build if a marketplace module was edited") +mxcli marketplace diff -p app.mpr --format json +``` + +Sketch of the output: + +``` +Administration — installed 4.3.2, latest 4.5.0 + + Local modifications (2 of 27 elements): + Forms$Page Account_Overview 3 statements differ + Microflows$Micro NewAccount 1 statement differs + + Not comparable (1 element): + Security$ModuleSecurity no DESCRIBE support + + Upgrading to 4.5.0 would touch 14 elements, 2 of which you have modified: + CONFLICT Forms$Page Account_Overview + CONFLICT Microflows$Microflow NewAccount +``` + +No MDL syntax is added. This is a CLI-only, read-only command. + +## Implementation Plan + +Phase 1 is the whole of this proposal; phase 2 is named only to show where it leads. + +### Phase 1 — `marketplace diff` (read-only) + +| File | Change | +|------|--------| +| `cmd/mxcli/cmd_marketplace.go` | New `diff` subcommand: flags `--to`, `--format`, module-name resolution | +| `cmd/mxcli/marketplace/compare.go` *(new)* | Orchestration: resolve installed version → download → convert → describe both sides → report | +| `cmd/mxcli/marketplace/scratch.go` *(new)* | Build the scratch project at the consuming project's version; wraps `mx convert` on the `.mpk` | +| `mdl/executor/` (describe paths) | Expose a programmatic "describe this element" entry point; today DESCRIBE is reachable only as a statement | +| `mdl/backend/` | Interface method to enumerate a module's elements with name + `$Type` (the catalog has this; it needs a backend-level accessor) | +| `docs-site/src/` | User-facing page for the command | + +### Phase 2 — `marketplace update` (deferred, not proposed here) + +Name-keyed, `$ID`-preserving merge, gated on a clean `diff`. Needs a decision on +conflict presentation and on what to do with elements the new version deletes. + +## Version Compatibility + +Not version-gated. The command works against any project mxcli can open. It does +require an `mx` binary for the conversion step (already a dependency of +`docker check`/`build`, and auto-downloadable via `setup mxbuild`), and marketplace +credentials for the download (existing `MENDIX_PAT` / `mxcli auth` layer). + +The one version-sensitive element is that the package must be converted **to the +consuming project's Mendix version** before comparison — comparing against an +unconverted package is measurably wrong (§3). + +## Test Plan + +The control from §3 is the primary test and it is fully reproducible: + +- **Negative control (must report zero drift):** a blank project at version *N*, whose + marketplace modules are untouched, diffed against their own published packages. If + this reports modifications, the normalisation is wrong. This is the test that would + have caught the naive BSON implementation. +- **Positive control (must report exactly one):** apply a single known edit through + MDL (e.g. `alter page Administration.Account_Overview …`), re-run, assert that + element and only that element is reported. +- **Coverage honesty:** assert that a document type with no DESCRIBE support is + reported as *not comparable*, never as clean. +- Fixtures in `mdl-examples/bug-tests/` are not the right home; this needs an + integration test under `-tags integration` because it shells out to `mx convert` + and the marketplace API. + +## Open Questions + +1. **Scratch-project conversion cost.** Each diff runs `mx convert` on a package + (~seconds). Acceptable for an explicit command; too slow to run implicitly inside + `mxcli check`. Should the converted package be cached under `~/.mxcli/`? +2. **DESCRIBE coverage is the real bound.** Which document types in a typical + marketplace module lack DESCRIBE today? `Security$ModuleSecurity` and + `Projects$ModuleSettings` appeared in the §3 control and need checking. The answer + sizes the "not comparable" bucket and therefore the feature's honesty. +3. **Is the widget-instance difference in §3 the same phenomenon as #716?** The + `TextTemplate`/`ClientTemplate` subtrees that differ are pluggable-widget envelope + fields. If the installed module's widget instances are reconciled against the + consuming project's widget packages on import, that connects this proposal to + [`PROPOSAL_widget_instance_reconciliation.md`](PROPOSAL_widget_instance_reconciliation.md) + and both may want the same comparison primitive. +4. **Modules with no recorded version.** `mx show-module-version` reports *"Module + 'DataWidgets' does not have a version"* while `mxcli show modules` reports + `Marketplace v3.5.0` — they read different fields. mxcli has no writer for + `AppStoreVersion`, so a hand-updated module cannot be recorded as such (FINDINGS + #37). Should this proposal include `set module version`, or does that belong with + phase 2? From 266643c3227e82405545d88067cdc3b65bb64b71 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 14:46:22 +0000 Subject: [PATCH 17/29] feat(widget): read-only plan for reconciling stored widget instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of `mxcli widget sync` (PROPOSAL_widget_instance_reconciliation): compare every stored pluggable-widget instance against the .mpk installed in the project and report what would change. Mutates nothing; `--dry-run` is the only mode this commit supports and applying returns an explicit "not implemented yet". Validated against `mx update-widgets` on a fixture authored against Data Widgets 3.4 and then upgraded to 3.11.3 (0 -> 40 CE0463). Every CE0463-affected instance is planned, and where both tools act they agree exactly: dataGrid21 update-widgets: remove `advanced`, add 17 plan: identical drop_downFilter* update-widgets: Required on refCaption, refCaptionExp plan: identical This reproduces both #716 diagnoses through an independent code path — the dropped `advanced` property and the refCaption/refCaptionExp Required drift. Two gaps remain, both measured rather than suspected, and both recorded in the package comment: - Forms$BuildingBlock is not scanned (FindAllCustomWidgetTypes covers pages and snippets only), so 44 changes across List_Cards / List_WithImage / Master_Detail are invisible. Layouts likewise. - Widgets in a theme module (IsThemeModule=true) must be left alone — update-widgets skips them and Mendix reports no CE0463 there, consistent with `mx module-import`'s dedicated refusal for theme modules. The plan currently proposes 16 changes it should not; model.Module does not carry the flag yet. Both are fixed by the same change: replace the per-widget-ID scan with a single-pass enumeration returning each instance with its container type and owning module. That is also O(units) rather than O(widget types x units), and it is the primitive `marketplace diff` needs (PROPOSAL_marketplace_module_upgrade). The planner reads the .mpk directly rather than the .def.json registry: the registry carries authoring routing, not the schema CE0463 compares. Note the default modelsdk engine has no FindAllCustomWidgetTypes, so this currently requires MXCLI_ENGINE=legacy — resolved by the same enumeration work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/cmd_widget_sync.go | 149 +++++++++++++ mdl/executor/widget_sync.go | 391 +++++++++++++++++++++++++++++++++++ 2 files changed, 540 insertions(+) create mode 100644 cmd/mxcli/cmd_widget_sync.go create mode 100644 mdl/executor/widget_sync.go diff --git a/cmd/mxcli/cmd_widget_sync.go b/cmd/mxcli/cmd_widget_sync.go new file mode 100644 index 000000000..2e8bbc97c --- /dev/null +++ b/cmd/mxcli/cmd_widget_sync.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "sort" + + "github.com/mendixlabs/mxcli/mdl/executor" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/spf13/cobra" +) + +// cmd_widget_sync.go is the CLI for reconciling stored widget instances against the +// widget packages installed in the project — the mxcli equivalent of Studio Pro's +// "Update all widgets" and `mx update-widgets`, without the latter's MPR v2 data loss. +// +// Currently read-only: it reports what would change. Applying is the next step. + +var widgetSyncCmd = &cobra.Command{ + Use: "sync", + Short: "Reconcile stored widget instances against the installed widget packages", + Long: `Compare every stored pluggable-widget instance against the .mpk currently +installed in the project, and report the schema differences. + +mxcli writes a widget instance correctly for the package installed at authoring +time. When that package is later upgraded, the stored instances go stale and +Mendix reports CE0463 "the definition of this widget has changed" on each one. +This command finds those instances. + +It reconciles SCHEMA — properties the package added, dropped, or redefined. It +never changes a property value you set, and it never touches a widget whose .mpk +is not installed. + +Examples: + mxcli widget sync -p app.mpr --dry-run + mxcli widget sync -p app.mpr --dry-run --widget com.mendix.widget.web.datagrid.Datagrid + mxcli widget sync -p app.mpr --dry-run --page MyModule.Overview`, + RunE: runWidgetSync, +} + +func init() { + widgetSyncCmd.Flags().StringP("project", "p", "", "Path to .mpr project file") + widgetSyncCmd.Flags().Bool("dry-run", false, "Report what would change without writing") + widgetSyncCmd.Flags().String("widget", "", "Only this widget type (full widget ID)") + widgetSyncCmd.Flags().String("page", "", "Only this page or snippet (qualified name)") + widgetSyncCmd.MarkFlagRequired("project") + widgetCmd.AddCommand(widgetSyncCmd) +} + +func runWidgetSync(cmd *cobra.Command, args []string) error { + projectPath, _ := cmd.Flags().GetString("project") + dryRun, _ := cmd.Flags().GetBool("dry-run") + widgetID, _ := cmd.Flags().GetString("widget") + page, _ := cmd.Flags().GetString("page") + + if !dryRun { + return fmt.Errorf("applying is not implemented yet — re-run with --dry-run to see what would change") + } + + exec, logger := newLoggedExecutor("subcommand") + defer logger.Close() + defer exec.Close() + + prog, errs := visitor.Build(fmt.Sprintf("CONNECT LOCAL '%s';", visitor.QuoteString(projectPath))) + if len(errs) > 0 { + return fmt.Errorf("connect: %v", errs[0]) + } + exec.SetQuiet(true) + if err := exec.ExecuteProgram(prog); err != nil { + return fmt.Errorf("connect to %s: %w", projectPath, err) + } + + plan, err := executor.PlanWidgetSync(exec.Backend(), projectPath, executor.SyncOptions{ + WidgetID: widgetID, + Container: page, + }) + if err != nil { + return err + } + renderSyncPlan(plan) + return nil +} + +// renderSyncPlan prints the plan. Every property is named rather than counted: +// removing a property discards its stored value, and that is exactly what a user +// needs to see before it happens. +func renderSyncPlan(plan *executor.SyncPlan) { + out := os.Stdout + + if len(plan.Unresolved) > 0 { + fmt.Fprintln(out, "Widgets used in the model with no installed .mpk (skipped, never modified):") + sort.Strings(plan.Unresolved) + for _, id := range plan.Unresolved { + fmt.Fprintf(out, " %s\n", id) + } + fmt.Fprintln(out) + } + + if plan.Empty() { + fmt.Fprintln(out, "Every stored widget instance already matches its installed package. Nothing to do.") + return + } + + container := "" + for _, w := range plan.Widgets { + if w.Container != container { + container = w.Container + fmt.Fprintf(out, "\n%s\n", container) + } + fmt.Fprintf(out, " %s (%s %s)\n", w.Widget, shortName(w.WidgetID), w.PackageVer) + for _, c := range w.Changes { + fmt.Fprintf(out, " %s %-24s %s\n", changeMarker(c.Kind), c.Key, c.Detail) + } + } + + fmt.Fprintf(out, "\n%d widget instance(s), %d property change(s) across %d container(s).\n", + len(plan.Widgets), plan.TotalChanges(), countContainers(plan)) + fmt.Fprintln(out, "Read-only: nothing was written.") +} + +func changeMarker(k executor.SyncChangeKind) string { + switch k { + case executor.SyncRemove: + return "-" + case executor.SyncAdd: + return "+" + default: + return "~" + } +} + +func countContainers(plan *executor.SyncPlan) int { + seen := map[string]bool{} + for _, w := range plan.Widgets { + seen[w.Container] = true + } + return len(seen) +} + +func shortName(widgetID string) string { + for i := len(widgetID) - 1; i >= 0; i-- { + if widgetID[i] == '.' { + return widgetID[i+1:] + } + } + return widgetID +} diff --git a/mdl/executor/widget_sync.go b/mdl/executor/widget_sync.go new file mode 100644 index 000000000..5c9e6be25 --- /dev/null +++ b/mdl/executor/widget_sync.go @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/sdk/widgets/mpk" +) + +// widget_sync.go plans the reconciliation of *stored* widget instances against the +// widget packages currently installed in the project. +// +// mxcli authors a pluggable-widget instance correctly for the .mpk installed at +// authoring time, and has nothing that revisits the instance when that package later +// changes. Studio Pro has "Update all widgets"; mxbuild has `mx update-widgets`, which +// on MPR v2 destroys mprcontents/. This is the mxcli equivalent that does not. +// +// This file is the READ-ONLY half: it produces a plan and mutates nothing. See +// PROPOSAL_widget_instance_reconciliation.md. +// +// The unit of work is a CustomWidgets$CustomWidget node, which carries two paired +// arrays that must always move together: +// +// Type.ObjectType.PropertyTypes[] — the schema, one entry per property, keyed by PropertyKey +// Object.Properties[] — the values, bound to their type by TypePointer +// +// Splitting a pair produces the StreamingBsonUnitReader "does not contain a +// constructor with a parameter of type WidgetValue" load failure, i.e. a project that +// will not open at all — so the plan is expressed in property keys, and applying it +// must move both halves. +// +// # Validated against `mx update-widgets` +// +// On a fixture authored against Data Widgets 3.4 and then upgraded to 3.11.3 (40 +// CE0463), this planner's change set was compared to what Mendix's own tool actually +// writes. Every CE0463-affected instance is planned — nothing is missed — and where +// both act they agree exactly (DataGrid2: remove `advanced` + add 17; the drop-down +// filters: `Required` on refCaption/refCaptionExp). +// +// # Two known gaps, both measured +// +// 1. CONTAINER COVERAGE. FindAllCustomWidgetTypes scans Forms$Page and Forms$Snippet +// only. Widgets in Forms$BuildingBlock are invisible to it — 44 changes across +// List_Cards, List_WithImage and Master_Detail that update-widgets makes and this +// plan does not see. Layouts are likewise unscanned. +// +// 2. THEME MODULES. update-widgets does not touch widgets in a module with +// IsThemeModule=true (FeedbackModule in a blank project), and Mendix reports no +// CE0463 on them either — `mx module-import` has a dedicated refusal for theme +// modules, so they are deliberately off-limits. This plan currently proposes 16 +// changes there that it should not. model.Module does not yet carry the flag. +// +// Both are fixed by replacing the per-widget-ID scan with a single-pass enumeration +// that returns every instance with its container type and owning module — which is +// also O(units) instead of O(widget types x units). That is the next step, and it is +// the primitive `marketplace diff` needs too (see PROPOSAL_marketplace_module_upgrade). + +// SyncChangeKind is what a plan proposes to do to one property. +type SyncChangeKind string + +const ( + // SyncRemove — the stored instance carries a PropertyKey the installed package no + // longer declares. This is the mendixlabs/mxcli#716 case ("advanced", dropped from + // Data Widgets after 3.4). Removing it DISCARDS the stored value. + SyncRemove SyncChangeKind = "remove" + // SyncAdd — the package declares a property the stored instance lacks; it is added + // with the package's default value. + SyncAdd SyncChangeKind = "add" + // SyncUpdate — the property survives but its own definition attributes changed. + SyncUpdate SyncChangeKind = "update" +) + +// SyncPropertyChange is one proposed change to one property of one widget instance. +type SyncPropertyChange struct { + Kind SyncChangeKind + // Key is the PropertyKey, qualified with its parent for nested object types + // (e.g. "columns/tooltip") so a report is unambiguous. + Key string + // Detail explains the change in the report ("dropped by the package", + // `Required false -> true`). + Detail string +} + +// SyncWidgetPlan is the set of changes for a single stored widget instance. +type SyncWidgetPlan struct { + Container string // qualified name of the page/snippet holding the widget + ContainerID string + Widget string // the instance's Name + WidgetID string // e.g. com.mendix.widget.web.datagrid.Datagrid + PackageVer string // version of the installed .mpk + Changes []SyncPropertyChange + StoredKeys int + PackageKeys int +} + +// SyncPlan is the whole read-only result. +type SyncPlan struct { + Widgets []SyncWidgetPlan + // Unresolved lists widget IDs found in the model with no installed .mpk. These are + // reported and never touched: deleting an instance's properties because its package + // is missing would be the worst possible failure mode. + Unresolved []string +} + +// TotalChanges counts proposed property changes across every instance. +func (p SyncPlan) TotalChanges() int { + n := 0 + for _, w := range p.Widgets { + n += len(w.Changes) + } + return n +} + +// Empty reports whether there is nothing to do. +func (p SyncPlan) Empty() bool { return p.TotalChanges() == 0 } + +// SyncOptions narrows the blast radius of a plan. +type SyncOptions struct { + WidgetID string // only this widget type + Container string // only this page/snippet (qualified name) +} + +// PlanWidgetSync compares every stored widget instance against the widget package +// installed in the project and returns what would change. It mutates nothing. +// +// Note it reads the .mpk **directly** rather than the .def.json definition registry. +// The registry carries authoring ROUTING (which MDL keyword feeds which property key), +// not the schema CE0463 compares — verified during #716, where regenerating every +// definition from the project's .mpk left the error count unchanged. +func PlanWidgetSync(b backend.WidgetBackend, projectPath string, opts SyncOptions) (*SyncPlan, error) { + if b == nil { + return nil, fmt.Errorf("not connected to a project") + } + + defs, err := installedWidgetDefs(projectPath) + if err != nil { + return nil, err + } + if len(defs) == 0 { + return &SyncPlan{}, nil + } + + plan := &SyncPlan{} + ids := make([]string, 0, len(defs)) + for id := range defs { + ids = append(ids, id) + } + sort.Strings(ids) + + for _, id := range ids { + if opts.WidgetID != "" && !strings.EqualFold(opts.WidgetID, id) { + continue + } + instances, err := b.FindAllCustomWidgetTypes(id) + if err != nil { + return nil, fmt.Errorf("find instances of %s: %w", id, err) + } + for _, inst := range instances { + if opts.Container != "" && !strings.EqualFold(opts.Container, inst.UnitName) { + continue + } + wp := planInstance(inst, defs[id]) + if len(wp.Changes) > 0 { + plan.Widgets = append(plan.Widgets, wp) + } + } + } + + sort.Slice(plan.Widgets, func(i, j int) bool { + if plan.Widgets[i].Container != plan.Widgets[j].Container { + return plan.Widgets[i].Container < plan.Widgets[j].Container + } + return plan.Widgets[i].Widget < plan.Widgets[j].Widget + }) + return plan, nil +} + +// planInstance diffs one stored instance against its package definition. +func planInstance(inst *types.RawCustomWidgetType, def *mpk.WidgetDefinition) SyncWidgetPlan { + wp := SyncWidgetPlan{ + Container: inst.UnitName, + ContainerID: inst.UnitID, + Widget: inst.WidgetName, + WidgetID: inst.WidgetID, + PackageVer: def.Version, + } + + stored := storedPropertyTypes(inst.RawType) + wp.StoredKeys = len(stored) + + // System properties (Label, Visibility, Editability) are not declared as regular + // properties in the widget XML but are stored on the instance. Treating them as + // "not in the package" would delete them. + system := def.SystemPropertyKeys() + + pkg := map[string]*mpk.PropertyDef{} + for i := range def.Properties { + pkg[def.Properties[i].Key] = &def.Properties[i] + } + wp.PackageKeys = len(pkg) + + for _, key := range sortedKeys(stored) { + if system[key] { + continue + } + p, ok := pkg[key] + if !ok { + wp.Changes = append(wp.Changes, SyncPropertyChange{ + Kind: SyncRemove, + Key: key, + Detail: fmt.Sprintf("not declared by %s %s", shortWidgetName(def.ID), def.Version), + }) + continue + } + wp.Changes = append(wp.Changes, attrChanges(key, stored[key], p)...) + } + + for _, key := range sortedDefKeys(def.Properties) { + if _, ok := stored[key]; !ok { + wp.Changes = append(wp.Changes, SyncPropertyChange{ + Kind: SyncAdd, + Key: key, + Detail: fmt.Sprintf("declared by %s %s, default %q", shortWidgetName(def.ID), def.Version, pkg[key].DefaultValue), + }) + } + } + return wp +} + +// attrChanges reports definition-attribute drift on a property that exists on both +// sides. Only attributes the stored ValueType ALREADY carries are considered: adding a +// key the node does not have invents a property this Mendix version may not define — +// the mendixlabs/mxcli#759 failure shape. +func attrChanges(key string, vt bson.D, p *mpk.PropertyDef) []SyncPropertyChange { + var out []SyncPropertyChange + if cur, ok := bsonLookup(vt, "Required"); ok { + if b, isBool := cur.(bool); isBool && b != p.Required { + out = append(out, SyncPropertyChange{ + Kind: SyncUpdate, + Key: key, + Detail: fmt.Sprintf("Required %v -> %v", b, p.Required), + }) + } + } + if cur, ok := bsonLookup(vt, "OnChangeProperty"); ok { + if s, isStr := cur.(string); isStr && s != p.OnChange { + out = append(out, SyncPropertyChange{ + Kind: SyncUpdate, + Key: key, + Detail: fmt.Sprintf("OnChangeProperty %q -> %q", s, p.OnChange), + }) + } + } + return out +} + +// storedPropertyTypes maps PropertyKey -> its ValueType document, for the top-level +// PropertyTypes of a stored widget instance. +func storedPropertyTypes(rawType any) map[string]bson.D { + out := map[string]bson.D{} + objType, ok := bsonLookup(rawType, "ObjectType") + if !ok { + return out + } + pts, ok := bsonLookup(objType, "PropertyTypes") + if !ok { + return out + } + for _, pt := range bsonArray(pts) { + key, ok := bsonLookup(pt, "PropertyKey") + if !ok { + continue + } + name, _ := key.(string) + if name == "" { + continue + } + vt := bson.D{} + if v, ok := bsonLookup(pt, "ValueType"); ok { + if d, ok := asDoc(v); ok { + vt = d + } + } + out[name] = vt + } + return out +} + +// installedWidgetDefs parses every .mpk under the project's widgets/ directory and +// returns the definitions keyed by widget ID. A single .mpk can bundle many widgets +// (Charts.mpk ships ten), so every one is registered. +// +// Unlike RefreshWidgetDefinitions this does NOT skip widgets that have a hand-written +// built-in definition: those (Gallery, the filters) are exactly the ones that go stale, +// and the built-in registry has no bearing on stored schema. +func installedWidgetDefs(projectPath string) (map[string]*mpk.WidgetDefinition, error) { + widgetsDir := filepath.Join(filepath.Dir(projectPath), "widgets") + matches, err := filepath.Glob(filepath.Join(widgetsDir, "*.mpk")) + if err != nil { + return nil, fmt.Errorf("scan widgets directory: %w", err) + } + defs := map[string]*mpk.WidgetDefinition{} + for _, path := range matches { + parsed, err := mpk.ParseMPKAll(path) + if err != nil { + // A single unreadable package must not fail the whole plan; it becomes an + // unresolved widget if the model references it. + continue + } + for _, d := range parsed { + if d != nil && d.ID != "" { + defs[d.ID] = d + } + } + } + return defs, nil +} + +// --- small BSON helpers ----------------------------------------------------- +// RawType/RawObject cross the backend boundary as `any` (mdl/types avoids a BSON +// driver dependency); underneath they are bson.D. + +func asDoc(v any) (bson.D, bool) { + switch d := v.(type) { + case bson.D: + return d, true + case *bson.D: + if d != nil { + return *d, true + } + } + return nil, false +} + +func bsonLookup(v any, key string) (any, bool) { + d, ok := asDoc(v) + if !ok { + return nil, false + } + for _, e := range d { + if e.Key == key { + return e.Value, true + } + } + return nil, false +} + +func bsonArray(v any) []any { + switch a := v.(type) { + case bson.A: + return a + case []any: + return a + } + return nil +} + +func sortedKeys(m map[string]bson.D) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedDefKeys(props []mpk.PropertyDef) []string { + out := make([]string, 0, len(props)) + for _, p := range props { + out = append(out, p.Key) + } + sort.Strings(out) + return out +} + +// shortWidgetName turns com.mendix.widget.web.datagrid.Datagrid into Datagrid for +// readable report lines. +func shortWidgetName(id string) string { + if i := strings.LastIndex(id, "."); i >= 0 && i < len(id)-1 { + return id[i+1:] + } + return id +} From 8ce65f030d5f00a25f1a1f49acd170bd689d0537 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:27:40 +0000 Subject: [PATCH 18/29] =?UTF-8?q?build:=20strip=20release/build=20binaries?= =?UTF-8?q?=20(-s=20-w=20-trimpath)=20=E2=80=94=20~112MB=20=E2=86=92=20~84?= =?UTF-8?q?MB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build and release targets shipped full symbol table + DWARF debug info (~25% of the binary, ≈28MB), which is unnecessary for distribution. Add a RELEASE_LDFLAGS (`-s -w`) plus `-trimpath` and use them for `build` and the six `release` cross-compile targets. `build-debug` keeps the unstripped, symbol-rich binary for debugging. Version/BuildTime injection is preserved. Also add a `make size` helper that prints the built binary size, to catch size regressions before a release. Measured: bin/mxcli 112M → 84M; `--version` still reports the injected string. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JXnEgoc2NQP1Y2TWMCMXC4 --- Makefile | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 673a8fa0c..97c55fce3 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,17 @@ CMD_PATH = ./cmd/mxcli VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") BUILD_TIME = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") LDFLAGS = -ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)" +# RELEASE_LDFLAGS additionally strips the symbol table (-s) and DWARF debug info +# (-w), which is ~25% of the binary (≈28MB on a ~112MB build) and unnecessary for +# distribution. Combined with -trimpath (GO_BUILD_FLAGS) for reproducible builds +# without local path leakage. The build-debug target keeps symbols for debugging. +RELEASE_LDFLAGS = -ldflags "-s -w -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)" +GO_BUILD_FLAGS = -trimpath # Clean version for VS Code extension (must be valid semver: major.minor.patch) VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0") -.PHONY: build build-debug release clean test engine-diff test-mdl check-mdl check-skill-mdl check-widget-versions grammar completions sync-skills sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt vet +.PHONY: build build-debug size release clean test engine-diff test-mdl check-mdl check-skill-mdl check-widget-versions grammar completions sync-skills sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt vet # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache) # Usage: $(call copy-if-changed,src,dst) @@ -103,9 +109,9 @@ completions: # Build for current platform (auto-syncs skills and commands) build: grammar sync-all completions @mkdir -p $(BUILD_DIR) - CGO_ENABLED=0 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) $(CMD_PATH) - CGO_ENABLED=0 go build -o $(BUILD_DIR)/source_tree ./cmd/source_tree - @echo "Built $(BUILD_DIR)/$(BINARY_NAME) $(BUILD_DIR)/source_tree" + CGO_ENABLED=0 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) $(CMD_PATH) + CGO_ENABLED=0 go build $(GO_BUILD_FLAGS) -o $(BUILD_DIR)/source_tree ./cmd/source_tree + @echo "Built $(BUILD_DIR)/$(BINARY_NAME) ($$(du -h $(BUILD_DIR)/$(BINARY_NAME) | cut -f1)) $(BUILD_DIR)/source_tree" # Build with debug tools (includes bson discover/compare/dump) build-debug: sync-all completions @@ -113,28 +119,33 @@ build-debug: sync-all completions CGO_ENABLED=0 go build -tags debug $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-debug $(CMD_PATH) @echo "Built $(BUILD_DIR)/$(BINARY_NAME)-debug (debug build with bson tools)" +# Report the built binary size (builds first if needed). Handy for catching size +# regressions before a release. +size: build + @echo "$(BINARY_NAME): $$(du -h $(BUILD_DIR)/$(BINARY_NAME) | cut -f1) (stripped release build)" + # Build for all platforms (CGO_ENABLED=0 for cross-compilation) release: clean grammar vscode-ext sync-all @mkdir -p $(BUILD_DIR) @echo "Building release binaries..." @echo " -> Linux (amd64)" - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 $(CMD_PATH) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 $(CMD_PATH) @echo " -> Linux (arm64)" - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64 $(CMD_PATH) + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64 $(CMD_PATH) @echo " -> macOS (amd64 - Intel)" - CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-amd64 $(CMD_PATH) + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-amd64 $(CMD_PATH) @echo " -> macOS (arm64 - Apple Silicon)" - CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64 $(CMD_PATH) + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64 $(CMD_PATH) @echo " -> Windows (amd64)" - CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe $(CMD_PATH) + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe $(CMD_PATH) @echo " -> Windows (arm64)" - CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-windows-arm64.exe $(CMD_PATH) + CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build $(GO_BUILD_FLAGS) $(RELEASE_LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-windows-arm64.exe $(CMD_PATH) @echo "" @echo "Release binaries:" From bab6f2f34f1d86ff6386ea79669a879404a04275 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:29:20 +0000 Subject: [PATCH 19/29] chore: regenerate LSP completions (drop stale V3 keyword) `make build` regenerates cmd/mxcli/lsp_completions_gen.go from the lexer grammar. The committed copy had drifted: it still listed `V3` as a keyword completion, but `V3` is now a lexer identifier (not a reserved keyword), so the generator drops it. Refresh the committed file to match the grammar. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JXnEgoc2NQP1Y2TWMCMXC4 --- cmd/mxcli/lsp_completions_gen.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 67e583e29..40b294377 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -525,7 +525,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "DRY", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "RUN", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "WIDGETTYPE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, - {Label: "V3", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, // Keyword {Label: "BUSINESS", Kind: protocol.CompletionItemKindKeyword, Detail: "Keyword"}, From ae0338d28f9e0fb3a21019a3040d76a829449f10 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:36:49 +0000 Subject: [PATCH 20/29] feat(widget): single-pass widget-instance scan covering building blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-widget-ID lookup in the sync planner with one pass over every widget-bearing unit type. Three concrete wins, and one rule discovered and then disproven. Coverage. The old path (WidgetBackend.FindAllCustomWidgetTypes) scanned Forms$Page and Forms$Snippet only. Building blocks and page templates are widget containers too, and mxcli supports authoring building blocks. Measured against `mx update-widgets` on the 3.4 -> 3.11.3 fixture, the plan now MISSES NOTHING: before this commit plan 504 changes, 44 misses vs update-widgets after this commit plan 776 changes, 0 misses vs update-widgets Engines. The scan is built on ListRawUnitsByType, which both the MPR and modelsdk backends implement, so `widget sync` now runs on the default engine. It previously required MXCLI_ENGINE=legacy, because FindAllCustomWidgetTypes exists only in the legacy backend. Both engines produce identical plans. Cost. One pass over the model instead of re-reading every unit once per installed widget type — O(units) rather than O(widget types x units), which on a blank project was ~15k unit reads (42 definitions x 370 units). No new backend interface method was needed. Rule discovered, then DISPROVEN — recorded so it is not re-derived. It looked as though update-widgets skips theme modules: it leaves four FeedbackModule Image instances short by four properties, and Mendix reports no CE0463 on them. But Atlas_Core, Atlas_Web_Content and DataWidgets are theme modules too, and update-widgets reconciles 18 Atlas_Web_Content containers. IsThemeModule is therefore recorded on the scan result but excludes nothing. That leaves a genuine open question, now stated in the code: update-widgets adds 12 properties to each stale Gallery but adds none to those four Images. So "add missing" is not unconditionally what Mendix does. Remove-stale and attribute-sync are the operations known to be necessary and faithful; the trigger for "add" needs more evidence before anything is written. 231 of the plan's 244 non-matching entries are adds of this kind. Still read-only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- mdl/executor/widget_scan.go | 199 ++++++++++++++++++++++++++++++++++++ mdl/executor/widget_sync.go | 70 +++++++------ mdl/types/infrastructure.go | 19 ++++ 3 files changed, 254 insertions(+), 34 deletions(-) create mode 100644 mdl/executor/widget_scan.go diff --git a/mdl/executor/widget_scan.go b/mdl/executor/widget_scan.go new file mode 100644 index 000000000..d0222e580 --- /dev/null +++ b/mdl/executor/widget_scan.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" +) + +// widget_scan.go enumerates every stored pluggable-widget instance in the model in a +// single pass, with the context needed to decide whether it may be modified. +// +// It replaces the per-widget-ID scan (WidgetBackend.FindAllCustomWidgetTypes), which +// had two measured defects: +// +// - it searches Forms$Page and Forms$Snippet only, so widgets inside a +// Forms$BuildingBlock are invisible — 44 changes `mx update-widgets` makes in a +// blank project's Atlas building blocks that a page/snippet scan cannot see; +// - it re-reads every unit once per widget type, i.e. O(widget types x units). +// 42 installed widget definitions over 370 units is ~15k unit reads for what is +// one pass over the model. +// +// It also runs on both engines: it is built on ListRawUnitsByType, which the MPR and +// modelsdk backends both implement, whereas FindAllCustomWidgetTypes exists only in +// the legacy backend. + +// widgetContainerTypes are the unit types that can hold a widget tree. Building +// blocks are included deliberately: mxcli supports authoring them, and Mendix's own +// update-widgets reconciles the widgets inside them. +var widgetContainerTypes = []string{ + "Forms$Page", + "Forms$Snippet", + "Forms$BuildingBlock", + "Forms$PageTemplate", + "Forms$Layout", +} + +// scanCustomWidgetInstances walks every widget-bearing unit and returns one entry per +// stored CustomWidgets$CustomWidget node. +func scanCustomWidgetInstances(b backend.RawUnitBackend) ([]*types.CustomWidgetInstance, error) { + modules, err := scanModules(b) + if err != nil { + return nil, err + } + + var out []*types.CustomWidgetInstance + for _, unitType := range widgetContainerTypes { + units, err := b.ListRawUnitsByType(unitType) + if err != nil { + // A unit type absent from this project is not an error. + continue + } + for _, u := range units { + var doc bson.D + if err := bson.Unmarshal(u.Contents, &doc); err != nil { + continue + } + unitName := bsonString(doc, "Name") + mod := modules.owner(string(u.ContainerID)) + collectWidgets(doc, func(w *types.CustomWidgetInstance) { + w.UnitID = string(u.ID) + w.UnitName = unitName + w.UnitType = unitType + w.ModuleName = mod.name + w.ModuleIsTheme = mod.isTheme + out = append(out, w) + }) + } + } + return out, nil +} + +// moduleInfo is what the scan needs to know about the module owning a unit. +type moduleInfo struct { + name string + isTheme bool +} + +// moduleIndex resolves a unit's owning module by walking container IDs upward. +type moduleIndex struct { + byID map[string]moduleInfo + parentOf map[string]string +} + +// owner walks up from a container ID until it reaches a module. Units nest inside +// folders, so the immediate ContainerID is often not the module itself. +func (m *moduleIndex) owner(containerID string) moduleInfo { + seen := map[string]bool{} + for id := containerID; id != "" && !seen[id]; id = m.parentOf[id] { + seen[id] = true + if info, ok := m.byID[id]; ok { + return info + } + } + return moduleInfo{} +} + +// scanModules builds the module index, recording IsThemeModule. +// +// IsThemeModule is recorded but deliberately NOT used to exclude anything. It looked +// like the rule that explains why update-widgets leaves four FeedbackModule images +// alone, and it is wrong: Atlas_Core, Atlas_Web_Content and DataWidgets are theme +// modules too, and update-widgets reconciles 18 Atlas_Web_Content containers. The flag +// is kept because it is cheap and callers may want to report it. +func scanModules(b backend.RawUnitBackend) (*moduleIndex, error) { + idx := &moduleIndex{byID: map[string]moduleInfo{}, parentOf: map[string]string{}} + + units, err := b.ListRawUnitsByType("Projects$ModuleImpl") + if err != nil { + return nil, fmt.Errorf("list modules: %w", err) + } + for _, u := range units { + var doc bson.D + if err := bson.Unmarshal(u.Contents, &doc); err != nil { + continue + } + idx.byID[string(u.ID)] = moduleInfo{ + name: bsonString(doc, "Name"), + isTheme: bsonBool(doc, "IsThemeModule"), + } + } + + // Folders sit between a document and its module, so the walk needs the full + // parent chain, not just the module units. + for _, t := range []string{"Projects$Folder", "Projects$ModuleImpl"} { + folders, err := b.ListRawUnitsByType(t) + if err != nil { + continue + } + for _, f := range folders { + idx.parentOf[string(f.ID)] = string(f.ContainerID) + } + } + return idx, nil +} + +// collectWidgets walks a unit document and calls visit for every CustomWidget node, +// including nested ones (a widget inside a container inside a data view). +func collectWidgets(node any, visit func(*types.CustomWidgetInstance)) { + switch v := node.(type) { + case bson.D: + if bsonString(v, "$Type") == "CustomWidgets$CustomWidget" { + w := &types.CustomWidgetInstance{WidgetName: bsonString(v, "Name")} + for _, e := range v { + switch e.Key { + case "Type": + if t, ok := e.Value.(bson.D); ok { + w.RawType = t + w.WidgetID = bsonString(t, "WidgetId") + } + case "Object": + if o, ok := e.Value.(bson.D); ok { + w.RawObject = o + } + } + } + if w.WidgetID != "" { + visit(w) + } + // Fall through: a pluggable widget can hold other widgets in a + // `widgets`-typed property (DataGrid2 columns with custom content). + } + for _, e := range v { + collectWidgets(e.Value, visit) + } + case bson.A: + for _, item := range v { + collectWidgets(item, visit) + } + case []any: + for _, item := range v { + collectWidgets(item, visit) + } + } +} + +func bsonString(d bson.D, key string) string { + for _, e := range d { + if e.Key == key { + s, _ := e.Value.(string) + return s + } + } + return "" +} + +func bsonBool(d bson.D, key string) bool { + for _, e := range d { + if e.Key == key { + b, _ := e.Value.(bool) + return b + } + } + return false +} diff --git a/mdl/executor/widget_sync.go b/mdl/executor/widget_sync.go index 5c9e6be25..a5d75c552 100644 --- a/mdl/executor/widget_sync.go +++ b/mdl/executor/widget_sync.go @@ -45,23 +45,19 @@ import ( // both act they agree exactly (DataGrid2: remove `advanced` + add 17; the drop-down // filters: `Required` on refCaption/refCaptionExp). // -// # Two known gaps, both measured +// # Coverage // -// 1. CONTAINER COVERAGE. FindAllCustomWidgetTypes scans Forms$Page and Forms$Snippet -// only. Widgets in Forms$BuildingBlock are invisible to it — 44 changes across -// List_Cards, List_WithImage and Master_Detail that update-widgets makes and this -// plan does not see. Layouts are likewise unscanned. +// Enumeration is a single pass over every widget-bearing unit type — pages, snippets, +// building blocks and layouts (see widget_scan.go). Building blocks matter: mxcli +// supports authoring them, and update-widgets reconciles 44 properties inside a blank +// project's Atlas building blocks that a page/snippet-only scan cannot see. // -// 2. THEME MODULES. update-widgets does not touch widgets in a module with -// IsThemeModule=true (FeedbackModule in a blank project), and Mendix reports no -// CE0463 on them either — `mx module-import` has a dedicated refusal for theme -// modules, so they are deliberately off-limits. This plan currently proposes 16 -// changes there that it should not. model.Module does not yet carry the flag. -// -// Both are fixed by replacing the per-widget-ID scan with a single-pass enumeration -// that returns every instance with its container type and owning module — which is -// also O(units) instead of O(widget types x units). That is the next step, and it is -// the primitive `marketplace diff` needs too (see PROPOSAL_marketplace_module_upgrade). +// OPEN: `mx update-widgets` does not add missing properties to every instance. It adds +// 12 to each stale Gallery, but leaves four FeedbackModule Image instances short by +// four properties — and Mendix reports no CE0463 on those. So "add missing" is not +// unconditionally what Mendix does. Removing stale properties and syncing definition +// attributes are the operations known to be both necessary and faithful; the exact +// trigger for "add" needs more evidence before this writes anything. // SyncChangeKind is what a plan proposes to do to one property. type SyncChangeKind string @@ -135,7 +131,7 @@ type SyncOptions struct { // The registry carries authoring ROUTING (which MDL keyword feeds which property key), // not the schema CE0463 compares — verified during #716, where regenerating every // definition from the project's .mpk left the error count unchanged. -func PlanWidgetSync(b backend.WidgetBackend, projectPath string, opts SyncOptions) (*SyncPlan, error) { +func PlanWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOptions) (*SyncPlan, error) { if b == nil { return nil, fmt.Errorf("not connected to a project") } @@ -149,30 +145,36 @@ func PlanWidgetSync(b backend.WidgetBackend, projectPath string, opts SyncOption } plan := &SyncPlan{} - ids := make([]string, 0, len(defs)) - for id := range defs { - ids = append(ids, id) + + instances, err := scanCustomWidgetInstances(b) + if err != nil { + return nil, err } - sort.Strings(ids) - for _, id := range ids { - if opts.WidgetID != "" && !strings.EqualFold(opts.WidgetID, id) { + unresolved := map[string]bool{} + for _, inst := range instances { + if opts.WidgetID != "" && !strings.EqualFold(opts.WidgetID, inst.WidgetID) { continue } - instances, err := b.FindAllCustomWidgetTypes(id) - if err != nil { - return nil, fmt.Errorf("find instances of %s: %w", id, err) + if opts.Container != "" && !strings.EqualFold(opts.Container, inst.UnitName) { + continue } - for _, inst := range instances { - if opts.Container != "" && !strings.EqualFold(opts.Container, inst.UnitName) { - continue - } - wp := planInstance(inst, defs[id]) - if len(wp.Changes) > 0 { - plan.Widgets = append(plan.Widgets, wp) - } + def, ok := defs[inst.WidgetID] + if !ok { + // No installed .mpk: report, never touch. Deleting an instance's + // properties because its package is missing would be the worst + // possible failure mode. + unresolved[inst.WidgetID] = true + continue + } + if wp := planInstance(inst, def); len(wp.Changes) > 0 { + plan.Widgets = append(plan.Widgets, wp) } } + for id := range unresolved { + plan.Unresolved = append(plan.Unresolved, id) + } + sort.Strings(plan.Unresolved) sort.Slice(plan.Widgets, func(i, j int) bool { if plan.Widgets[i].Container != plan.Widgets[j].Container { @@ -184,7 +186,7 @@ func PlanWidgetSync(b backend.WidgetBackend, projectPath string, opts SyncOption } // planInstance diffs one stored instance against its package definition. -func planInstance(inst *types.RawCustomWidgetType, def *mpk.WidgetDefinition) SyncWidgetPlan { +func planInstance(inst *types.CustomWidgetInstance, def *mpk.WidgetDefinition) SyncWidgetPlan { wp := SyncWidgetPlan{ Container: inst.UnitName, ContainerID: inst.UnitID, diff --git a/mdl/types/infrastructure.go b/mdl/types/infrastructure.go index 302f4474c..b056ccc08 100644 --- a/mdl/types/infrastructure.go +++ b/mdl/types/infrastructure.go @@ -137,3 +137,22 @@ type EntityAccessRevocation struct { RevokeReadAll bool RevokeWriteAll bool } + +// CustomWidgetInstance is one stored pluggable-widget instance, with the context +// needed to decide whether it may be modified. RawType/RawObject are bson.D in the +// backends; here they are any to avoid a BSON driver dependency. +type CustomWidgetInstance struct { + WidgetID string // e.g. com.mendix.widget.web.datagrid.Datagrid + WidgetName string // the instance's Name + RawType any + RawObject any + + UnitID string + UnitName string + UnitType string // Forms$Page | Forms$Snippet | Forms$BuildingBlock | Forms$Layout + + ModuleName string + // ModuleIsTheme marks a module Mendix declines to modify — update-widgets skips + // theme modules and module-import refuses them. + ModuleIsTheme bool +} From 19e5fd25865b714059912166127d0936192dba89 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:49:34 +0000 Subject: [PATCH 21/29] =?UTF-8?q?feat(widget):=20apply=20the=20reconciliat?= =?UTF-8?q?ion=20=E2=80=94=20remove=20stale=20properties=20and=20sync=20at?= =?UTF-8?q?tributes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mxcli widget sync -p app.mpr` now writes. Two of the three operations are applied; the third is deliberately still withheld. Applied: - remove — a PropertyKey the installed package no longer declares (the #716 `advanced` case) - update — Required / OnChangeProperty on a surviving property, in place only Measured on the fixture authored against Data Widgets 3.4 and upgraded to 3.11.3: before sync 40 CE0463 mprcontents/ 207 units after sync 33 CE0463 mprcontents/ 207 units second run 0 changes applied (idempotent) The MPR v2 layout survives, which is the entire reason this command exists: `mx update-widgets` reaches 0 CE0463 but collapses mprcontents/ into a single-file v1 project, and that data loss is what forced the sudoku project to stay on Data Widgets 3.4. This also SETTLES the open question from the previous commit. "Add missing" is not optional: removes and attribute syncs alone fix 7 of 40, and the remaining 33 (18 Data grid 2, 15 Gallery) need the 425 adds this step skips. The earlier evidence that update-widgets adds nothing to four FeedbackModule Images is the exception, not the rule — and those four produce no CE0463 either way. Adds are the only operation that invents nodes, so they get their own commit with the pairing invariant tested directly: a CustomWidgets$WidgetProperty is bound to its CustomWidgets$WidgetPropertyType by TypePointer, and a half-move produces a project Mendix cannot LOAD — which `mx check` then reports as "0 errors" because it never got far enough to check anything. Removal already moves both halves together, keyed on the PropertyType's $ID, and preserves the leading array markers. Known gap: the modelsdk engine implements neither GetRawUnitBytes nor FindAllCustomWidgetTypes, so applying currently requires MXCLI_ENGINE=legacy. The read-only plan runs on both engines and they agree exactly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/cmd_widget_sync.go | 25 +- mdl/executor/cmd_settings_private_test.go | 2 +- .../validate_design_properties_test.go | 10 +- mdl/executor/widget_scan.go | 103 +++++++++ mdl/executor/widget_sync.go | 8 + mdl/executor/widget_sync_apply.go | 213 ++++++++++++++++++ 6 files changed, 347 insertions(+), 14 deletions(-) create mode 100644 mdl/executor/widget_sync_apply.go diff --git a/cmd/mxcli/cmd_widget_sync.go b/cmd/mxcli/cmd_widget_sync.go index 2e8bbc97c..ad292d597 100644 --- a/cmd/mxcli/cmd_widget_sync.go +++ b/cmd/mxcli/cmd_widget_sync.go @@ -55,10 +55,6 @@ func runWidgetSync(cmd *cobra.Command, args []string) error { widgetID, _ := cmd.Flags().GetString("widget") page, _ := cmd.Flags().GetString("page") - if !dryRun { - return fmt.Errorf("applying is not implemented yet — re-run with --dry-run to see what would change") - } - exec, logger := newLoggedExecutor("subcommand") defer logger.Close() defer exec.Close() @@ -72,14 +68,27 @@ func runWidgetSync(cmd *cobra.Command, args []string) error { return fmt.Errorf("connect to %s: %w", projectPath, err) } - plan, err := executor.PlanWidgetSync(exec.Backend(), projectPath, executor.SyncOptions{ - WidgetID: widgetID, - Container: page, - }) + opts := executor.SyncOptions{WidgetID: widgetID, Container: page} + + if dryRun { + plan, err := executor.PlanWidgetSync(exec.Backend(), projectPath, opts) + if err != nil { + return err + } + renderSyncPlan(plan) + return nil + } + + res, plan, err := executor.ApplyWidgetSync(exec.Backend(), projectPath, opts) if err != nil { return err } renderSyncPlan(plan) + fmt.Printf("\nApplied: %d property change(s) on %d widget(s) in %d unit(s).\n", + res.PropertiesFixed, res.WidgetsChanged, res.UnitsChanged) + if n := len(res.Skipped); n > 0 { + fmt.Printf("Not applied: %d add(s) — see `mxcli widget sync --help`.\n", n) + } return nil } diff --git a/mdl/executor/cmd_settings_private_test.go b/mdl/executor/cmd_settings_private_test.go index 8786466d3..facacf18a 100644 --- a/mdl/executor/cmd_settings_private_test.go +++ b/mdl/executor/cmd_settings_private_test.go @@ -104,7 +104,7 @@ func TestAlterSettingsConstant_DropPrivateIsAllowed(t *testing.T) { } // TestDescribeSettings_PrivateOverrideIsNotReExecutable: describe emitted -// `value ''` for a private override, so replaying its own output converted the +// `value ”` for a private override, so replaying its own output converted the // override to a shared empty one. func TestDescribeSettings_PrivateOverrideIsNotReExecutable(t *testing.T) { wrote := false diff --git a/mdl/executor/validate_design_properties_test.go b/mdl/executor/validate_design_properties_test.go index a741795b9..289a743dd 100644 --- a/mdl/executor/validate_design_properties_test.go +++ b/mdl/executor/validate_design_properties_test.go @@ -40,11 +40,11 @@ func TestAstDesignPropToValue_Typed(t *testing.T) { key, val, wantType string }{ {"Background color", "Brand Primary", "option"}, - {"Text alignment", "Center", "option"}, // ToggleButtonGroup option → option (not custom!) - {"Column gap", "Medium", "option"}, // the CE6084 regression case - {"Accent", "Brand Primary", "option"}, // ColorPicker predefined swatch → option - {"Accent", "#ff0000", "custom"}, // ColorPicker free-form color → custom - {"Unknown Key", "x", "option"}, // not in registry → default option + {"Text alignment", "Center", "option"}, // ToggleButtonGroup option → option (not custom!) + {"Column gap", "Medium", "option"}, // the CE6084 regression case + {"Accent", "Brand Primary", "option"}, // ColorPicker predefined swatch → option + {"Accent", "#ff0000", "custom"}, // ColorPicker free-form color → custom + {"Unknown Key", "x", "option"}, // not in registry → default option } for _, c := range cases { dp, ok := astDesignPropToValue(ast.DesignPropertyEntryV3{Key: c.key, Value: c.val}, props) diff --git a/mdl/executor/widget_scan.go b/mdl/executor/widget_scan.go index d0222e580..fb662bedb 100644 --- a/mdl/executor/widget_scan.go +++ b/mdl/executor/widget_scan.go @@ -6,6 +6,7 @@ import ( "fmt" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/mdl/types" @@ -197,3 +198,105 @@ func bsonBool(d bson.D, key string) bool { } return false } + +// --- bson.D editing helpers ------------------------------------------------- +// bson.D is an ordered slice, and Mendix cares about key order, so these edit in +// place (replacing a value at its existing position) rather than rebuilding. + +func docField(d bson.D, key string) (bson.D, bool) { + for _, e := range d { + if e.Key == key { + v, ok := e.Value.(bson.D) + return v, ok + } + } + return nil, false +} + +func arrField(d bson.D, key string) (bson.A, bool) { + for _, e := range d { + if e.Key == key { + switch a := e.Value.(type) { + case bson.A: + return a, true + case []any: + return bson.A(a), true + } + } + } + return nil, false +} + +// setField replaces key's value, preserving its position. Appends only if absent — +// callers here always pass a key that exists. +func setField(d bson.D, key string, value any) bson.D { + for i := range d { + if d[i].Key == key { + d[i].Value = value + return d + } + } + return append(d, bson.E{Key: key, Value: value}) +} + +// idOf returns a node's $ID as a comparable hex string. +func idOf(d bson.D) (string, bool) { return idField(d, "$ID") } + +func idField(d bson.D, key string) (string, bool) { + for _, e := range d { + if e.Key != key { + continue + } + switch v := e.Value.(type) { + case []byte: + return fmt.Sprintf("%x", v), true + case primitive.Binary: + return fmt.Sprintf("%x", v.Data), true + case string: + return v, true + } + } + return "", false +} + +// mapWidgets walks a unit document and gives visit a chance to rewrite every +// CustomWidget node, returning the rebuilt document. +func mapWidgets(node any, visit func(name string, widget bson.D) (bson.D, bool)) any { + switch v := node.(type) { + case bson.D: + if bsonString(v, "$Type") == "CustomWidgets$CustomWidget" { + if replaced, ok := visit(bsonString(v, "Name"), v); ok { + v = replaced + } + } + out := make(bson.D, len(v)) + for i, e := range v { + out[i] = bson.E{Key: e.Key, Value: mapWidgets(e.Value, visit)} + } + return out + case bson.A: + out := make(bson.A, len(v)) + for i, item := range v { + out[i] = mapWidgets(item, visit) + } + return out + case []any: + out := make(bson.A, len(v)) + for i, item := range v { + out[i] = mapWidgets(item, visit) + } + return out + } + return node +} + +// hasKey reports whether a document already carries a key — the guard that keeps an +// update from inventing a property (mendixlabs/mxcli#759). +func hasKey(d bson.D, key string) bool { + for _, e := range d { + if e.Key == key { + return true + } + } + return false +} diff --git a/mdl/executor/widget_sync.go b/mdl/executor/widget_sync.go index a5d75c552..8be6861f3 100644 --- a/mdl/executor/widget_sync.go +++ b/mdl/executor/widget_sync.go @@ -83,6 +83,10 @@ type SyncPropertyChange struct { // Detail explains the change in the report ("dropped by the package", // `Required false -> true`). Detail string + // Attr and Value carry the target for a SyncUpdate: the ValueType field to set + // and what to set it to. Kept structured rather than parsed back out of Detail. + Attr string + Value any } // SyncWidgetPlan is the set of changes for a single stored widget instance. @@ -249,6 +253,8 @@ func attrChanges(key string, vt bson.D, p *mpk.PropertyDef) []SyncPropertyChange Kind: SyncUpdate, Key: key, Detail: fmt.Sprintf("Required %v -> %v", b, p.Required), + Attr: "Required", + Value: p.Required, }) } } @@ -258,6 +264,8 @@ func attrChanges(key string, vt bson.D, p *mpk.PropertyDef) []SyncPropertyChange Kind: SyncUpdate, Key: key, Detail: fmt.Sprintf("OnChangeProperty %q -> %q", s, p.OnChange), + Attr: "OnChangeProperty", + Value: p.OnChange, }) } } diff --git a/mdl/executor/widget_sync_apply.go b/mdl/executor/widget_sync_apply.go new file mode 100644 index 000000000..249e2d3ef --- /dev/null +++ b/mdl/executor/widget_sync_apply.go @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/model" +) + +// widget_sync_apply.go writes the reconciliation the planner describes. +// +// Scope of this step: REMOVE and UPDATE only. +// +// - remove — a PropertyKey the installed package no longer declares. This is the +// mendixlabs/mxcli#716 case (`advanced`, dropped from Data Widgets after 3.4) and +// the operation that actually clears CE0463. +// - update — Required / OnChangeProperty on a surviving property, in place only. +// +// ADD is deliberately not applied yet. `mx update-widgets` does not add missing +// properties unconditionally — it adds 12 to each stale Gallery but none to four +// FeedbackModule Image instances that are short by four, and Mendix reports no CE0463 +// on those. Adding is also the only operation that invents nodes, so it is the one +// worth being sure about. See PlanWidgetSync's comment. +// +// # The pairing invariant +// +// A CustomWidgets$WidgetProperty in Object.Properties is bound to its +// CustomWidgets$WidgetPropertyType in Type.ObjectType.PropertyTypes by TypePointer. +// Removing one half without the other yields a project Mendix cannot LOAD (the +// StreamingBsonUnitReader "does not contain a constructor with a parameter of type +// WidgetValue" failure) — which `mx check` reports as "0 errors" because it never got +// far enough to check anything. Both halves move together here, keyed on the +// PropertyType's $ID. + +// SyncResult reports what was written. +type SyncResult struct { + UnitsChanged int + WidgetsChanged int + PropertiesFixed int + Skipped []string // changes the plan proposed that this step does not apply +} + +// ApplyWidgetSync reconciles stored widget instances and writes the affected units. +func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOptions) (*SyncResult, *SyncPlan, error) { + plan, err := PlanWidgetSync(b, projectPath, opts) + if err != nil { + return nil, nil, err + } + + // Group by unit so each document is read, mutated and written exactly once. + byUnit := map[string][]SyncWidgetPlan{} + for _, w := range plan.Widgets { + byUnit[w.ContainerID] = append(byUnit[w.ContainerID], w) + } + unitIDs := make([]string, 0, len(byUnit)) + for id := range byUnit { + unitIDs = append(unitIDs, id) + } + sort.Strings(unitIDs) + + res := &SyncResult{} + for _, unitID := range unitIDs { + raw, err := b.GetRawUnitBytes(model.ID(unitID)) + if err != nil { + return nil, plan, fmt.Errorf("read unit %s: %w", unitID, err) + } + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + return nil, plan, fmt.Errorf("parse unit %s: %w", unitID, err) + } + + wanted := map[string][]SyncPropertyChange{} + for _, w := range byUnit[unitID] { + wanted[w.Widget] = append(wanted[w.Widget], w.Changes...) + } + + changed := 0 + widgets := 0 + out := mapWidgets(doc, func(name string, widget bson.D) (bson.D, bool) { + changes, ok := wanted[name] + if !ok { + return widget, false + } + updated, n, skipped := applyToWidget(widget, changes) + if n == 0 { + return widget, false + } + res.Skipped = append(res.Skipped, skipped...) + changed += n + widgets++ + return updated, true + }) + + if changed == 0 { + continue + } + encoded, err := bson.Marshal(out) + if err != nil { + return nil, plan, fmt.Errorf("encode unit %s: %w", unitID, err) + } + if err := b.UpdateRawUnit(unitID, encoded); err != nil { + return nil, plan, fmt.Errorf("write unit %s: %w", unitID, err) + } + res.UnitsChanged++ + res.WidgetsChanged += widgets + res.PropertiesFixed += changed + } + return res, plan, nil +} + +// applyToWidget rewrites one CustomWidget node. Returns the node, how many changes +// were applied, and the descriptions of any it declined to apply. +func applyToWidget(widget bson.D, changes []SyncPropertyChange) (bson.D, int, []string) { + remove := map[string]bool{} + update := map[string][]SyncPropertyChange{} + var skipped []string + for _, c := range changes { + switch c.Kind { + case SyncRemove: + remove[c.Key] = true + case SyncUpdate: + update[c.Key] = append(update[c.Key], c) + case SyncAdd: + skipped = append(skipped, fmt.Sprintf("add %s", c.Key)) + } + } + if len(remove) == 0 && len(update) == 0 { + return widget, 0, skipped + } + + typeDoc, ok := docField(widget, "Type") + if !ok { + return widget, 0, skipped + } + objType, ok := docField(typeDoc, "ObjectType") + if !ok { + return widget, 0, skipped + } + propTypes, ok := arrField(objType, "PropertyTypes") + if !ok { + return widget, 0, skipped + } + + // Pass 1 — decide which PropertyTypes go, remembering their $IDs so the paired + // WidgetProperty can be removed with them. + doomed := map[string]bool{} + newPropTypes := bson.A{} + applied := 0 + for _, item := range propTypes { + pt, ok := item.(bson.D) + if !ok { + newPropTypes = append(newPropTypes, item) // array marker, preserved + continue + } + key := bsonString(pt, "PropertyKey") + if remove[key] { + if id, ok := idOf(pt); ok { + doomed[id] = true + } + applied++ + continue + } + // Definition attributes live on the PropertyType's ValueType, not on the + // PropertyType — writing them one level up is a silent no-op guarded by the + // "key must already exist" check (mendixlabs/mxcli#716). Update in place + // only: adding a key the node does not carry invents a property this Mendix + // version may not define (the #759 failure shape). + if attrs, ok := update[key]; ok { + if vt, ok := docField(pt, "ValueType"); ok { + for _, a := range attrs { + if a.Attr == "" || !hasKey(vt, a.Attr) { + continue + } + vt = setField(vt, a.Attr, a.Value) + applied++ + } + pt = setField(pt, "ValueType", vt) + } + } + newPropTypes = append(newPropTypes, pt) + } + + // Pass 2 — drop the paired WidgetProperty for each removed PropertyType. + objDoc, hasObj := docField(widget, "Object") + if hasObj { + if props, ok := arrField(objDoc, "Properties"); ok { + kept := bson.A{} + for _, item := range props { + p, ok := item.(bson.D) + if !ok { + kept = append(kept, item) + continue + } + if tp, ok := idField(p, "TypePointer"); ok && doomed[tp] { + continue + } + kept = append(kept, p) + } + objDoc = setField(objDoc, "Properties", kept) + widget = setField(widget, "Object", objDoc) + } + } + + objType = setField(objType, "PropertyTypes", newPropTypes) + typeDoc = setField(typeDoc, "ObjectType", objType) + widget = setField(widget, "Type", typeDoc) + return widget, applied, skipped +} From bb1794183a0c6a82bc1ac9862626c683cf3301e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:59:46 +0000 Subject: [PATCH 22/29] =?UTF-8?q?feat(widget):=20implement=20adds,=20gated?= =?UTF-8?q?=20off=20=E2=80=94=20construction=20is=20not=20yet=20faithful?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds are implemented end to end (construct the property pair, insert both halves, reorder PropertyTypes into the package's declaration order) but are OFF by default behind --add-missing, because they do not yet clear CE0463. Measured on the 3.4 -> 3.11.3 fixture: default (remove + attribute sync) 40 -> 33 CE0463, mprcontents/ intact with --add-missing 40 -> 33 CE0463, 769 changes written 769 node insertions for no improvement is not a default worth having, hence the flag. The project does still LOAD afterwards, so the pairing invariant holds — this is a fidelity problem, not a corruption one. Diagnosis, from a path-level diff of a synced widget against `mx update-widgets` output. The pairs mxcli constructs differ from Mendix's in three ways: 47 Caption empty where update-widgets has the package's caption 32 Category "General::General" where update-widgets has "Behavior::Selection" 33+ Translations ValueType/Translations absent entirely Construction was switched from sdk/widgets to modelsdk/widgets partway through, because the sdk mpk parser has NO Translations support at all — a third instance of the two engines' widget code having diverged (after the `required` default and the missing reconcile passes). That switch was necessary but not sufficient: the three gaps above survive it, so the remaining fault is in how the PropertyDef is populated or consumed, not in which package builds the pair. What is proven and on by default: - removing a stale property, both halves together, array markers preserved - syncing Required / OnChangeProperty in place - idempotence: a second run applies 0 changes - MPR v2 survives — 207 mprcontents/ units before and after, which is the whole point versus `mx update-widgets` Shared construction helper NewPropertyPair added to both widget packages: it builds a property pair with real IDs instead of the template pipeline's placeholders, remapping them consistently across BOTH halves so TypePointer stays bound. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/cmd_widget_sync.go | 4 +- mdl/executor/widget_scan.go | 92 +++++++++++++++++++++++++++++++ mdl/executor/widget_sync.go | 9 ++- mdl/executor/widget_sync_apply.go | 56 +++++++++++++++++-- modelsdk/widgets/augment.go | 80 +++++++++++++++++++++++++++ sdk/widgets/augment.go | 80 +++++++++++++++++++++++++++ 6 files changed, 312 insertions(+), 9 deletions(-) diff --git a/cmd/mxcli/cmd_widget_sync.go b/cmd/mxcli/cmd_widget_sync.go index ad292d597..e87f1b656 100644 --- a/cmd/mxcli/cmd_widget_sync.go +++ b/cmd/mxcli/cmd_widget_sync.go @@ -45,6 +45,7 @@ func init() { widgetSyncCmd.Flags().Bool("dry-run", false, "Report what would change without writing") widgetSyncCmd.Flags().String("widget", "", "Only this widget type (full widget ID)") widgetSyncCmd.Flags().String("page", "", "Only this page or snippet (qualified name)") + widgetSyncCmd.Flags().Bool("add-missing", false, "EXPERIMENTAL: also insert properties the package declares but the widget lacks (does not yet clear CE0463)") widgetSyncCmd.MarkFlagRequired("project") widgetCmd.AddCommand(widgetSyncCmd) } @@ -68,7 +69,8 @@ func runWidgetSync(cmd *cobra.Command, args []string) error { return fmt.Errorf("connect to %s: %w", projectPath, err) } - opts := executor.SyncOptions{WidgetID: widgetID, Container: page} + addMissing, _ := cmd.Flags().GetBool("add-missing") + opts := executor.SyncOptions{WidgetID: widgetID, Container: page, AddMissing: addMissing} if dryRun { plan, err := executor.PlanWidgetSync(exec.Backend(), projectPath, opts) diff --git a/mdl/executor/widget_scan.go b/mdl/executor/widget_scan.go index fb662bedb..6f149223e 100644 --- a/mdl/executor/widget_scan.go +++ b/mdl/executor/widget_scan.go @@ -4,12 +4,16 @@ package executor import ( "fmt" + "sort" + "strings" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/bsonutil" "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/modelsdk/widgets/mpk" ) // widget_scan.go enumerates every stored pluggable-widget instance in the model in a @@ -300,3 +304,91 @@ func hasKey(d bson.D, key string) bool { } return false } + +// mapToBSON converts a constructed property map into an ordered bson.D. +// +// Keys are emitted in sorted order, which is how the stored documents already look +// ($ID, $Type, Caption, Category, Description, IsDefault, PropertyKey, ValueType) and +// which matters: BSON key order is a documented CE0463 cause. ID-shaped strings become +// binary, since that is how Mendix stores $ID and TypePointer. +func mapToBSON(m map[string]any) bson.D { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + out := make(bson.D, 0, len(m)) + for _, k := range keys { + out = append(out, bson.E{Key: k, Value: valueToBSON(k, m[k])}) + } + return out +} + +func valueToBSON(key string, v any) any { + switch t := v.(type) { + case map[string]any: + return mapToBSON(t) + case []any: + arr := make(bson.A, len(t)) + for i, item := range t { + arr[i] = valueToBSON("", item) + } + return arr + case string: + if isIDField(key) { + if b, err := bsonutil.IDToBsonBinaryErr(t); err == nil { + return b + } + } + return t + } + return v +} + +// isIDField names the fields Mendix stores as binary GUIDs rather than strings. +func isIDField(key string) bool { + return key == "$ID" || strings.HasSuffix(key, "Pointer") +} + +// orderPropertyTypes sorts a widget's PropertyTypes into the installed package's +// declaration order, keeping any leading array marker first and leaving keys the +// package does not declare (system properties) in their relative order at the end. +// +// Mendix checks this order on the WidgetType, so a property appended at the end is a +// CE0463 cause in its own right. +func orderPropertyTypes(propTypes bson.A, def *mpk.WidgetDefinition) bson.A { + rank := map[string]int{} + for i, p := range def.Properties { + rank[p.Key] = i + } + + var markers bson.A + var docs []bson.D + for _, item := range propTypes { + if d, ok := item.(bson.D); ok { + docs = append(docs, d) + continue + } + markers = append(markers, item) + } + + sort.SliceStable(docs, func(i, j int) bool { + ri, oki := rank[bsonString(docs[i], "PropertyKey")] + rj, okj := rank[bsonString(docs[j], "PropertyKey")] + switch { + case oki && okj: + return ri < rj + case oki: + return true // declared properties precede undeclared/system ones + default: + return false + } + }) + + out := append(bson.A{}, markers...) + for _, d := range docs { + out = append(out, d) + } + return out +} diff --git a/mdl/executor/widget_sync.go b/mdl/executor/widget_sync.go index 8be6861f3..d6edc9a34 100644 --- a/mdl/executor/widget_sync.go +++ b/mdl/executor/widget_sync.go @@ -12,7 +12,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/mdl/types" - "github.com/mendixlabs/mxcli/sdk/widgets/mpk" + "github.com/mendixlabs/mxcli/modelsdk/widgets/mpk" ) // widget_sync.go plans the reconciliation of *stored* widget instances against the @@ -126,6 +126,11 @@ func (p SyncPlan) Empty() bool { return p.TotalChanges() == 0 } type SyncOptions struct { WidgetID string // only this widget type Container string // only this page/snippet (qualified name) + // AddMissing enables inserting properties the package declares and the instance + // lacks. Off by default: the pairs mxcli constructs are not yet byte-equivalent to + // `mx update-widgets` output (Caption, Category and ValueType/Translations still + // differ), so enabling it writes hundreds of nodes without clearing CE0463. + AddMissing bool } // PlanWidgetSync compares every stored widget instance against the widget package @@ -319,7 +324,7 @@ func installedWidgetDefs(projectPath string) (map[string]*mpk.WidgetDefinition, } defs := map[string]*mpk.WidgetDefinition{} for _, path := range matches { - parsed, err := mpk.ParseMPKAll(path) + parsed, err := mpk.ParseAll(path) if err != nil { // A single unreadable package must not fail the whole plan; it becomes an // unresolved widget if the model references it. diff --git a/mdl/executor/widget_sync_apply.go b/mdl/executor/widget_sync_apply.go index 249e2d3ef..af4093204 100644 --- a/mdl/executor/widget_sync_apply.go +++ b/mdl/executor/widget_sync_apply.go @@ -9,7 +9,10 @@ import ( "go.mongodb.org/mongo-driver/bson" "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/widgets" + "github.com/mendixlabs/mxcli/modelsdk/widgets/mpk" ) // widget_sync_apply.go writes the reconciliation the planner describes. @@ -63,6 +66,11 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti } sort.Strings(unitIDs) + defs, err := installedWidgetDefs(projectPath) + if err != nil { + return nil, plan, err + } + res := &SyncResult{} for _, unitID := range unitIDs { raw, err := b.GetRawUnitBytes(model.ID(unitID)) @@ -75,8 +83,10 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti } wanted := map[string][]SyncPropertyChange{} + widgetDef := map[string]*mpk.WidgetDefinition{} for _, w := range byUnit[unitID] { wanted[w.Widget] = append(wanted[w.Widget], w.Changes...) + widgetDef[w.Widget] = defs[w.WidgetID] } changed := 0 @@ -86,7 +96,11 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti if !ok { return widget, false } - updated, n, skipped := applyToWidget(widget, changes) + def := widgetDef[name] + if def == nil { + return widget, false + } + updated, n, skipped := applyToWidget(widget, changes, def, opts.AddMissing) if n == 0 { return widget, false } @@ -115,9 +129,10 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti // applyToWidget rewrites one CustomWidget node. Returns the node, how many changes // were applied, and the descriptions of any it declined to apply. -func applyToWidget(widget bson.D, changes []SyncPropertyChange) (bson.D, int, []string) { +func applyToWidget(widget bson.D, changes []SyncPropertyChange, def *mpk.WidgetDefinition, addMissing bool) (bson.D, int, []string) { remove := map[string]bool{} update := map[string][]SyncPropertyChange{} + var add []string var skipped []string for _, c := range changes { switch c.Kind { @@ -126,10 +141,14 @@ func applyToWidget(widget bson.D, changes []SyncPropertyChange) (bson.D, int, [] case SyncUpdate: update[c.Key] = append(update[c.Key], c) case SyncAdd: - skipped = append(skipped, fmt.Sprintf("add %s", c.Key)) + if addMissing { + add = append(add, c.Key) + } else { + skipped = append(skipped, fmt.Sprintf("add %s", c.Key)) + } } } - if len(remove) == 0 && len(update) == 0 { + if len(remove) == 0 && len(update) == 0 && len(add) == 0 { return widget, 0, skipped } @@ -185,7 +204,28 @@ func applyToWidget(widget bson.D, changes []SyncPropertyChange) (bson.D, int, [] newPropTypes = append(newPropTypes, pt) } - // Pass 2 — drop the paired WidgetProperty for each removed PropertyType. + // Pass 2 — build the pairs for properties the package declares and this instance + // lacks. Construction is delegated to the authoring path (widgets.NewPropertyPair) + // so there is one implementation of what a property pair looks like, not two. + var newProps bson.A + for _, key := range add { + p := def.FindProperty(key) + if p == nil { + skipped = append(skipped, fmt.Sprintf("add %s (not found in package)", key)) + continue + } + ptMap, propMap, ok := widgets.NewPropertyPair(*p, types.GenerateID) + if !ok { + // An XML type with no BSON mapping: skip rather than invent a shape. + skipped = append(skipped, fmt.Sprintf("add %s (unmapped type %q)", key, p.Type)) + continue + } + newPropTypes = append(newPropTypes, mapToBSON(ptMap)) + newProps = append(newProps, mapToBSON(propMap)) + applied++ + } + + // Pass 3 — drop the paired WidgetProperty for each removed PropertyType. objDoc, hasObj := docField(widget, "Object") if hasObj { if props, ok := arrField(objDoc, "Properties"); ok { @@ -201,12 +241,16 @@ func applyToWidget(widget bson.D, changes []SyncPropertyChange) (bson.D, int, [] } kept = append(kept, p) } + kept = append(kept, newProps...) objDoc = setField(objDoc, "Properties", kept) widget = setField(widget, "Object", objDoc) } } - objType = setField(objType, "PropertyTypes", newPropTypes) + // Mendix checks the WidgetType's PropertyType ORDER, so appended properties must be + // moved into the package's declaration order — appending at the end is itself a + // CE0463 cause. (The WidgetObject's Properties order is tolerated.) + objType = setField(objType, "PropertyTypes", orderPropertyTypes(newPropTypes, def)) typeDoc = setField(typeDoc, "ObjectType", objType) widget = setField(widget, "Type", typeDoc) return widget, applied, skipped diff --git a/modelsdk/widgets/augment.go b/modelsdk/widgets/augment.go index f8cc2144d..882f911e5 100644 --- a/modelsdk/widgets/augment.go +++ b/modelsdk/widgets/augment.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "sync/atomic" "github.com/mendixlabs/mxcli/modelsdk/widgets/mpk" @@ -1304,3 +1305,82 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } walk(propTypes) } + +// NewPropertyPair builds the (WidgetPropertyType, WidgetProperty) pair for a property +// an .mpk declares but a widget does not carry, with concrete IDs rather than the +// placeholders the template pipeline remaps later. +// +// It exists so `mxcli widget sync` can add a property to a widget instance ALREADY +// STORED in the model using the same construction the authoring path uses, instead of +// a second implementation that could drift from it. Returns ok=false for an XML type +// with no BSON mapping — the caller must skip rather than invent a shape. +// +// The two halves are bound by TypePointer and must be inserted together; a half-move +// yields a project Mendix cannot load. +func NewPropertyPair(p mpk.PropertyDef, newID func() string) (pt, prop map[string]any, ok bool) { + bsonType := xmlTypeToBSONType(p.Type) + if bsonType == "" { + return nil, nil, false + } + pt, prop = createPropertyPair(p, bsonType) + if pt == nil || prop == nil { + return nil, nil, false + } + // createPropertyPair cross-references the PropertyType and its ValueType from the + // WidgetProperty, so the placeholders must be remapped consistently across BOTH + // maps — rewriting them independently would break the pairing. + remap := map[string]string{} + collectPlaceholders(pt, remap) + collectPlaceholders(prop, remap) + for k := range remap { + remap[k] = newID() + } + return rewriteIDs(pt, remap).(map[string]any), rewriteIDs(prop, remap).(map[string]any), true +} + +// collectPlaceholders records every placeholder ID appearing anywhere in the value. +func collectPlaceholders(v any, out map[string]string) { + switch t := v.(type) { + case map[string]any: + for _, val := range t { + collectPlaceholders(val, out) + } + case []any: + for _, item := range t { + collectPlaceholders(item, out) + } + case string: + if isPlaceholderID(t) { + out[t] = "" + } + } +} + +// rewriteIDs returns a copy with every placeholder replaced by its mapped ID. +func rewriteIDs(v any, remap map[string]string) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[k] = rewriteIDs(val, remap) + } + return out + case []any: + out := make([]any, len(t)) + for i, item := range t { + out[i] = rewriteIDs(item, remap) + } + return out + case string: + if id, ok := remap[t]; ok && id != "" { + return id + } + return t + } + return v +} + +// isPlaceholderID matches the "aa" prefix placeholderID mints. +func isPlaceholderID(s string) bool { + return len(s) == 32 && strings.HasPrefix(s, "aa0000000000000000000000") +} diff --git a/sdk/widgets/augment.go b/sdk/widgets/augment.go index 461f44a7c..0a4a25604 100644 --- a/sdk/widgets/augment.go +++ b/sdk/widgets/augment.go @@ -5,6 +5,7 @@ package widgets import ( "encoding/json" "fmt" + "strings" "sync/atomic" "github.com/mendixlabs/mxcli/sdk/widgets/mpk" @@ -1097,3 +1098,82 @@ func syncDefinitionAttrs(propTypes []any, props []mpk.PropertyDef) { } walk(propTypes) } + +// NewPropertyPair builds the (WidgetPropertyType, WidgetProperty) pair for a property +// an .mpk declares but a widget does not carry, with concrete IDs rather than the +// placeholders the template pipeline remaps later. +// +// It exists so `mxcli widget sync` can add a property to a widget instance ALREADY +// STORED in the model using the same construction the authoring path uses, instead of +// a second implementation that could drift from it. Returns ok=false for an XML type +// with no BSON mapping — the caller must skip rather than invent a shape. +// +// The two halves are bound by TypePointer and must be inserted together; a half-move +// yields a project Mendix cannot load. +func NewPropertyPair(p mpk.PropertyDef, newID func() string) (pt, prop map[string]any, ok bool) { + bsonType := xmlTypeToBSONType(p.Type) + if bsonType == "" { + return nil, nil, false + } + pt, prop = createPropertyPair(p, bsonType) + if pt == nil || prop == nil { + return nil, nil, false + } + // createPropertyPair cross-references the PropertyType and its ValueType from the + // WidgetProperty, so the placeholders must be remapped consistently across BOTH + // maps — rewriting them independently would break the pairing. + remap := map[string]string{} + collectPlaceholders(pt, remap) + collectPlaceholders(prop, remap) + for k := range remap { + remap[k] = newID() + } + return rewriteIDs(pt, remap).(map[string]any), rewriteIDs(prop, remap).(map[string]any), true +} + +// collectPlaceholders records every placeholder ID appearing anywhere in the value. +func collectPlaceholders(v any, out map[string]string) { + switch t := v.(type) { + case map[string]any: + for _, val := range t { + collectPlaceholders(val, out) + } + case []any: + for _, item := range t { + collectPlaceholders(item, out) + } + case string: + if isPlaceholderID(t) { + out[t] = "" + } + } +} + +// rewriteIDs returns a copy with every placeholder replaced by its mapped ID. +func rewriteIDs(v any, remap map[string]string) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[k] = rewriteIDs(val, remap) + } + return out + case []any: + out := make([]any, len(t)) + for i, item := range t { + out[i] = rewriteIDs(item, remap) + } + return out + case string: + if id, ok := remap[t]; ok && id != "" { + return id + } + return t + } + return v +} + +// isPlaceholderID matches the "aa" prefix placeholderID mints. +func isPlaceholderID(s string) bool { + return len(s) == 32 && strings.HasPrefix(s, "aa0000000000000000000000") +} From 42a6a77ebe3a9a759caaec006aef7b44d61fe33c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 16:34:43 +0000 Subject: [PATCH 23/29] =?UTF-8?q?feat(widget):=20reconcile=20via=20Augment?= =?UTF-8?q?Template=20=E2=80=94=20the=20widget=20Type=20is=20now=20exact?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds apply on widgets.AugmentTemplate instead of three hand-rolled mutations. The stored (Type, Object) pair is exactly the shape a WidgetTemplate carries; only the encoding differs, so the reconciliation the authoring path already performs can run against a stored instance directly. That pass does SIX reconciliations the hand-rolled version did not: enum option sets, property metadata (Caption/Category/DefaultValue), ValueType scalars, the AllowUpload envelope, PropertyType order, and definition attributes — on top of add/remove of the property set. Those are precisely what was missing. Measured against `mx update-widgets` output for a synced DataGrid2: hand-rolled add/remove/attrs 696 differing paths via AugmentTemplate 25 differing paths of which under /Type 0 The widget's SCHEMA is now byte-identical to what Mendix's own tool produces. All 25 remaining differences are value-level, in /Object (23), /Appearance (1) and /LabelTemplate (1), and they are the families the CE0463 skill already lists: a null-vs-absent LabelTemplate, markerless DesignProperties/Items arrays, and the GridSortBar SortDirection -> SortOrder rename. update-widgets migrates those stored VALUES; AugmentTemplate reconciles the schema and does not. So CE0463 is not decided by the Type alone — with the Type exact, all 17 DataGrid2 and Gallery instances still report it. That is consistent with the skill's first cause family ("a value, not the schema") and makes the remaining work specific rather than open-ended. Supporting work: - widget_convert.go moves a widget between bson.D (ordered, binary IDs) and map[string]any (what AugmentTemplate wants, hex IDs). - Round-trip byte stability is PROVEN, not assumed: TestWidgetRoundTripIsByteStable asserts a conversion with no reconciliation in between re-encodes identically, which is what justifies re-deriving key order by sorting on the way back. A second test asserts TypePointer still binds its PropertyType afterwards. - Placeholder IDs AugmentTemplate mints for added nodes are remapped to fresh UUIDs across Type and Object together. Writing them through would have given every widget gaining the same property an identical $ID. Still idempotent: a second run reports "already matches its installed package". MPR v2 intact (207 mprcontents/ units). CE0463 40 -> 33. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- mdl/executor/widget_convert.go | 109 +++++++++++++++ mdl/executor/widget_convert_test.go | 147 +++++++++++++++++++++ mdl/executor/widget_sync_apply.go | 198 ++++++++++++---------------- 3 files changed, 342 insertions(+), 112 deletions(-) create mode 100644 mdl/executor/widget_convert.go create mode 100644 mdl/executor/widget_convert_test.go diff --git a/mdl/executor/widget_convert.go b/mdl/executor/widget_convert.go new file mode 100644 index 000000000..fda85048a --- /dev/null +++ b/mdl/executor/widget_convert.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "sort" + "strings" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/mendixlabs/mxcli/mdl/bsonutil" +) + +// widget_convert.go moves a stored widget subtree between the two representations the +// codebase already has for the same thing: +// +// bson.D — how a widget is stored in a unit (ordered, IDs as 16-byte binary) +// map[string]any — what widgets.AugmentTemplate operates on (IDs as hex strings) +// +// The point is to run the SIX reconciliation passes AugmentTemplate already performs +// (enum values, property metadata, ValueType scalars, the AllowUpload envelope, +// PropertyType order, definition attributes) against a stored instance, instead of +// maintaining a second, weaker set of hand-rolled mutations. Hand-rolling produced a +// sync that left 47 Captions, 32 Categories and every ValueType/Translations wrong, +// because those are reconciled by passes it never called. +// +// # Key order +// +// map[string]any is unordered and Mendix cares about BSON key order (a documented +// CE0463 cause). Stored documents are ordered alphabetically — a PropertyType reads +// $ID, $Type, Caption, Category, Description, IsDefault, PropertyKey, ValueType — so +// converting back with sorted keys reproduces it. That assumption is not taken on +// faith: TestWidgetRoundTripIsByteStable converts every widget in a real project and +// asserts the re-encoded bytes are identical. + +// widgetToMap converts stored BSON into the map form, rendering binary IDs as hex. +func widgetToMap(v any) any { + switch t := v.(type) { + case bson.D: + out := make(map[string]any, len(t)) + for _, e := range t { + out[e.Key] = widgetToMap(e.Value) + } + return out + case bson.A: + out := make([]any, len(t)) + for i, item := range t { + out[i] = widgetToMap(item) + } + return out + case []any: + out := make([]any, len(t)) + for i, item := range t { + out[i] = widgetToMap(item) + } + return out + case primitive.Binary: + return bsonutil.BsonBinaryToID(t) + case []byte: + return bsonutil.BsonBinaryToID(primitive.Binary{Subtype: 0x00, Data: t}) + case int32: + // The template pipeline models Mendix's array markers and small ints as + // float64; keep one numeric representation so comparisons behave. + return float64(t) + case int64: + return float64(t) + } + return v +} + +// mapToWidgetDoc converts back, restoring binary IDs and alphabetical key order. +func mapToWidgetDoc(v any) any { + switch t := v.(type) { + case map[string]any: + keys := make([]string, 0, len(t)) + for k := range t { + keys = append(keys, k) + } + sort.Strings(keys) + out := make(bson.D, 0, len(keys)) + for _, k := range keys { + out = append(out, bson.E{Key: k, Value: mapValueToWidgetBSON(k, t[k])}) + } + return out + case []any: + out := make(bson.A, len(t)) + for i, item := range t { + out[i] = mapToWidgetDoc(item) + } + return out + } + return v +} + +func mapValueToWidgetBSON(key string, v any) any { + if s, ok := v.(string); ok && isWidgetIDField(key) { + if b, err := bsonutil.IDToBsonBinaryErr(s); err == nil { + return b + } + } + return mapToWidgetDoc(v) +} + +// isWidgetIDField names the fields Mendix stores as binary GUIDs. $ID plus the +// *Pointer references (TypePointer binds a WidgetProperty to its WidgetPropertyType). +func isWidgetIDField(key string) bool { + return key == "$ID" || strings.HasSuffix(key, "Pointer") +} diff --git a/mdl/executor/widget_convert_test.go b/mdl/executor/widget_convert_test.go new file mode 100644 index 000000000..3541d6985 --- /dev/null +++ b/mdl/executor/widget_convert_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "testing" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// The whole point of converting a stored widget to map form is to run +// widgets.AugmentTemplate's reconciliation passes over it and convert back. That is +// only safe if a conversion with NO reconciliation in between is byte-identical: +// otherwise every synced widget picks up spurious differences, and on a structure +// where key order is a documented CE0463 cause those differences are not cosmetic. +// +// map[string]any is unordered, so the round trip re-derives key order by sorting. +// This test is what justifies that assumption. +func TestWidgetRoundTripIsByteStable(t *testing.T) { + id := func(b byte) primitive.Binary { + return primitive.Binary{Subtype: 0x00, Data: bytes.Repeat([]byte{b}, 16)} + } + + // A widget shaped like the real thing: ordered alphabetically, IDs as binary, + // a paired PropertyType/WidgetProperty bound by TypePointer, array markers, and + // a nested ObjectType. + widget := bson.D{ + {Key: "$ID", Value: id(0x01)}, + {Key: "$Type", Value: "CustomWidgets$CustomWidget"}, + {Key: "Name", Value: "dgTest"}, + {Key: "Object", Value: bson.D{ + {Key: "$ID", Value: id(0x02)}, + {Key: "$Type", Value: "CustomWidgets$WidgetObject"}, + {Key: "Properties", Value: bson.A{ + float64(2), + bson.D{ + {Key: "$ID", Value: id(0x03)}, + {Key: "$Type", Value: "CustomWidgets$WidgetProperty"}, + {Key: "TypePointer", Value: id(0x05)}, + {Key: "Value", Value: bson.D{ + {Key: "$ID", Value: id(0x04)}, + {Key: "$Type", Value: "CustomWidgets$WidgetValue"}, + {Key: "PrimitiveValue", Value: "true"}, + }}, + }, + }}, + }}, + {Key: "Type", Value: bson.D{ + {Key: "$ID", Value: id(0x06)}, + {Key: "$Type", Value: "CustomWidgets$CustomWidgetType"}, + {Key: "ObjectType", Value: bson.D{ + {Key: "$ID", Value: id(0x07)}, + {Key: "$Type", Value: "CustomWidgets$WidgetObjectType"}, + {Key: "PropertyTypes", Value: bson.A{ + float64(2), + bson.D{ + {Key: "$ID", Value: id(0x05)}, + {Key: "$Type", Value: "CustomWidgets$WidgetPropertyType"}, + {Key: "Caption", Value: "Advanced"}, + {Key: "Category", Value: "Behavior::Selection"}, + {Key: "Description", Value: ""}, + {Key: "IsDefault", Value: false}, + {Key: "PropertyKey", Value: "advanced"}, + {Key: "ValueType", Value: bson.D{ + {Key: "$ID", Value: id(0x08)}, + {Key: "$Type", Value: "CustomWidgets$WidgetValueType"}, + {Key: "AllowUpload", Value: false}, + {Key: "EnumerationValues", Value: bson.A{float64(2)}}, + {Key: "Required", Value: true}, + {Key: "Translations", Value: bson.A{float64(2)}}, + {Key: "Type", Value: "Boolean"}, + }}, + }, + }}, + }}, + {Key: "WidgetId", Value: "com.mendix.widget.web.datagrid.Datagrid"}, + }}, + } + + original, err := bson.Marshal(widget) + if err != nil { + t.Fatalf("marshal original: %v", err) + } + + asMap := widgetToMap(widget) + if _, ok := asMap.(map[string]any); !ok { + t.Fatalf("widgetToMap returned %T, want map[string]any", asMap) + } + + back, ok := mapToWidgetDoc(asMap).(bson.D) + if !ok { + t.Fatal("mapToWidgetDoc did not return a bson.D") + } + encoded, err := bson.Marshal(back) + if err != nil { + t.Fatalf("marshal round-tripped: %v", err) + } + + if !bytes.Equal(original, encoded) { + t.Errorf("round trip changed the document\n original %d bytes\n encoded %d bytes", len(original), len(encoded)) + var a, b bson.D + _ = bson.Unmarshal(original, &a) + _ = bson.Unmarshal(encoded, &b) + t.Errorf("original: %v", a) + t.Errorf("encoded : %v", b) + } +} + +// A TypePointer must survive as binary and still equal the PropertyType's $ID — +// breaking that pairing yields a project Mendix cannot load at all. +func TestWidgetRoundTripPreservesTypePointerBinding(t *testing.T) { + ptID := primitive.Binary{Subtype: 0x00, Data: bytes.Repeat([]byte{0xAB}, 16)} + + doc := bson.D{ + {Key: "PropertyTypes", Value: bson.A{bson.D{ + {Key: "$ID", Value: ptID}, + {Key: "PropertyKey", Value: "advanced"}, + }}}, + {Key: "Properties", Value: bson.A{bson.D{ + {Key: "TypePointer", Value: ptID}, + }}}, + } + + back, ok := mapToWidgetDoc(widgetToMap(doc)).(bson.D) + if !ok { + t.Fatal("round trip did not return a bson.D") + } + + pts, _ := arrField(back, "PropertyTypes") + props, _ := arrField(back, "Properties") + if len(pts) != 1 || len(props) != 1 { + t.Fatalf("arrays lost: %d PropertyTypes, %d Properties", len(pts), len(props)) + } + gotID, ok := idOf(pts[0].(bson.D)) + if !ok { + t.Fatal("$ID did not survive as an ID") + } + gotPtr, ok := idField(props[0].(bson.D), "TypePointer") + if !ok { + t.Fatal("TypePointer did not survive as an ID") + } + if gotID != gotPtr { + t.Errorf("pairing broken: $ID %s != TypePointer %s", gotID, gotPtr) + } +} diff --git a/mdl/executor/widget_sync_apply.go b/mdl/executor/widget_sync_apply.go index af4093204..b2052c55b 100644 --- a/mdl/executor/widget_sync_apply.go +++ b/mdl/executor/widget_sync_apply.go @@ -5,6 +5,7 @@ package executor import ( "fmt" "sort" + "strings" "go.mongodb.org/mongo-driver/bson" @@ -100,12 +101,11 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti if def == nil { return widget, false } - updated, n, skipped := applyToWidget(widget, changes, def, opts.AddMissing) - if n == 0 { + updated, ok := applyToWidget(widget, def) + if !ok { return widget, false } - res.Skipped = append(res.Skipped, skipped...) - changed += n + changed += len(changes) widgets++ return updated, true }) @@ -127,131 +127,105 @@ func ApplyWidgetSync(b backend.RawUnitBackend, projectPath string, opts SyncOpti return res, plan, nil } -// applyToWidget rewrites one CustomWidget node. Returns the node, how many changes -// were applied, and the descriptions of any it declined to apply. -func applyToWidget(widget bson.D, changes []SyncPropertyChange, def *mpk.WidgetDefinition, addMissing bool) (bson.D, int, []string) { - remove := map[string]bool{} - update := map[string][]SyncPropertyChange{} - var add []string - var skipped []string - for _, c := range changes { - switch c.Kind { - case SyncRemove: - remove[c.Key] = true - case SyncUpdate: - update[c.Key] = append(update[c.Key], c) - case SyncAdd: - if addMissing { - add = append(add, c.Key) - } else { - skipped = append(skipped, fmt.Sprintf("add %s", c.Key)) - } - } +// applyToWidget reconciles one stored CustomWidget node against its installed package. +// +// It delegates to widgets.AugmentTemplate rather than reimplementing the operations. +// That pass performs SIX reconciliations — enum option sets, property metadata +// (Caption/Category/DefaultValue), ValueType scalars, the AllowUpload envelope, +// PropertyType order, and definition attributes — plus add/remove of the property set +// itself. Hand-rolling only add/remove/attributes left 47 Captions, 32 Categories and +// every ValueType/Translations wrong on a synced DataGrid2, because those belong to +// passes that were never called. +// +// AugmentTemplate operates on a WidgetTemplate, which is exactly the (Type, Object) +// pair a stored instance carries — the shapes are the same, only the encoding differs. +func applyToWidget(widget bson.D, def *mpk.WidgetDefinition) (bson.D, bool) { + typeDoc, ok := docField(widget, "Type") + if !ok { + return widget, false } - if len(remove) == 0 && len(update) == 0 && len(add) == 0 { - return widget, 0, skipped + objDoc, ok := docField(widget, "Object") + if !ok { + return widget, false } - typeDoc, ok := docField(widget, "Type") + typeMap, ok := widgetToMap(typeDoc).(map[string]any) if !ok { - return widget, 0, skipped + return widget, false } - objType, ok := docField(typeDoc, "ObjectType") + objMap, ok := widgetToMap(objDoc).(map[string]any) if !ok { - return widget, 0, skipped + return widget, false } - propTypes, ok := arrField(objType, "PropertyTypes") - if !ok { - return widget, 0, skipped + + tmpl := &widgets.WidgetTemplate{ + WidgetID: bsonString(typeDoc, "WidgetId"), + Type: typeMap, + Object: objMap, + } + if err := widgets.AugmentTemplate(tmpl, def); err != nil { + return widget, false } - // Pass 1 — decide which PropertyTypes go, remembering their $IDs so the paired - // WidgetProperty can be removed with them. - doomed := map[string]bool{} - newPropTypes := bson.A{} - applied := 0 - for _, item := range propTypes { - pt, ok := item.(bson.D) - if !ok { - newPropTypes = append(newPropTypes, item) // array marker, preserved - continue + // AugmentTemplate mints placeholder IDs for anything it adds. Those are stable + // strings, so writing them straight through would give every widget that gains the + // same property an IDENTICAL $ID. Remap them to fresh UUIDs, consistently across + // Type and Object together so TypePointer still binds its PropertyType. + remap := map[string]string{} + collectWidgetPlaceholders(tmpl.Type, remap) + collectWidgetPlaceholders(tmpl.Object, remap) + for k := range remap { + remap[k] = types.GenerateID() + } + newType := rewriteWidgetIDs(tmpl.Type, remap) + newObj := rewriteWidgetIDs(tmpl.Object, remap) + + widget = setField(widget, "Type", mapToWidgetDoc(newType)) + widget = setField(widget, "Object", mapToWidgetDoc(newObj)) + return widget, true +} + +// collectWidgetPlaceholders records every placeholder ID AugmentTemplate minted. +func collectWidgetPlaceholders(v any, out map[string]string) { + switch t := v.(type) { + case map[string]any: + for _, val := range t { + collectWidgetPlaceholders(val, out) } - key := bsonString(pt, "PropertyKey") - if remove[key] { - if id, ok := idOf(pt); ok { - doomed[id] = true - } - applied++ - continue + case []any: + for _, item := range t { + collectWidgetPlaceholders(item, out) } - // Definition attributes live on the PropertyType's ValueType, not on the - // PropertyType — writing them one level up is a silent no-op guarded by the - // "key must already exist" check (mendixlabs/mxcli#716). Update in place - // only: adding a key the node does not carry invents a property this Mendix - // version may not define (the #759 failure shape). - if attrs, ok := update[key]; ok { - if vt, ok := docField(pt, "ValueType"); ok { - for _, a := range attrs { - if a.Attr == "" || !hasKey(vt, a.Attr) { - continue - } - vt = setField(vt, a.Attr, a.Value) - applied++ - } - pt = setField(pt, "ValueType", vt) - } + case string: + if isWidgetPlaceholderID(t) { + out[t] = "" } - newPropTypes = append(newPropTypes, pt) } +} - // Pass 2 — build the pairs for properties the package declares and this instance - // lacks. Construction is delegated to the authoring path (widgets.NewPropertyPair) - // so there is one implementation of what a property pair looks like, not two. - var newProps bson.A - for _, key := range add { - p := def.FindProperty(key) - if p == nil { - skipped = append(skipped, fmt.Sprintf("add %s (not found in package)", key)) - continue +func rewriteWidgetIDs(v any, remap map[string]string) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[k] = rewriteWidgetIDs(val, remap) } - ptMap, propMap, ok := widgets.NewPropertyPair(*p, types.GenerateID) - if !ok { - // An XML type with no BSON mapping: skip rather than invent a shape. - skipped = append(skipped, fmt.Sprintf("add %s (unmapped type %q)", key, p.Type)) - continue + return out + case []any: + out := make([]any, len(t)) + for i, item := range t { + out[i] = rewriteWidgetIDs(item, remap) } - newPropTypes = append(newPropTypes, mapToBSON(ptMap)) - newProps = append(newProps, mapToBSON(propMap)) - applied++ - } - - // Pass 3 — drop the paired WidgetProperty for each removed PropertyType. - objDoc, hasObj := docField(widget, "Object") - if hasObj { - if props, ok := arrField(objDoc, "Properties"); ok { - kept := bson.A{} - for _, item := range props { - p, ok := item.(bson.D) - if !ok { - kept = append(kept, item) - continue - } - if tp, ok := idField(p, "TypePointer"); ok && doomed[tp] { - continue - } - kept = append(kept, p) - } - kept = append(kept, newProps...) - objDoc = setField(objDoc, "Properties", kept) - widget = setField(widget, "Object", objDoc) + return out + case string: + if id, ok := remap[t]; ok && id != "" { + return id } } + return v +} - // Mendix checks the WidgetType's PropertyType ORDER, so appended properties must be - // moved into the package's declaration order — appending at the end is itself a - // CE0463 cause. (The WidgetObject's Properties order is tolerated.) - objType = setField(objType, "PropertyTypes", orderPropertyTypes(newPropTypes, def)) - typeDoc = setField(typeDoc, "ObjectType", objType) - widget = setField(widget, "Type", typeDoc) - return widget, applied, skipped +// isWidgetPlaceholderID matches the "aa"-prefixed IDs the template pipeline mints. +func isWidgetPlaceholderID(s string) bool { + return len(s) == 32 && strings.HasPrefix(s, "aa0000000000000000000000") } From e966706ece20630f3c49a981a71ae5170a571f03 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 16:51:26 +0000 Subject: [PATCH 24/29] docs: record the four remaining value migrations, and one disproven fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The widget Type is now byte-identical to `mx update-widgets` output, so what keeps CE0463 alive is value-level. Naming the four families in the code so the next step is not re-derived: - Appearance/DesignProperties — an empty list needs the `[3]` marker - LabelTemplate — written as an explicit null, not omitted - Forms$GridSortBar — SortDirection -> SortOrder, list marker 3 -> 2 - a newly ADDED TextTemplate property's value — null, not a populated template The last one was tested rather than assumed, and the blunt version is wrong: nulling every TextTemplate value in a synced project takes CE0463 from 33 to 127, because instances that legitimately carry a caption need it. mxcli's authoring path is right to populate one (an empty required textTemplate is CE4899) — the null applies only to properties the sync itself introduced. Negative result added to the CE0463 skill, where the elimination list is worth as much as the fix. No behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/diagnose-ce0463.md | 7 +++++++ mdl/executor/widget_sync_apply.go | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.claude/skills/diagnose-ce0463.md b/.claude/skills/diagnose-ce0463.md index 76a396a14..b20e6f158 100644 --- a/.claude/skills/diagnose-ce0463.md +++ b/.claude/skills/diagnose-ce0463.md @@ -159,6 +159,13 @@ Ordered by how often they have actually been the answer. - **A hypothesis that survives only because you have not tested it is not evidence.** Patch it, measure, and write down the negative result — the elimination list is worth as much as the fix. +- **A value fix that is right for NEW properties is usually wrong applied to all.** + `mx update-widgets` stores `TextTemplate: null` on a property it has just added, + where mxcli's authoring path builds a populated `Forms$ClientTemplate` (correct + there — an empty required textTemplate is CE4899). Nulling *every* TextTemplate + value in a synced project took CE0463 from **33 to 127**: instances that + legitimately carry a caption need it. Scope such a fix to the properties the + operation itself introduced. - **Test any candidate fix against the bundled package too.** Pruning the fields the `update-widgets` reference omits fixes 2 widgets on Data Widgets 3.10 and takes the bundled 3.4 from **0 → 139**. diff --git a/mdl/executor/widget_sync_apply.go b/mdl/executor/widget_sync_apply.go index b2052c55b..ac05a7d3d 100644 --- a/mdl/executor/widget_sync_apply.go +++ b/mdl/executor/widget_sync_apply.go @@ -18,7 +18,18 @@ import ( // widget_sync_apply.go writes the reconciliation the planner describes. // -// Scope of this step: REMOVE and UPDATE only. +// Reconciliation is delegated to widgets.AugmentTemplate (see applyToWidget), which +// makes the stored widget's TYPE byte-identical to `mx update-widgets` output. What +// remains are value-level migrations update-widgets also performs and this does not: +// +// - Appearance/DesignProperties: an empty list needs the `[3]` marker +// - LabelTemplate: written as an explicit null, not omitted +// - Forms$GridSortBar: SortDirection -> SortOrder, list marker 3 -> 2 +// - a newly added TextTemplate property's value: null, not a populated template +// +// The last one is scoped carefully: nulling EVERY TextTemplate value in a synced +// project takes CE0463 from 33 to 127, because instances that legitimately carry a +// caption need it. Only properties this operation introduced may be nulled. // // - remove — a PropertyKey the installed package no longer declares. This is the // mendixlabs/mxcli#716 case (`advanced`, dropped from Data Widgets after 3.4) and From f34abd1567ed8a30487e1891cdd96a91b5ee203b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 17:52:40 +0000 Subject: [PATCH 25/29] docs: three of the four value migrations disproven MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tested each candidate in isolation against the fixture, per the CE0463 skill's own procedure. None moved the count: baseline (Type byte-identical to update-widgets) 33 CE0463 + `[3]` marker on empty DesignProperties 33 (and matched 0 nodes — the selector was wrong too) + explicit null LabelTemplate 33 (29 edits, 10 units) + GridSortBar SortDirection -> SortOrder, marker 33 (12 edits, 6 units) They are genuine differences from `mx update-widgets` output and they are not what CE0463 reacts to. Recorded in the code and in the skill so they are not retried. The remaining untested candidate is the scoped TextTemplate null — added properties only, since the blanket version was already shown to take 33 -> 127. If that also fails, the cause is not among the 25 value paths and the next move is the splice bisection. Also re-ran the diff with binary values visible rather than masked as '', since the skill warns that normalisation is where the answer hides: 260 of 285 differences are $ID/pointer identity, which differs by construction. No hidden signal there. Lesson added to the skill: a difference from the reference is not automatically a cause. The diff bounds the search; it does not rank it. No behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- .claude/skills/diagnose-ce0463.md | 7 +++++++ mdl/executor/widget_sync_apply.go | 17 ++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.claude/skills/diagnose-ce0463.md b/.claude/skills/diagnose-ce0463.md index b20e6f158..ad342ab9b 100644 --- a/.claude/skills/diagnose-ce0463.md +++ b/.claude/skills/diagnose-ce0463.md @@ -159,6 +159,13 @@ Ordered by how often they have actually been the answer. - **A hypothesis that survives only because you have not tested it is not evidence.** Patch it, measure, and write down the negative result — the elimination list is worth as much as the fix. +- **A difference from the reference is not automatically a cause.** With a synced + DataGrid2's `Type` byte-identical to `mx update-widgets` output, 25 value-level + differences remained. Three were patched in isolation — the `[3]` marker on empty + `DesignProperties`, an explicit null `LabelTemplate`, and the `GridSortBar` + `SortDirection`→`SortOrder` + marker migration — and **none moved the count** + (33 → 33 each). Real differences, not the cause. Budget for this: the diff bounds + the search, it does not rank it. - **A value fix that is right for NEW properties is usually wrong applied to all.** `mx update-widgets` stores `TextTemplate: null` on a property it has just added, where mxcli's authoring path builds a populated `Forms$ClientTemplate` (correct diff --git a/mdl/executor/widget_sync_apply.go b/mdl/executor/widget_sync_apply.go index ac05a7d3d..9ea140dbb 100644 --- a/mdl/executor/widget_sync_apply.go +++ b/mdl/executor/widget_sync_apply.go @@ -31,16 +31,15 @@ import ( // project takes CE0463 from 33 to 127, because instances that legitimately carry a // caption need it. Only properties this operation introduced may be nulled. // -// - remove — a PropertyKey the installed package no longer declares. This is the -// mendixlabs/mxcli#716 case (`advanced`, dropped from Data Widgets after 3.4) and -// the operation that actually clears CE0463. -// - update — Required / OnChangeProperty on a surviving property, in place only. +// Three of those four were TESTED IN ISOLATION against the fixture and NONE moved the +// count (33 before, 33 after each): adding the `[3]` marker to empty DesignProperties, +// writing LabelTemplate as an explicit null, and the GridSortBar SortDirection -> +// SortOrder + marker migration. They are real differences from update-widgets output +// but they are not what CE0463 is reacting to. Recorded so they are not retried. // -// ADD is deliberately not applied yet. `mx update-widgets` does not add missing -// properties unconditionally — it adds 12 to each stale Gallery but none to four -// FeedbackModule Image instances that are short by four, and Mendix reports no CE0463 -// on those. Adding is also the only operation that invents nodes, so it is the one -// worth being sure about. See PlanWidgetSync's comment. +// The remaining untested candidate is the scoped TextTemplate null (added properties +// only). If that also fails, the cause is not among the 25 value paths and the next +// move is the splice bisection in .claude/skills/diagnose-ce0463.md. // // # The pairing invariant // From 4007d16c16ed321fcd7287026af75c8d1bcd7859 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 22:06:02 +0000 Subject: [PATCH 26/29] docs(widget): mark sync as partial in help, guide and proposal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parking the branch, so the user-facing text must say what the command actually does rather than what it was aimed at. - CLI help leads with PARTIAL: clears 7 of 40 CE0463 on the reference fixture where `mx update-widgets` clears 40, and names the tradeoff that makes it worth having anyway (update-widgets destroys mprcontents/ on MPR v2). Also states that applying needs MXCLI_ENGINE=legacy. - docs-site pluggable-widgets guide gains a section for the command, with the same caveat and a "preview with --dry-run, confirm with mx check" instruction. - PROPOSAL_widget_instance_reconciliation moves to "Partial — shipped and parked" with a status table of what is verified, where it stops, and the ordered next moves. The three disproven value migrations are recorded there too, so resuming does not start by retrying them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012XR649rKk68z6gBpngu6MA --- cmd/mxcli/cmd_widget_sync.go | 15 ++++++-- docs-site/src/guides/pluggable-widgets.md | 26 ++++++++++++++ ...PROPOSAL_widget_instance_reconciliation.md | 36 +++++++++++++++++-- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/cmd/mxcli/cmd_widget_sync.go b/cmd/mxcli/cmd_widget_sync.go index e87f1b656..c592aa31d 100644 --- a/cmd/mxcli/cmd_widget_sync.go +++ b/cmd/mxcli/cmd_widget_sync.go @@ -16,13 +16,24 @@ import ( // widget packages installed in the project — the mxcli equivalent of Studio Pro's // "Update all widgets" and `mx update-widgets`, without the latter's MPR v2 data loss. // -// Currently read-only: it reports what would change. Applying is the next step. +// PARTIAL. On the reference fixture (Data Widgets 3.4 -> 3.11.3) this clears 7 of 40 +// CE0463 while `mx update-widgets` clears all 40. What it does is faithful — the +// widget Type it writes is byte-identical to Mendix's own output — but a value-level +// difference not yet identified keeps DataGrid2 and Gallery instances erroring. The +// help text says so rather than implying the command finishes the job. var widgetSyncCmd = &cobra.Command{ Use: "sync", Short: "Reconcile stored widget instances against the installed widget packages", Long: `Compare every stored pluggable-widget instance against the .mpk currently -installed in the project, and report the schema differences. +installed in the project, and reconcile the differences. + +PARTIAL — this does not yet fully replace "Update all widgets". On the reference +fixture it clears 7 of 40 CE0463 errors; Mendix's own 'mx update-widgets' clears +all 40 but destroys the mprcontents/ folder on MPR v2 projects, which this does +not. Preview with --dry-run and verify with 'mx check' before relying on it. + +Applying currently requires MXCLI_ENGINE=legacy; --dry-run works on both engines. mxcli writes a widget instance correctly for the package installed at authoring time. When that package is later upgraded, the stored instances go stale and diff --git a/docs-site/src/guides/pluggable-widgets.md b/docs-site/src/guides/pluggable-widgets.md index 33775a94b..1655fca66 100644 --- a/docs-site/src/guides/pluggable-widgets.md +++ b/docs-site/src/guides/pluggable-widgets.md @@ -170,6 +170,32 @@ Both the properties and the rules are available as JSON for scripting: } ``` +## Reconciling stored widgets after a package upgrade (`mxcli widget sync`) + +Everything above keeps a widget correct **as it is authored**. When you later upgrade +the widget package, the instances already stored in your pages go stale and Mendix +reports CE0463 on each one. Studio Pro fixes this with "Update all widgets"; mxbuild +has `mx update-widgets`, which works but **destroys `mprcontents/`** on an MPR v2 +project, collapsing it to a single-file v1 layout. + +`mxcli widget sync` is the equivalent that preserves the v2 layout: + +```bash +mxcli widget sync -p app.mpr --dry-run # preview; names every property +mxcli widget sync -p app.mpr # apply +``` + +It reconciles **schema** — properties the package added, dropped or redefined — and +never changes a property value you set. It skips any widget whose `.mpk` is not +installed rather than guessing. + +**This is partial today.** On the reference fixture (Data Widgets 3.4 → 3.11.3) it +clears 7 of 40 CE0463 errors where `mx update-widgets` clears all 40. The widget +*type* it writes is byte-identical to Mendix's own output; a value-level difference +that has not yet been identified keeps DataGrid2 and Gallery instances erroring. +Preview with `--dry-run` and confirm with `mx check` before relying on it. Applying +currently needs `MXCLI_ENGINE=legacy`; `--dry-run` works on either engine. + ## Marketplace and custom widgets `mxcli widget describe -p app.mpr ` works for **any** widget installed in the diff --git a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md index 974f99bf7..9744ad247 100644 --- a/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md +++ b/docs/11-proposals/PROPOSAL_widget_instance_reconciliation.md @@ -1,7 +1,39 @@ # Proposal: `mxcli widget sync` — reconcile stored widget instances against installed .mpk packages -**Status:** Draft -**Date:** 2026-08-03 +**Status:** Partial — shipped and parked +**Date:** 2026-08-03 (proposal), 2026-08-04 (implementation status) + +## Implementation status (2026-08-04) — partial, parked + +`mxcli widget sync` exists and writes. On the reference fixture (v0.10 widgets +authored against Data Widgets 3.4, package upgraded to 3.11.3, 40 CE0463) it clears +**7 of 40**, against `mx update-widgets`'s 40. + +Shipped and verified: + +| | | +|---|---| +| MPR v2 preserved | 207 `mprcontents/` units before and after — the whole point | +| Coverage | pages, snippets, **building blocks**, page templates, layouts; **zero misses** vs `update-widgets` | +| Widget `Type` written | **byte-identical** to `update-widgets` output (0 differing paths) | +| Idempotent | second run reports "already matches its installed package" | +| Engines | plan runs on both and they agree; **apply needs `MXCLI_ENGINE=legacy`** | + +Where it stops: with the `Type` exact, 17 DataGrid2 and Gallery instances still +report CE0463, so the trigger is value-level. Four candidate value migrations were +identified by diffing against `update-widgets`; **three were tested in isolation and +none moved the count** (the `[3]` marker on empty `DesignProperties`, an explicit +null `LabelTemplate`, and the `GridSortBar` `SortDirection`→`SortOrder` + marker +migration). The untested fourth is a TextTemplate null scoped to added properties +only — the blanket version takes 33 → 127. + +Next moves, in order: the scoped TextTemplate null; if that fails, the splice +bisection from [`diagnose-ce0463.md`](../../.claude/skills/diagnose-ce0463.md); +then `GetRawUnitBytes`/`UpdateRawUnit` on the modelsdk backend to drop the engine +gate. + +Open question 2 (retyped property) and 3 (implicit sync during build) remain +untouched. Open question 1 (naming) resolved as `sync`. ## Problem Statement From b054123936515fba8607341387d665a094acfd51 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 22:35:54 +0000 Subject: [PATCH 27/29] feat(pages): dynamic-text parameter formatting via a FORMAT block (ledger #75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Decimal/DateTime rendered through `dynamictext` always used the hardcoded default format ("5068.38000000"), and a widget-level `decimalPrecision:` was silently dropped. The Mendix model always stored a ClientTemplateParameter.FormattingInfo, but all three writers hardcoded it and ignored user intent, and there was no MDL syntax to set it. Add a per-parameter FORMAT block that maps to FormattingInfo: dynamictext amt (Content: '{1}', ContentParams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)]) ContentParams: [{1} = DueOn format (dateFormat: DateTime)] ContentParams: [{1} = DueOn format (dateFormat: Custom, customDateFormat: 'dd-MM-yyyy')] ContentParams: [{1} = Status format (enumFormat: Text)] Full-stack: - Grammar: `paramAssignmentV3` gains `(FORMAT paramFormatV3)?`. The FORMAT keyword is required — a bare `(…)` after the value is ambiguous with a function-call argument list because `:` is a valid OQL division operator in expressions. - AST/visitor: ParamAssignmentV3.Format carries the raw key/value props. - Builder: formattingInfoFromParamFormat starts from the Mendix defaults and applies only the keys the user set. - Writers (modelsdk default + legacy): use the parameter's FormattingInfo when present; a nil FormattingInfo reproduces the previous hardcoded defaults, so every existing/unformatted parameter is byte-identical to before (zero risk). - DESCRIBE: emits the format block for non-default FormattingInfo, so it round-trips (previously formatting present in an .mpr was dropped on read too). - Validation: MDL-WIDGET18 turns a widget-level format key into an actionable error (no more silent drop) and validates keys / enum values at check time. - sdk FormattingInfo gains CustomDateFormat (was missing from the struct). Verified against Mendix 11.12.1: exec → `mx check` 0 errors; DESCRIBE PAGE round-trips the format block. Tests: visitor parse, builder coercion, validation (unknown key / bad enum / widget-level key / custom-without-Custom), describe suffix. Repro mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JXnEgoc2NQP1Y2TWMCMXC4 --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/create-page.md | 26 ++++ docs/01-project/MDL_QUICK_REFERENCE.md | 7 + .../ledger-75-dynamictext-formatting.mdl | 54 ++++++++ mdl/ast/ast_page_v3.go | 34 ++++- mdl/backend/modelsdk/widget_write.go | 35 ++++- mdl/executor/cmd_pages_builder_v3_widgets.go | 36 ++++++ mdl/executor/cmd_pages_describe_output.go | 44 +++++++ mdl/executor/dynamictext_format_test.go | 120 ++++++++++++++++++ mdl/executor/validate_widgets.go | 105 +++++++++++++++ mdl/grammar/domains/MDLPage.g4 | 16 ++- mdl/visitor/visitor_page_v3.go | 32 +++++ mdl/visitor/visitor_test.go | 25 ++++ sdk/mpr/writer_widgets.go | 28 +++- sdk/pages/pages_widgets_input.go | 3 +- 15 files changed, 551 insertions(+), 15 deletions(-) create mode 100644 mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl create mode 100644 mdl/executor/dynamictext_format_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 7cb416ca1..69ebfdd6d 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -325,6 +325,7 @@ cases for these three BSON types — they fell to `default: return nil`. | Freshly authored **Gallery** widgets fail `mx check` with **CE0463** on Data Widgets 3.10+, while the 3.4 package bundled with Mendix 11.12/11.13 is clean — so it looks like ordinary post-upgrade staleness. Every schema-level explanation is disproven (property sets, list markers, ordering, pointer topology, `GenerateFromMPK`) | `syncDefinitionAttrs` reconciled a surviving property's definition attributes (`Required`, `OnChangeProperty`) from the installed `.mpk` onto the `CustomWidgets$WidgetPropertyType` node — but they live one level down, on its **`ValueType`**. The "only update a key that already exists" guard then never fired, making the entire pass a **silent no-op**, so the Gallery kept the embedded template's `OnChangeProperty = "onConfigurationChange"` where 3.10 expects `""` | `sdk/widgets/augment.go` + `modelsdk/widgets/augment.go` (`syncDefinitionAttrs`) | Descend to `ValueType` before the update (`getMapField(ptMap, "ValueType")`, falling back to the PropertyType). **Diagnosis method that found it**: diff against `mx update-widgets` output at the PATH level — the differing paths were `Type/ObjectType/PropertyTypes[N]/ValueType/OnChangeProperty`, and the path told me which node to write. **Generalisable — the trap that cost the most here**: a guarded update (`if _, ok := m[k]; ok`) aimed at the wrong node is *invisible*. It cannot fail loudly, so measurements read as "the fix didn't help" rather than "the fix never ran". When a change measurably does nothing, verify it executed before concluding the hypothesis was wrong. Result: fresh-authoring CE0463 on 3.10 went 6 → 2; bundled 3.4 stayed at 0. **Still open**: the two datagrid dropdown filters, whose residual diff is `ValueType/AllowUpload` (absent from the modelsdk template copy, present in sdk's — the engines' template sets have diverged); syncing them does not clear it. Repro `mdl-examples/bug-tests/716-widget-package-upgrade.mdl`. Issue #716 | | Every authored **Data grid 2** fails `mx check` with **CE0463** on the *bundled* Data Widgets 3.4 (Mendix 11.12/11.13) — but only on the **legacy** engine; `modelsdk` is clean on the same script. Appeared the moment `syncDefinitionAttrs` was corrected to write to `ValueType` (the row above), i.e. the moment that pass first actually executed | The Mendix pluggable-widget XML schema defaults `required` to **true**; only an explicit `required="false"` is optional. `sdk/widgets/mpk` read a missing attribute as `false` (`p.Required == "true"`), so the now-live sync overwrote 24 correct `true`s with `false` on DataGrid2 3.4 (which omits `required=` on 24 of its 40 properties). `modelsdk/widgets/mpk` already read it correctly (`p.Required != "false"`, fixed under #600) — the engines had silently diverged on the default | `sdk/widgets/mpk/mpk.go` (all three `PropertyDef` construction sites: `walkPropertyGroup` top-level + nested-direct, `collectNestedProperties`) | Read `p.Required != "false"` so absent means true, matching `modelsdk` and `mx update-widgets`. **Generalisable — the shape to look for**: when two engines carry parallel copies of a parser, a fix applied to one leaves a *latent* divergence in the other that stays invisible until some unrelated change starts consuming the value. Grep the sibling package for the same expression before assuming a defect is engine-specific. **Diagnosis trap hit here**: an A/B ran out of disk mid-run, the exec silently created no widgets, and `mx check` reported 0 CE0463 — reading as "the pre-fix binary is clean". Always assert the artifact exists (`show widgets | grep `) before trusting a zero. Unit repro `sdk/widgets/mpk/required_default_test.go`; integration `TestMxCheck_DataGridPage`/`TestMxCheck_DataGridNoColumns`. Issue #716 | | Freshly authored **drop-down filter** widgets fail `mx check` with **CE0463** on Data Widgets 3.10 while every other widget in the same script is clean. The stored BSON is missing `ValueType/AllowUpload` on all 25 ValueTypes and carries `Required=false` on `refCaption`/`refCaptionExp`, which the 3.10 XML declares `required="true"` — so the embedded template looks like the culprit, but replacing it changes nothing | `AugmentTemplate` never ran on this widget. Its opening guard — "nothing to add or remove at top level, and no nested children" — was written when the function only synced the property SET, and returns before the **six value-level passes** bolted on later (`reconcileEnumValues`, `reconcilePropertyMetadata`, `reconcileValueTypesFromMPK`, `completeValueTypeEnvelope`, `reorderPropertyTypes`, `syncDefinitionAttrs`). DataGrid2 never hit it because `columns` has nested children; the drop-down filter declares exactly the 25 keys the 11.6-era template already has, so it took the exit every time | `modelsdk/widgets/augment.go` + `sdk/widgets/augment.go` (`AugmentTemplate`) | Wrap the add/remove block in `if len(missing) > 0 \|\| len(stale) > 0 \|\| hasNestedChildren { … }` instead of returning early, so the value-level passes always run. **Generalisable — the shape to look for**: an early return placed correctly for a function's original job silently disables everything appended after it. When a pass "does nothing" for one input and works for others, check whether it *reached* the pass before theorising about the data (same root shape as the `ValueType` no-op two rows up). The `hasNestedChildren` clause in that guard is the tell: someone had already patched around the same bug for one widget rather than fixing it. **Diagnosis method**: a probe calling `augmentFromMPK` directly and counting `ValueTypes with AllowUpload` — 0/25 for the drop-down filter vs 44/44 for Gallery — localised it to "augment did not run" in one step, before any BSON theorising. Result: modelsdk on DW 3.10 went 2 → **0**. Tests `TestAugmentTemplate_MatchingKeysStillReconcilesValues` in both engines. Issue #716 | +| A **Decimal/DateTime** in `dynamictext` always rendered with the hardcoded default format ("5068.38000000"); no MDL way to set the per-parameter Format, and a widget-level `decimalPrecision:` was **silently dropped** (mxcli check ✓, exec ✓, but gone) | The model always stored `ClientTemplateParameter.FormattingInfo`, but **all three writers hardcoded it** (`DecimalPrecision:2, GroupDigits:false, DateFormat:Date, EnumFormat:Text`) and ignored `param.FormattingInfo`; the grammar had no syntax to set it and DESCRIBE dropped it on read too | grammar `mdl/grammar/domains/MDLPage.g4` (`paramAssignmentV3` + `paramFormatV3`), `mdl/ast/ast_page_v3.go` (`ParamFormatV3`), `mdl/visitor/visitor_page_v3.go` (`buildParamFormatV3`), builder `mdl/executor/cmd_pages_builder_v3_widgets.go` (`formattingInfoFromParamFormat`), writers `mdl/backend/modelsdk/widget_write.go` (`formattingInfoToGen`) + `sdk/mpr/writer_widgets.go` (`serializeClientTemplateParameter`), describe `cmd_pages_describe_output.go` (`formatParamFormatSuffix`), validate `validate_widgets.go` (`validateDynamicTextFormatting`/MDL-WIDGET18) | Add a per-param `FORMAT (decimalPrecision: N, groupDigits: bool, dateFormat: …, customDateFormat: '…', enumFormat: …)` block: `{1} = Amount format (decimalPrecision: 2, groupDigits: true)`. The **FORMAT keyword is required** — a bare `(…)` after the value is ambiguous with a function call because `:` is OQL division in expressions. Writers use the param's FormattingInfo when set, else the same hardcoded defaults (nil → byte-identical to before, zero risk to existing widgets). MDL-WIDGET18 turns a widget-level format key into an actionable error (no more silent drop) and validates keys/enums. Verified: exec → `mx check` (11.12.1) 0 errors + DESCRIBE round-trip. Repro `mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl`. Ledger #75 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/.claude/skills/mendix/create-page.md b/.claude/skills/mendix/create-page.md index 8a5a60c1c..ba131ff63 100644 --- a/.claude/skills/mendix/create-page.md +++ b/.claude/skills/mendix/create-page.md @@ -237,6 +237,32 @@ dynamictext title (Attribute: Title) | `AttrName` | Current DataView/Gallery entity | `Name`, `Email` | | `'literal'` | String literal expression | `'Hello'` | +**Formatting a parameter (Decimal / DateTime / Enum):** append a `format (…)` +block to a content parameter. Without it, a Decimal renders with the platform +default (e.g. `5068.38000000`). + +```sql +-- Decimal: 2 decimals + thousands separator -> "5,068.38" +dynamictext amt (content: '{1}', contentparams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)]) + +-- DateTime: date + time, or a custom pattern +dynamictext due (content: '{1}', contentparams: [{1} = DueOn format (dateFormat: DateTime)]) +dynamictext day (content: '{1}', contentparams: [{1} = DueOn format (dateFormat: Custom, customDateFormat: 'dd-MM-yyyy')]) +``` + +| Format key | Applies to | Values | +|------------|-----------|--------| +| `decimalPrecision` | Decimal / Float | a non-negative integer | +| `groupDigits` | Decimal / Float | `true` \| `false` | +| `dateFormat` | DateTime | `Date` \| `DateTime` \| `Time` \| `Custom` | +| `customDateFormat` | DateTime | a pattern string, requires `dateFormat: Custom` | +| `enumFormat` | Enumeration | `Text` \| `Image` | + +> The **`format` keyword is required** — a bare `(…)` after the value is +> ambiguous with a function call. Putting a format key at the **widget** level +> (e.g. `dynamictext x (…, decimalPrecision: 2)`) is an error (MDL-WIDGET18): +> formatting is per-parameter, so it must go inside the `contentparams` block. + > **Never leave a `{N}` placeholder unbound.** `content: '{1}'` with no > `Attribute:`/`ContentParams:` is an orphaned template — `mxcli check` rejects it > (MDL-WIDGET04), MxBuild fails with CE0720, and Studio Pro throws a diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index cc2925ceb..ac4e35c32 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -994,6 +994,13 @@ create page MyModule.Customer_Edit - Actions: `actionbutton`, `linkbutton`, `navigationlist` - Structure: `dataview`, `header`, `footer`, `controlbar`, `snippetcall` +**DynamicText parameter formatting** — append a `format (…)` block to a content parameter (the `format` keyword is required): +```sql +dynamictext amt (content: '{1}', contentparams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)]) +dynamictext due (content: '{1}', contentparams: [{1} = DueOn format (dateFormat: DateTime)]) +``` +Keys: `decimalPrecision` (int), `groupDigits` (bool), `dateFormat` (`Date`|`DateTime`|`Time`|`Custom`), `customDateFormat` (pattern, with `dateFormat: Custom`), `enumFormat` (`Text`|`Image`). + ## ALTER PAGE / ALTER SNIPPET Modify an existing page or snippet's widget tree in-place without full `create or replace`. Works directly on the raw BSON tree, preserving unsupported widget types. diff --git a/mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl b/mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl new file mode 100644 index 000000000..003fa7f6a --- /dev/null +++ b/mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl @@ -0,0 +1,54 @@ +-- ============================================================================ +-- Ledger finding #75: Dynamic Text formatting (FormattingInfo) is reachable +-- ============================================================================ +-- +-- Before this fix, a Decimal rendered through `dynamictext` always showed with +-- the hardcoded default precision (e.g. "5068.38000000") — there was no way to +-- set the per-parameter Format (decimals, thousands separator, date format). +-- The Mendix model always stored a ClientTemplateParameter.FormattingInfo, but +-- every writer hardcoded it and ignored user intent, and a widget-level format +-- property was silently dropped. +-- +-- Now each content parameter accepts a FORMAT block mapping to FormattingInfo: +-- {1} = Amount format (decimalPrecision: 2, groupDigits: true) +-- {1} = DueOn format (dateFormat: DateTime) +-- {1} = DueOn format (dateFormat: Custom, customDateFormat: 'dd-MM-yyyy') +-- {1} = Status format (enumFormat: Text) +-- +-- The FORMAT keyword is required: a bare `(…)` after the expression is +-- ambiguous with a function-call argument list because `:` is a valid OQL +-- division operator in expressions. +-- +-- Verified: mxcli exec -> mx check (Mendix 11.12.1) = 0 errors; DESCRIBE PAGE +-- round-trips the format block. +-- ============================================================================ + +create entity Ledger.LineItem ( + Amount: Decimal, + DueOn: DateTime +); + +create page Ledger.LineItems ( + Title: 'Line items', + Layout: Atlas_Core.Atlas_Default +) +{ + layoutgrid grid { + row r { + column c (DesktopWidth: 12) { + listview lv (DataSource: DATABASE FROM Ledger.LineItem) { + -- Decimal with 2 decimals and a thousands separator. + dynamictext txtAmount ( + Content: '{1}', + ContentParams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)] + ) + -- DateTime shown with date + time. + dynamictext txtDue ( + Content: '{1}', + ContentParams: [{1} = DueOn format (dateFormat: DateTime)] + ) + } + } + } + } +} diff --git a/mdl/ast/ast_page_v3.go b/mdl/ast/ast_page_v3.go index fd26daeb2..eb18154d1 100644 --- a/mdl/ast/ast_page_v3.go +++ b/mdl/ast/ast_page_v3.go @@ -129,8 +129,38 @@ type ColumnV3 struct { // ParamAssignmentV3 represents a template parameter: {1} = value type ParamAssignmentV3 struct { - Index int // Parameter index (1, 2, 3, ...) - Value any // Expression value + Index int // Parameter index (1, 2, 3, ...) + Value any // Expression value + Format *ParamFormatV3 // Optional per-parameter formatting (dynamic text), else nil +} + +// ParamFormatV3 holds the optional per-parameter formatting of a dynamic-text +// parameter, mapping to the Mendix ClientTemplateParameter FormattingInfo. Props +// preserve the raw key/value pairs exactly as written, so the validator can flag +// unknown keys / bad values and DESCRIBE can round-trip them. +// +// {1} = Amount format (decimalPrecision: 2, groupDigits: true) +type ParamFormatV3 struct { + Props []ParamFormatProp +} + +// ParamFormatProp is one `key: value` entry inside a parameter format block. +type ParamFormatProp struct { + Key string // lowercased key, e.g. "decimalprecision" + Value string // raw value text, with surrounding quotes stripped for strings +} + +// Get returns the value for a (case-insensitive) key and whether it was present. +func (f *ParamFormatV3) Get(key string) (string, bool) { + if f == nil { + return "", false + } + for _, p := range f.Props { + if p.Key == key { + return p.Value, true + } + } + return "", false } // DesignPropertyEntryV3 represents a single design property entry. It is either diff --git a/mdl/backend/modelsdk/widget_write.go b/mdl/backend/modelsdk/widget_write.go index cb5016d55..203e1a5cf 100644 --- a/mdl/backend/modelsdk/widget_write.go +++ b/mdl/backend/modelsdk/widget_write.go @@ -954,7 +954,7 @@ func clientTemplateParameterToGen(p *pages.ClientTemplateParameter) element.Elem if p.SourceVariable != "" { g.SetSourceVariable(sourceVariableToGen(p.SourceVariable, p.SourceVariableKind)) } - g.SetFormattingInfo(newFormattingInfo()) + g.SetFormattingInfo(formattingInfoToGen(p.FormattingInfo)) return g } @@ -979,13 +979,36 @@ func sourceVariableToGen(name, kind string) element.Element { // newFormattingInfo builds the default Forms$FormattingInfo (matches the legacy // serializer; TimeFormat is intentionally omitted — it triggers CE0463). func newFormattingInfo() element.Element { + return formattingInfoToGen(nil) +} + +// formattingInfoToGen builds a Forms$FormattingInfo, using the parameter's +// per-parameter formatting when present and the standard defaults otherwise. A +// nil fi reproduces the previous hardcoded defaults, so every unformatted +// parameter is byte-identical to before. TimeFormat is intentionally omitted — +// it is not a schema field and triggers CE0463. +func formattingInfoToGen(fi *pages.FormattingInfo) element.Element { + dateFormat, customDateFormat, enumFormat := "Date", "", "Text" + decimalPrecision := 2 + groupDigits := false + if fi != nil { + if fi.DateFormat != "" { + dateFormat = fi.DateFormat + } + customDateFormat = fi.CustomDateFormat + if fi.EnumFormat != "" { + enumFormat = fi.EnumFormat + } + decimalPrecision = fi.DecimalPrecision + groupDigits = fi.GroupDigits + } f := genPg.NewFormattingInfo() assignID(f) - f.SetCustomDateFormat("") - f.SetDateFormat("Date") - f.SetDecimalPrecision(2) - f.SetEnumFormat("Text") - f.SetGroupDigits(false) + f.SetCustomDateFormat(customDateFormat) + f.SetDateFormat(dateFormat) + f.SetDecimalPrecision(int32(decimalPrecision)) + f.SetEnumFormat(enumFormat) + f.SetGroupDigits(groupDigits) return f } diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index f43d4a7f8..9fd649cf0 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -5,6 +5,7 @@ package executor import ( "fmt" "regexp" + "strconv" "strings" "github.com/mendixlabs/mxcli/mdl/ast" @@ -654,6 +655,7 @@ func (pb *pageBuilder) buildDynamicTextV3(w *ast.WidgetV3) (*pages.DynamicText, pb.resolveTemplateAttributePathFull(strVal, param) } } + param.FormattingInfo = formattingInfoFromParamFormat(p.Format) dt.Content.Parameters = append(dt.Content.Parameters, param) } } @@ -665,6 +667,40 @@ func (pb *pageBuilder) buildDynamicTextV3(w *ast.WidgetV3) (*pages.DynamicText, return dt, nil } +// formattingInfoFromParamFormat coerces a parsed per-parameter format block into +// an SDK FormattingInfo. Returns nil when there is no block, so the writers keep +// emitting the existing hardcoded defaults for every unformatted parameter. When +// a block IS present it starts from those same defaults and applies the user's +// keys on top, so only the specified fields change. Unknown keys / invalid values +// are ignored here — check-time validation (MDL-WIDGET18) reports them. +func formattingInfoFromParamFormat(f *ast.ParamFormatV3) *pages.FormattingInfo { + if f == nil || len(f.Props) == 0 { + return nil + } + fi := &pages.FormattingInfo{ + DateFormat: "Date", + DecimalPrecision: 2, + EnumFormat: "Text", + } + for _, p := range f.Props { + switch p.Key { + case "decimalprecision": + if n, err := strconv.Atoi(p.Value); err == nil { + fi.DecimalPrecision = n + } + case "groupdigits": + fi.GroupDigits = strings.EqualFold(p.Value, "true") + case "dateformat": + fi.DateFormat = p.Value + case "customdateformat": + fi.CustomDateFormat = p.Value + case "enumformat": + fi.EnumFormat = p.Value + } + } + return fi +} + func (pb *pageBuilder) buildTitleV3(w *ast.WidgetV3) (*pages.Title, error) { title := &pages.Title{ BaseWidget: pages.BaseWidget{ diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 375ce5c65..563f5882b 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -1392,11 +1392,15 @@ func extractClientTemplateParameters(ctx *ExecContext, w map[string]any, fieldNa return nil } var result []string + var suffixes []string // per-param format block " (decimalPrecision: 2, …)", "" when default for _, p := range params { pMap, ok := p.(map[string]any) if !ok { continue } + // One suffix per emitted param, in the same order, so it can be zipped in + // after the value string is chosen (round-trips per-parameter formatting). + suffixes = append(suffixes, formatParamFormatSuffix(pMap)) // Check for Expression first (literal value) if expr, ok := pMap["Expression"].(string); ok && expr != "" { // A non-String attribute binding (Integer/DateTime/…) is written as @@ -1449,9 +1453,49 @@ func extractClientTemplateParameters(ctx *ExecContext, w map[string]any, fieldNa // Parameter exists but has no binding - mark as unbound result = append(result, "") } + // Append each param's format block to its value string. result and suffixes are + // aligned (one of each per param that had a valid pMap). + for i := range result { + if i < len(suffixes) && suffixes[i] != "" { + result[i] += suffixes[i] + } + } return result } +// formatParamFormatSuffix renders a dynamic-text parameter's FormattingInfo back +// to the MDL format block " (decimalPrecision: 2, groupDigits: true, …)", +// emitting only the fields that differ from the Mendix defaults (DateFormat=Date, +// DecimalPrecision=2, EnumFormat=Text, GroupDigits=false, CustomDateFormat=""), +// so an unformatted parameter round-trips as before (empty string). Mirrors the +// writer defaults in formattingInfoFromParamFormat / formattingInfoToGen. +func formatParamFormatSuffix(pMap map[string]any) string { + fi, ok := pMap["FormattingInfo"].(map[string]any) + if !ok || fi == nil { + return "" + } + var parts []string + if dp := extractInt(fi["DecimalPrecision"]); dp != 2 { + parts = append(parts, fmt.Sprintf("decimalPrecision: %d", dp)) + } + if gd, ok := fi["GroupDigits"].(bool); ok && gd { + parts = append(parts, "groupDigits: true") + } + if df := extractString(fi["DateFormat"]); df != "" && df != "Date" { + parts = append(parts, "dateFormat: "+df) + } + if cdf := extractString(fi["CustomDateFormat"]); cdf != "" { + parts = append(parts, "customDateFormat: '"+cdf+"'") + } + if ef := extractString(fi["EnumFormat"]); ef != "" && ef != "Text" { + parts = append(parts, "enumFormat: "+ef) + } + if len(parts) == 0 { + return "" + } + return " format (" + strings.Join(parts, ", ") + ")" +} + // associationTemplateParamPath reconstructs the "Assoc/.../Attr" navigation of a // template parameter whose AttributeRef binds an attribute over one or more // associations (AttributeRef.EntityRef = DomainModels$IndirectEntityRef). Returns diff --git a/mdl/executor/dynamictext_format_test.go b/mdl/executor/dynamictext_format_test.go new file mode 100644 index 000000000..8f5dd6693 --- /dev/null +++ b/mdl/executor/dynamictext_format_test.go @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// TestFormattingInfoFromParamFormat: a format block starts from the Mendix +// defaults and applies only the keys the user set (ledger #75). +func TestFormattingInfoFromParamFormat(t *testing.T) { + if fi := formattingInfoFromParamFormat(nil); fi != nil { + t.Errorf("nil format must yield nil FormattingInfo (keeps default write path), got %+v", fi) + } + if fi := formattingInfoFromParamFormat(&ast.ParamFormatV3{}); fi != nil { + t.Errorf("empty format must yield nil, got %+v", fi) + } + + f := &ast.ParamFormatV3{Props: []ast.ParamFormatProp{ + {Key: "decimalprecision", Value: "4"}, + {Key: "groupdigits", Value: "true"}, + }} + fi := formattingInfoFromParamFormat(f) + if fi == nil { + t.Fatal("expected FormattingInfo") + } + if fi.DecimalPrecision != 4 || !fi.GroupDigits { + t.Errorf("decimal/group = %d/%v, want 4/true", fi.DecimalPrecision, fi.GroupDigits) + } + // Untouched fields keep the defaults (so the writer stays schema-aligned). + if fi.DateFormat != "Date" || fi.EnumFormat != "Text" { + t.Errorf("defaults not preserved: DateFormat=%q EnumFormat=%q", fi.DateFormat, fi.EnumFormat) + } + + dt := formattingInfoFromParamFormat(&ast.ParamFormatV3{Props: []ast.ParamFormatProp{ + {Key: "dateformat", Value: "Custom"}, {Key: "customdateformat", Value: "dd-MM-yyyy"}, + }}) + if dt.DateFormat != "Custom" || dt.CustomDateFormat != "dd-MM-yyyy" { + t.Errorf("custom date: %q / %q", dt.DateFormat, dt.CustomDateFormat) + } +} + +func dtWidget(params []ast.ParamAssignmentV3, props map[string]any) *ast.WidgetV3 { + all := map[string]any{"ContentParams": params} + for k, v := range props { + all[k] = v + } + return &ast.WidgetV3{Name: "txt", Type: "dynamictext", Properties: all} +} + +func TestValidateDynamicTextFormatting(t *testing.T) { + fmtBlock := func(props ...ast.ParamFormatProp) []ast.ParamAssignmentV3 { + return []ast.ParamAssignmentV3{{Index: 1, Value: "Amount", Format: &ast.ParamFormatV3{Props: props}}} + } + tests := []struct { + name string + w *ast.WidgetV3 + wantSub string // "" = no violations expected + }{ + {"valid decimal", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "decimalprecision", Value: "2"}, + ast.ParamFormatProp{Key: "groupdigits", Value: "true"}), nil), ""}, + {"unknown key", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "decimalprecison", Value: "2"}), nil), "unknown format key"}, + {"bad decimal", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "decimalprecision", Value: "x"}), nil), "non-negative integer"}, + {"bad dateformat", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "dateformat", Value: "Nope"}), nil), "dateFormat must be one of"}, + {"bad enumformat", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "enumformat", Value: "Nope"}), nil), "enumFormat must be"}, + {"custom without Custom", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "customdateformat", Value: "yyyy"}), nil), "requires `dateFormat: Custom`"}, + {"widget-level format key", dtWidget(nil, map[string]any{"decimalPrecision": 2}), "per-parameter format"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := validateDynamicTextFormatting(tt.w, "page P") + if tt.wantSub == "" { + if len(got) != 0 { + t.Fatalf("expected no violations, got %d: %+v", len(got), got) + } + return + } + if len(got) == 0 { + t.Fatalf("expected a violation containing %q, got none", tt.wantSub) + } + found := false + for _, v := range got { + if v.RuleID == "MDL-WIDGET18" && strings.Contains(v.Message, tt.wantSub) { + found = true + } + } + if !found { + t.Errorf("no MDL-WIDGET18 with %q; got %+v", tt.wantSub, got) + } + }) + } +} + +func TestFormatParamFormatSuffix(t *testing.T) { + // Default FormattingInfo → empty suffix (unformatted params round-trip as before). + def := map[string]any{"FormattingInfo": map[string]any{ + "DecimalPrecision": int64(2), "GroupDigits": false, "DateFormat": "Date", "EnumFormat": "Text", "CustomDateFormat": "", + }} + if s := formatParamFormatSuffix(def); s != "" { + t.Errorf("default FormattingInfo must yield empty suffix, got %q", s) + } + non := map[string]any{"FormattingInfo": map[string]any{ + "DecimalPrecision": int64(4), "GroupDigits": true, "DateFormat": "DateTime", "EnumFormat": "Text", "CustomDateFormat": "", + }} + s := formatParamFormatSuffix(non) + for _, want := range []string{" format (", "decimalPrecision: 4", "groupDigits: true", "dateFormat: DateTime"} { + if !strings.Contains(s, want) { + t.Errorf("suffix %q missing %q", s, want) + } + } +} diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index ca64f76ad..4d46b6ec1 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -119,6 +119,7 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc out = append(out, validatePluggableWidgetProperties(w, registry, locationPrefix)...) out = append(out, validateWidgetVisibility(w, registry, locationPrefix)...) out = append(out, validateStaticWidget(w, locationPrefix)...) + out = append(out, validateDynamicTextFormatting(w, locationPrefix)...) out = append(out, validateDatasourceXPathAssociationEmpty(w, locationPrefix)...) out = append(out, validateComboBoxAssociation(w, locationPrefix)...) // Unknown-property warning applies only to built-in widgets; pluggable @@ -578,6 +579,12 @@ func validateStaticWidgetUnknownProps(w *ast.WidgetV3, locationPrefix string) [] if isKnownStaticWidgetProp(key) { continue } + // Dynamic-text format keys placed at the widget level are reported by + // MDL-WIDGET18 (with actionable move-into-format-block guidance); don't + // also warn about them here. + if paramFormatKeys[strings.ToLower(key)] { + continue + } hint := "" if suggestion := nearestKey(key, staticWidgetKnownPropList); suggestion != "" { hint = fmt.Sprintf(" — did you mean `%s`?", suggestion) @@ -594,6 +601,104 @@ func validateStaticWidgetUnknownProps(w *ast.WidgetV3, locationPrefix string) [] return out } +// paramFormatKeys are the recognized keys inside a dynamic-text parameter format +// block, e.g. `{1} = Amount (decimalPrecision: 2, groupDigits: true)`. They map to +// the Mendix ClientTemplateParameter FormattingInfo fields. +var paramFormatKeys = map[string]bool{ + "decimalprecision": true, "groupdigits": true, + "dateformat": true, "customdateformat": true, "enumformat": true, +} + +var paramFormatKeyList = []string{"decimalPrecision", "groupDigits", "dateFormat", "customDateFormat", "enumFormat"} +var paramDateFormats = map[string]bool{"date": true, "datetime": true, "time": true, "custom": true} +var paramEnumFormats = map[string]bool{"text": true, "image": true} + +// validateDynamicTextFormatting (MDL-WIDGET18) checks per-parameter formatting on +// dynamic text. It (1) turns a widget-level format property (e.g. a bare +// `decimalPrecision:` on the widget) into an actionable ERROR pointing at the +// ContentParams format block — instead of the old silent drop (ledger #75) — and +// (2) validates the keys/values inside each format block so typos and bad enum +// values fail at `check` time rather than building wrong. +func validateDynamicTextFormatting(w *ast.WidgetV3, locationPrefix string) []linter.Violation { + if w == nil { + return nil + } + var out []linter.Violation + + // (1) Format keys placed at the widget level are silently dropped on write — + // formatting is per-parameter. Flag them with the correct location. + if strings.EqualFold(w.Type, "dynamictext") { + for key := range w.Properties { + if paramFormatKeys[strings.ToLower(key)] { + out = append(out, linter.Violation{ + RuleID: "MDL-WIDGET18", + Severity: linter.SeverityError, + Message: fmt.Sprintf( + "%s: widget `%s`: `%s` is a per-parameter format, not a widget property — put it in the ContentParams format block, e.g. `ContentParams: [{1} = Attr (%s: )]`. A widget-level `%s` is dropped on write.", + locationPrefix, w.Name, key, strings.ToLower(key), key, + ), + }) + } + } + } + + // (2) Validate the keys/values inside each parameter format block. + for _, p := range w.GetContentParams() { + if p.Format == nil { + continue + } + for _, fp := range p.Format.Props { + if !paramFormatKeys[fp.Key] { + hint := "" + if s := nearestKey(fp.Key, paramFormatKeyList); s != "" { + hint = fmt.Sprintf(" — did you mean `%s`?", s) + } + out = append(out, violation18(locationPrefix, w, + fmt.Sprintf("unknown format key `%s`%s", fp.Key, hint))) + continue + } + switch fp.Key { + case "decimalprecision": + if n, err := strconv.Atoi(fp.Value); err != nil || n < 0 { + out = append(out, violation18(locationPrefix, w, + fmt.Sprintf("decimalPrecision must be a non-negative integer, got `%s`", fp.Value))) + } + case "groupdigits": + if !strings.EqualFold(fp.Value, "true") && !strings.EqualFold(fp.Value, "false") { + out = append(out, violation18(locationPrefix, w, + fmt.Sprintf("groupDigits must be true or false, got `%s`", fp.Value))) + } + case "dateformat": + if !paramDateFormats[strings.ToLower(fp.Value)] { + out = append(out, violation18(locationPrefix, w, + fmt.Sprintf("dateFormat must be one of Date, DateTime, Time, Custom, got `%s`", fp.Value))) + } + case "enumformat": + if !paramEnumFormats[strings.ToLower(fp.Value)] { + out = append(out, violation18(locationPrefix, w, + fmt.Sprintf("enumFormat must be Text or Image, got `%s`", fp.Value))) + } + } + } + // customDateFormat is only meaningful with dateFormat: Custom. + if _, hasCustom := p.Format.Get("customdateformat"); hasCustom { + if df, _ := p.Format.Get("dateformat"); !strings.EqualFold(df, "Custom") { + out = append(out, violation18(locationPrefix, w, + "customDateFormat requires `dateFormat: Custom`")) + } + } + } + return out +} + +func violation18(locationPrefix string, w *ast.WidgetV3, msg string) linter.Violation { + return linter.Violation{ + RuleID: "MDL-WIDGET18", + Severity: linter.SeverityError, + Message: fmt.Sprintf("%s: widget `%s`: %s", locationPrefix, w.Name, msg), + } +} + // validateStaticWidget checks value-level constraints on built-in (non-pluggable) // widgets that the grammar can't express and that otherwise fail silently or at // build time rather than at `mxcli check` time. diff --git a/mdl/grammar/domains/MDLPage.g4 b/mdl/grammar/domains/MDLPage.g4 index 3d5eb935c..82132ba22 100644 --- a/mdl/grammar/domains/MDLPage.g4 +++ b/mdl/grammar/domains/MDLPage.g4 @@ -463,7 +463,21 @@ paramListV3 ; paramAssignmentV3 - : LBRACE NUMBER_LITERAL RBRACE EQUALS expression + : LBRACE NUMBER_LITERAL RBRACE EQUALS expression (FORMAT paramFormatV3)? + ; + +// Optional per-parameter formatting for a dynamic-text parameter, mapping to the +// Mendix ClientTemplateParameter FormattingInfo (decimalPrecision, groupDigits, +// dateFormat, customDateFormat, enumFormat). The FORMAT keyword introduces the +// block — a bare `(…)` after the expression is ambiguous with a function call +// because COLON is a valid expression (OQL division) operator. +// {1} = Amount FORMAT (decimalPrecision: 2, groupDigits: true) +paramFormatV3 + : LPAREN paramFormatPropV3 (COMMA paramFormatPropV3)* RPAREN + ; + +paramFormatPropV3 + : IDENTIFIER COLON propertyValueV3 ; // V3 Render modes diff --git a/mdl/visitor/visitor_page_v3.go b/mdl/visitor/visitor_page_v3.go index 795174d52..d0df006e2 100644 --- a/mdl/visitor/visitor_page_v3.go +++ b/mdl/visitor/visitor_page_v3.go @@ -1146,10 +1146,42 @@ func buildParamAssignmentV3(ctx parser.IParamAssignmentV3Context) ast.ParamAssig if expr := paCtx.Expression(); expr != nil { param.Value = stripExpressionIdentifierQuotes(expr.GetText()) } + if fmtCtx := paCtx.ParamFormatV3(); fmtCtx != nil { + param.Format = buildParamFormatV3(fmtCtx) + } return param } +// buildParamFormatV3 collects the raw key/value pairs of a parameter format +// block, e.g. `(decimalPrecision: 2, groupDigits: true)`. Keys are lowercased; +// string values have surrounding quotes stripped. Validation of keys/values +// happens later at check time (validate_widgets), not here. +func buildParamFormatV3(ctx parser.IParamFormatV3Context) *ast.ParamFormatV3 { + fc, ok := ctx.(*parser.ParamFormatV3Context) + if !ok { + return nil + } + f := &ast.ParamFormatV3{} + for _, p := range fc.AllParamFormatPropV3() { + pp, ok := p.(*parser.ParamFormatPropV3Context) + if !ok { + continue + } + id := pp.IDENTIFIER() + vc := pp.PropertyValueV3() + if id == nil || vc == nil { + continue + } + val := strings.Trim(vc.GetText(), "'\"") + f.Props = append(f.Props, ast.ParamFormatProp{ + Key: strings.ToLower(id.GetText()), + Value: val, + }) + } + return f +} + // buildXPathString builds a WHERE string from xpath constraints and and/or operators. // xpathTokenRe matches a Mendix XPath token like [%CurrentUser%] or // [%UserRole_Admin%]. The body is anything but % or ]. diff --git a/mdl/visitor/visitor_test.go b/mdl/visitor/visitor_test.go index 75254310f..c0ce76449 100644 --- a/mdl/visitor/visitor_test.go +++ b/mdl/visitor/visitor_test.go @@ -2574,6 +2574,31 @@ func TestKeywordWidgetName(t *testing.T) { } } +// TestParamFormatBlock guards ledger #75: a dynamic-text content parameter may +// carry a per-parameter FORMAT block, parsed into ParamAssignmentV3.Format.Props. +func TestParamFormatBlock(t *testing.T) { + prog, errs := Build(`create page M.P ( Title: 'P', Layout: Atlas_Core.Atlas_Default ) { + container body { + dynamictext amt (Content: '{1}', ContentParams: [{1} = Amount format (decimalPrecision: 4, groupDigits: true)]) + } +}`) + if len(errs) > 0 { + t.Fatalf("parse error: %v", errs[0]) + } + page := prog.Statements[0].(*ast.CreatePageStmtV3) + dt := page.Widgets[0].Children[0] + params := dt.GetContentParams() + if len(params) != 1 || params[0].Format == nil { + t.Fatalf("expected 1 param with a format block, got %+v", params) + } + if v, ok := params[0].Format.Get("decimalprecision"); !ok || v != "4" { + t.Errorf("decimalPrecision = %q,%v want 4", v, ok) + } + if v, ok := params[0].Format.Get("groupdigits"); !ok || v != "true" { + t.Errorf("groupDigits = %q,%v want true", v, ok) + } +} + func widgetName(ws []*ast.WidgetV3) string { if len(ws) == 0 { return "" diff --git a/sdk/mpr/writer_widgets.go b/sdk/mpr/writer_widgets.go index d5e9654ce..0d59d9a2e 100644 --- a/sdk/mpr/writer_widgets.go +++ b/sdk/mpr/writer_widgets.go @@ -646,14 +646,32 @@ func serializeClientTemplateParameter(param *pages.ClientTemplateParameter) bson // EnumFormat / GroupDigits). Writing TimeFormat here triggers Studio // Pro CE0463 "widget definition changed" on pluggable widgets that // embed this struct (e.g. Gallery / DataGrid2 column captions). + // + // Use the parameter's per-parameter formatting when present; a nil + // FormattingInfo reproduces the previous hardcoded defaults, so every + // unformatted parameter is byte-identical to before. + dateFormat, customDateFormat, enumFormat := "Date", "", "Text" + decimalPrecision := int64(2) + groupDigits := false + if fi := param.FormattingInfo; fi != nil { + if fi.DateFormat != "" { + dateFormat = fi.DateFormat + } + customDateFormat = fi.CustomDateFormat + if fi.EnumFormat != "" { + enumFormat = fi.EnumFormat + } + decimalPrecision = int64(fi.DecimalPrecision) + groupDigits = fi.GroupDigits + } formattingInfo := bson.D{ {Key: "$ID", Value: idToBsonBinary(generateUUID())}, {Key: "$Type", Value: "Forms$FormattingInfo"}, - {Key: "CustomDateFormat", Value: ""}, - {Key: "DateFormat", Value: "Date"}, - {Key: "DecimalPrecision", Value: int64(2)}, - {Key: "EnumFormat", Value: "Text"}, - {Key: "GroupDigits", Value: false}, + {Key: "CustomDateFormat", Value: customDateFormat}, + {Key: "DateFormat", Value: dateFormat}, + {Key: "DecimalPrecision", Value: decimalPrecision}, + {Key: "EnumFormat", Value: enumFormat}, + {Key: "GroupDigits", Value: groupDigits}, } // Build SourceVariable if present (references a page/snippet parameter) diff --git a/sdk/pages/pages_widgets_input.go b/sdk/pages/pages_widgets_input.go index b0ded0ce0..167cf7b75 100644 --- a/sdk/pages/pages_widgets_input.go +++ b/sdk/pages/pages_widgets_input.go @@ -39,7 +39,8 @@ type TextArea struct { type FormattingInfo struct { model.BaseElement DateFormat string `json:"dateFormat,omitempty"` - TimeFormat string `json:"timeFormat,omitempty"` + CustomDateFormat string `json:"customDateFormat,omitempty"` // pattern paired with DateFormat="Custom" + TimeFormat string `json:"timeFormat,omitempty"` // not a Forms$FormattingInfo schema field — never serialized (CE0463) DecimalPrecision int `json:"decimalPrecision,omitempty"` GroupDigits bool `json:"groupDigits,omitempty"` EnumFormat string `json:"enumFormat,omitempty"` From 741bac06b947294e0c59ddd58793fd667f00a005 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 09:35:36 +0000 Subject: [PATCH 28/29] fix(pages): bind non-String dynamic-text params as AttributeRef so formatting applies (ledger #76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the FORMAT block (#75), from runtime test feedback: the format block wrote valid FormattingInfo and `mx check` passed, but values rendered *unformatted* — a Decimal showed `-12` not `-12.00`. Root cause: a bare non-String attribute (Decimal/DateTime/…) was serialized as `Expression: toString($currentObject/Attr)`, and Mendix applies FormattingInfo **only to attribute-bound parameters** — an Expression parameter bypasses it. The BSON was valid but inert. Bind a bare non-String attribute as a structured AttributeRef instead. The runtime then renders it through the parameter's FormattingInfo, exactly as Studio Pro does. `toString()` was never required by mxbuild — an AttributeRef for a Decimal/DateTime in a text template passes `mx check` → 0 errors (verified on 11.12.1), and engine read-parity holds. Also fixes the MDL-WIDGET18 hint, which omitted the required `format` keyword (`[{1} = Attr format (…)]`). Scope note: the rarer `$param.Attr` non-String path (same function) still uses `toString()`; left for a follow-up. DataGrid2 dynamic-text *columns* (ledger #77) are a separate unimplemented feature. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JXnEgoc2NQP1Y2TWMCMXC4 --- .claude/skills/fix-issue.md | 1 + mdl/executor/cmd_pages_builder_v3.go | 24 +++++++++++------------- mdl/executor/validate_widgets.go | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 69ebfdd6d..53592ff03 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -326,6 +326,7 @@ cases for these three BSON types — they fell to `default: return nil`. | Every authored **Data grid 2** fails `mx check` with **CE0463** on the *bundled* Data Widgets 3.4 (Mendix 11.12/11.13) — but only on the **legacy** engine; `modelsdk` is clean on the same script. Appeared the moment `syncDefinitionAttrs` was corrected to write to `ValueType` (the row above), i.e. the moment that pass first actually executed | The Mendix pluggable-widget XML schema defaults `required` to **true**; only an explicit `required="false"` is optional. `sdk/widgets/mpk` read a missing attribute as `false` (`p.Required == "true"`), so the now-live sync overwrote 24 correct `true`s with `false` on DataGrid2 3.4 (which omits `required=` on 24 of its 40 properties). `modelsdk/widgets/mpk` already read it correctly (`p.Required != "false"`, fixed under #600) — the engines had silently diverged on the default | `sdk/widgets/mpk/mpk.go` (all three `PropertyDef` construction sites: `walkPropertyGroup` top-level + nested-direct, `collectNestedProperties`) | Read `p.Required != "false"` so absent means true, matching `modelsdk` and `mx update-widgets`. **Generalisable — the shape to look for**: when two engines carry parallel copies of a parser, a fix applied to one leaves a *latent* divergence in the other that stays invisible until some unrelated change starts consuming the value. Grep the sibling package for the same expression before assuming a defect is engine-specific. **Diagnosis trap hit here**: an A/B ran out of disk mid-run, the exec silently created no widgets, and `mx check` reported 0 CE0463 — reading as "the pre-fix binary is clean". Always assert the artifact exists (`show widgets | grep `) before trusting a zero. Unit repro `sdk/widgets/mpk/required_default_test.go`; integration `TestMxCheck_DataGridPage`/`TestMxCheck_DataGridNoColumns`. Issue #716 | | Freshly authored **drop-down filter** widgets fail `mx check` with **CE0463** on Data Widgets 3.10 while every other widget in the same script is clean. The stored BSON is missing `ValueType/AllowUpload` on all 25 ValueTypes and carries `Required=false` on `refCaption`/`refCaptionExp`, which the 3.10 XML declares `required="true"` — so the embedded template looks like the culprit, but replacing it changes nothing | `AugmentTemplate` never ran on this widget. Its opening guard — "nothing to add or remove at top level, and no nested children" — was written when the function only synced the property SET, and returns before the **six value-level passes** bolted on later (`reconcileEnumValues`, `reconcilePropertyMetadata`, `reconcileValueTypesFromMPK`, `completeValueTypeEnvelope`, `reorderPropertyTypes`, `syncDefinitionAttrs`). DataGrid2 never hit it because `columns` has nested children; the drop-down filter declares exactly the 25 keys the 11.6-era template already has, so it took the exit every time | `modelsdk/widgets/augment.go` + `sdk/widgets/augment.go` (`AugmentTemplate`) | Wrap the add/remove block in `if len(missing) > 0 \|\| len(stale) > 0 \|\| hasNestedChildren { … }` instead of returning early, so the value-level passes always run. **Generalisable — the shape to look for**: an early return placed correctly for a function's original job silently disables everything appended after it. When a pass "does nothing" for one input and works for others, check whether it *reached* the pass before theorising about the data (same root shape as the `ValueType` no-op two rows up). The `hasNestedChildren` clause in that guard is the tell: someone had already patched around the same bug for one widget rather than fixing it. **Diagnosis method**: a probe calling `augmentFromMPK` directly and counting `ValueTypes with AllowUpload` — 0/25 for the drop-down filter vs 44/44 for Gallery — localised it to "augment did not run" in one step, before any BSON theorising. Result: modelsdk on DW 3.10 went 2 → **0**. Tests `TestAugmentTemplate_MatchingKeysStillReconcilesValues` in both engines. Issue #716 | | A **Decimal/DateTime** in `dynamictext` always rendered with the hardcoded default format ("5068.38000000"); no MDL way to set the per-parameter Format, and a widget-level `decimalPrecision:` was **silently dropped** (mxcli check ✓, exec ✓, but gone) | The model always stored `ClientTemplateParameter.FormattingInfo`, but **all three writers hardcoded it** (`DecimalPrecision:2, GroupDigits:false, DateFormat:Date, EnumFormat:Text`) and ignored `param.FormattingInfo`; the grammar had no syntax to set it and DESCRIBE dropped it on read too | grammar `mdl/grammar/domains/MDLPage.g4` (`paramAssignmentV3` + `paramFormatV3`), `mdl/ast/ast_page_v3.go` (`ParamFormatV3`), `mdl/visitor/visitor_page_v3.go` (`buildParamFormatV3`), builder `mdl/executor/cmd_pages_builder_v3_widgets.go` (`formattingInfoFromParamFormat`), writers `mdl/backend/modelsdk/widget_write.go` (`formattingInfoToGen`) + `sdk/mpr/writer_widgets.go` (`serializeClientTemplateParameter`), describe `cmd_pages_describe_output.go` (`formatParamFormatSuffix`), validate `validate_widgets.go` (`validateDynamicTextFormatting`/MDL-WIDGET18) | Add a per-param `FORMAT (decimalPrecision: N, groupDigits: bool, dateFormat: …, customDateFormat: '…', enumFormat: …)` block: `{1} = Amount format (decimalPrecision: 2, groupDigits: true)`. The **FORMAT keyword is required** — a bare `(…)` after the value is ambiguous with a function call because `:` is OQL division in expressions. Writers use the param's FormattingInfo when set, else the same hardcoded defaults (nil → byte-identical to before, zero risk to existing widgets). MDL-WIDGET18 turns a widget-level format key into an actionable error (no more silent drop) and validates keys/enums. Verified: exec → `mx check` (11.12.1) 0 errors + DESCRIBE round-trip. Repro `mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl`. Ledger #75 | +| dynamic-text `format (…)` writes valid FormattingInfo, `mx check` ✓, but **renders unformatted** at runtime — a Decimal shows `-12` not `-12.00`, dates ignore the format | The parameter was serialized as `Expression: toString($currentObject/Attr)` (a non-String attribute was wrapped in `toString()`), and Mendix applies FormattingInfo **only to attribute-bound** params — an Expression param bypasses it. The BSON was valid but inert; `mx check` and DESCRIBE can't catch a *render* problem (the exact trap the runtime-verify skill exists for — I shipped #75 without it and the tester caught this) | `mdl/executor/cmd_pages_builder_v3.go` (`resolveTemplateAttributePathFull`, the bare-attribute branch) | Bind a bare non-String attribute as a structured **`AttributeRef`**, not a `toString()` Expression — the runtime then renders it through FormattingInfo, exactly as Studio Pro does. `toString()` was never required by mxbuild (AttributeRef for a Decimal/DateTime in a text template passes `mx check` → 0 errors). Engine read-parity holds. Note the `$param.Attr` non-String path (same function) still uses `toString()` — rarer, left for follow-up. Ledger #76 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/mdl/executor/cmd_pages_builder_v3.go b/mdl/executor/cmd_pages_builder_v3.go index a6cd3d14c..70aa6d9f7 100644 --- a/mdl/executor/cmd_pages_builder_v3.go +++ b/mdl/executor/cmd_pages_builder_v3.go @@ -1504,9 +1504,10 @@ func (pb *pageBuilder) resolveTemplateAttributePath(attrRef string) string { // When attrRef is $paramName.Attribute (where paramName is a page/snippet parameter), // it sets SourceVariable to paramName and AttributeRef to the resolved entity path. // -// For non-String attributes (Integer, Decimal, DateTime, Boolean, etc.), the binding -// is automatically converted to a toString() expression since DYNAMICTEXT template -// parameters require String values. +// Non-String attributes (Integer, Decimal, DateTime, Boolean, …) bind as a +// structured AttributeRef, not a `toString(...)` Expression — the runtime renders +// them through the parameter's FormattingInfo (decimalPrecision, dateFormat, …), +// which an Expression parameter bypasses entirely (ledger #76). func (pb *pageBuilder) resolveTemplateAttributePathFull(attrRef string, param *pages.ClientTemplateParameter) { if attrRef == "" { return @@ -1565,16 +1566,13 @@ func (pb *pageBuilder) resolveTemplateAttributePathFull(attrRef string, param *p return } - // For other patterns, resolve and check type - resolved := pb.resolveTemplateAttributePath(attrRef) - if !strings.HasPrefix(attrRef, "$") && pb.isNonStringAttribute(resolved) { - // Convert bare attribute names to toString() for non-String types. - // Only for bare names (e.g., "TotalOrders") in DataView context, - // not for $param.Attr references which are resolved via SourceVariable. - param.Expression = "toString($currentObject/" + attrRef + ")" - return - } - param.AttributeRef = resolved + // For other patterns, resolve to a structured AttributeRef. A non-String + // attribute (Decimal/DateTime/Integer/…) binds directly as an AttributeRef — + // the runtime renders it via the parameter's FormattingInfo, exactly as Studio + // Pro does. (Previously mxcli wrapped non-String attrs in a + // `toString($currentObject/Attr)` Expression, which bypassed FormattingInfo so + // decimalPrecision/dateFormat had no runtime effect — ledger #76.) + param.AttributeRef = pb.resolveTemplateAttributePath(attrRef) } // resolveTemplateAssociationPath resolves a template-parameter value that diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 4d46b6ec1..09dcf6e81 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -634,7 +634,7 @@ func validateDynamicTextFormatting(w *ast.WidgetV3, locationPrefix string) []lin RuleID: "MDL-WIDGET18", Severity: linter.SeverityError, Message: fmt.Sprintf( - "%s: widget `%s`: `%s` is a per-parameter format, not a widget property — put it in the ContentParams format block, e.g. `ContentParams: [{1} = Attr (%s: )]`. A widget-level `%s` is dropped on write.", + "%s: widget `%s`: `%s` is a per-parameter format, not a widget property — put it in the ContentParams format block, e.g. `ContentParams: [{1} = Attr format (%s: )]`. A widget-level `%s` is dropped on write.", locationPrefix, w.Name, key, strings.ToLower(key), key, ), }) From 55921cccd575e412f371e4df3658cc2bcbfad7f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 10:17:51 +0000 Subject: [PATCH 29/29] =?UTF-8?q?fix(pages):=20DataGrid2=20dynamic-text=20?= =?UTF-8?q?columns=20=E2=80=94=20carry=20FORMAT=20block=20+=20fix=20CE0463?= =?UTF-8?q?=20(ledger=20#77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DataGrid2 column can render its cell as a dynamic-text template (ShowContentAs: dynamicText) with the same per-parameter FORMAT block as a listview dynamictext. Two defects made this fail end-to-end: 1. The per-parameter `format (...)` block was silently dropped for columns. The shared buildClientTemplateParams helper (object-list column path and ALTER PAGE column path) never read p.Format, and the column-scoped serializer SerializeColumnClientTemplateParameter hardcoded FormattingInfo. Route the FORMAT block through the shared helper and honour param.FormattingInfo in the serializer (nil stays byte-identical). DESCRIBE now round-trips the format suffix for columns too. 2. CE0463 "the definition of this widget has changed" on load. A dynamic-text column (no attribute, no content widgets) was classified as the default item kind, whose tooltip serialized as TextTemplate:null. Studio Pro stores an empty Forms$ClientTemplate there (as for an attribute column). Add an itemKindDynamicText classification and give it the attribute column's tooltip empty-ClientTemplate rule (exportValue stays null). The CE0463 was surfaced by raw `mx check` / the serve build; `mxcli docker check` masks it by running `mx update-widgets` first. Diagnosed via a path-level flatten-diff of the datagrid subtree against the update-widgets reconciled reference, which isolated columns[dynamicText]/tooltip/TextTemplate. Verified end-to-end on Mendix 11.12.1: raw `mx check` 0 errors (pre-fix: 1x CE0463), DESCRIBE round-trips the format block, and the cell renders the formatted value at runtime (run --local + Playwright: raw DB -1234.5 -> "-1,234.50"). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JXnEgoc2NQP1Y2TWMCMXC4 --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/create-page.md | 17 +++ .../ledger-77-datagrid-dynamictext-column.mdl | 57 ++++++++++ mdl/backend/widgetobj/builder.go | 52 +++++++-- .../widgetobj/column_formatting_test.go | 103 ++++++++++++++++++ mdl/executor/cmd_pages_builder_v3_widgets.go | 6 + mdl/executor/cmd_pages_describe_pluggable.go | 12 ++ 7 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 mdl-examples/bug-tests/ledger-77-datagrid-dynamictext-column.mdl create mode 100644 mdl/backend/widgetobj/column_formatting_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 53592ff03..da6f88926 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -327,6 +327,7 @@ cases for these three BSON types — they fell to `default: return nil`. | Freshly authored **drop-down filter** widgets fail `mx check` with **CE0463** on Data Widgets 3.10 while every other widget in the same script is clean. The stored BSON is missing `ValueType/AllowUpload` on all 25 ValueTypes and carries `Required=false` on `refCaption`/`refCaptionExp`, which the 3.10 XML declares `required="true"` — so the embedded template looks like the culprit, but replacing it changes nothing | `AugmentTemplate` never ran on this widget. Its opening guard — "nothing to add or remove at top level, and no nested children" — was written when the function only synced the property SET, and returns before the **six value-level passes** bolted on later (`reconcileEnumValues`, `reconcilePropertyMetadata`, `reconcileValueTypesFromMPK`, `completeValueTypeEnvelope`, `reorderPropertyTypes`, `syncDefinitionAttrs`). DataGrid2 never hit it because `columns` has nested children; the drop-down filter declares exactly the 25 keys the 11.6-era template already has, so it took the exit every time | `modelsdk/widgets/augment.go` + `sdk/widgets/augment.go` (`AugmentTemplate`) | Wrap the add/remove block in `if len(missing) > 0 \|\| len(stale) > 0 \|\| hasNestedChildren { … }` instead of returning early, so the value-level passes always run. **Generalisable — the shape to look for**: an early return placed correctly for a function's original job silently disables everything appended after it. When a pass "does nothing" for one input and works for others, check whether it *reached* the pass before theorising about the data (same root shape as the `ValueType` no-op two rows up). The `hasNestedChildren` clause in that guard is the tell: someone had already patched around the same bug for one widget rather than fixing it. **Diagnosis method**: a probe calling `augmentFromMPK` directly and counting `ValueTypes with AllowUpload` — 0/25 for the drop-down filter vs 44/44 for Gallery — localised it to "augment did not run" in one step, before any BSON theorising. Result: modelsdk on DW 3.10 went 2 → **0**. Tests `TestAugmentTemplate_MatchingKeysStillReconcilesValues` in both engines. Issue #716 | | A **Decimal/DateTime** in `dynamictext` always rendered with the hardcoded default format ("5068.38000000"); no MDL way to set the per-parameter Format, and a widget-level `decimalPrecision:` was **silently dropped** (mxcli check ✓, exec ✓, but gone) | The model always stored `ClientTemplateParameter.FormattingInfo`, but **all three writers hardcoded it** (`DecimalPrecision:2, GroupDigits:false, DateFormat:Date, EnumFormat:Text`) and ignored `param.FormattingInfo`; the grammar had no syntax to set it and DESCRIBE dropped it on read too | grammar `mdl/grammar/domains/MDLPage.g4` (`paramAssignmentV3` + `paramFormatV3`), `mdl/ast/ast_page_v3.go` (`ParamFormatV3`), `mdl/visitor/visitor_page_v3.go` (`buildParamFormatV3`), builder `mdl/executor/cmd_pages_builder_v3_widgets.go` (`formattingInfoFromParamFormat`), writers `mdl/backend/modelsdk/widget_write.go` (`formattingInfoToGen`) + `sdk/mpr/writer_widgets.go` (`serializeClientTemplateParameter`), describe `cmd_pages_describe_output.go` (`formatParamFormatSuffix`), validate `validate_widgets.go` (`validateDynamicTextFormatting`/MDL-WIDGET18) | Add a per-param `FORMAT (decimalPrecision: N, groupDigits: bool, dateFormat: …, customDateFormat: '…', enumFormat: …)` block: `{1} = Amount format (decimalPrecision: 2, groupDigits: true)`. The **FORMAT keyword is required** — a bare `(…)` after the value is ambiguous with a function call because `:` is OQL division in expressions. Writers use the param's FormattingInfo when set, else the same hardcoded defaults (nil → byte-identical to before, zero risk to existing widgets). MDL-WIDGET18 turns a widget-level format key into an actionable error (no more silent drop) and validates keys/enums. Verified: exec → `mx check` (11.12.1) 0 errors + DESCRIBE round-trip. Repro `mdl-examples/bug-tests/ledger-75-dynamictext-formatting.mdl`. Ledger #75 | | dynamic-text `format (…)` writes valid FormattingInfo, `mx check` ✓, but **renders unformatted** at runtime — a Decimal shows `-12` not `-12.00`, dates ignore the format | The parameter was serialized as `Expression: toString($currentObject/Attr)` (a non-String attribute was wrapped in `toString()`), and Mendix applies FormattingInfo **only to attribute-bound** params — an Expression param bypasses it. The BSON was valid but inert; `mx check` and DESCRIBE can't catch a *render* problem (the exact trap the runtime-verify skill exists for — I shipped #75 without it and the tester caught this) | `mdl/executor/cmd_pages_builder_v3.go` (`resolveTemplateAttributePathFull`, the bare-attribute branch) | Bind a bare non-String attribute as a structured **`AttributeRef`**, not a `toString()` Expression — the runtime then renders it through FormattingInfo, exactly as Studio Pro does. `toString()` was never required by mxbuild (AttributeRef for a Decimal/DateTime in a text template passes `mx check` → 0 errors). Engine read-parity holds. Note the `$param.Attr` non-String path (same function) still uses `toString()` — rarer, left for follow-up. Ledger #76 | +| A **DataGrid2 dynamic-text column** (`column x (ShowContentAs: dynamicText, Content: '{1}', ContentParams: [{1} = Attr format (…)])`) fails to open with **CE0463**, *and* its `format (…)` block is silently dropped. `mxcli docker check` hides the CE0463 because it runs `mx update-widgets` first; raw `mx check` and `mxbuild --serve` (run --local) surface it | Two independent gaps in the full-page object-list column path. (1) The shared `buildClientTemplateParams` never read the parsed `p.Format`, and the column-scoped serializer `SerializeColumnClientTemplateParameter` **hardcoded** FormattingInfo — so a column param's format was dropped at both write points. (2) A dynamic-text column has no attribute and no content widgets, so `detectObjectListItemKind` classified it as the **default** kind, which has no empty-ClientTemplate rules → its `tooltip` serialized as `TextTemplate:null`. Studio Pro stores an **empty `Forms$ClientTemplate`** there (as for an attribute column), so the widget failed to load | `mdl/executor/cmd_pages_builder_v3_widgets.go` (`buildClientTemplateParams` → apply `formattingInfoFromParamFormat(p.Format)`), `mdl/backend/widgetobj/builder.go` (`SerializeColumnClientTemplateParameter` honours `param.FormattingInfo`; new `itemKindDynamicText` + `emptyClientTemplateRules` tooltip entry; `detectObjectListItemKind`), describe `mdl/executor/cmd_pages_describe_pluggable.go` (`extractTextTemplateParameters` zips the format suffix) | Route the FORMAT block through the *shared* params helper (fixes the object-list column path **and** the ALTER PAGE column path at once) and stop hardcoding FormattingInfo in the column serializer. Classify a `showContentAs: dynamicText` column as its own item kind and give it the attribute column's `tooltip → empty CT` rule (exportValue stays null). **Diagnosis method that found the CE0463**: `update-widgets` on a *copy* cleared it → Case B (our BSON); a path-level flatten-diff of the datagrid subtree, mine vs the reconciled reference, isolated the single differing path `columns[dynamicText]/tooltip/TextTemplate` null↔empty. **Trap**: `docker check`'s built-in `update-widgets` masks the very defect you're hunting — measure with raw `mx check` or the serve build. Repro `mdl-examples/bug-tests/ledger-77-datagrid-dynamictext-column.mdl`; verified end-to-end (raw `mx check` 0 errors + Playwright cell renders `-1,234.50`). Ledger #77 | **Key insight:** `microflows$ListRange` stores offset/limit inside a nested `CustomRange` map — must cast `raw["CustomRange"].(map[string]any)` before diff --git a/.claude/skills/mendix/create-page.md b/.claude/skills/mendix/create-page.md index ba131ff63..d49da690c 100644 --- a/.claude/skills/mendix/create-page.md +++ b/.claude/skills/mendix/create-page.md @@ -424,6 +424,23 @@ datagrid gridName ( Only non-default column properties appear in `describe page` output. +**Dynamic-text columns (`ShowContentAs: dynamicText`):** a column can render its cell as a formatted text template instead of a bare attribute — the same `Content` / `ContentParams` / `format (...)` syntax as a `dynamictext` widget (see below). The column needs no `Attribute`; give it a `Caption` for the header. + +```sql +datagrid gridName (datasource: database from Module.Entity) { + -- Decimal with 2 decimals + thousands separator: renders e.g. "Amt: -1,234.50" + column amount ( + Caption: 'Amount', + ShowContentAs: dynamicText, + Content: 'Amt: {1}', + ContentParams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)] + ) + column due (attribute: DueOn, caption: 'Due') +} +``` + +The `format (...)` block accepts `decimalPrecision`, `groupDigits`, `dateFormat` (`Date` / `DateTime` / `Time` / `Custom`), `customDateFormat`, and `enumFormat` (`Text` / `Image`). Formatting is applied by Mendix only to **attribute-bound** parameters — bind the bare attribute (`Amount`), not `toString(...)`. + ```sql column colPrice ( attribute: Price, caption: 'Unit Price', diff --git a/mdl-examples/bug-tests/ledger-77-datagrid-dynamictext-column.mdl b/mdl-examples/bug-tests/ledger-77-datagrid-dynamictext-column.mdl new file mode 100644 index 000000000..ff46f25c9 --- /dev/null +++ b/mdl-examples/bug-tests/ledger-77-datagrid-dynamictext-column.mdl @@ -0,0 +1,57 @@ +-- ============================================================================ +-- Ledger finding #77: DataGrid2 dynamic-text columns (Show: dynamic text) +-- ============================================================================ +-- +-- A DataGrid2 column can render its cell as a dynamic-text template +-- (ShowContentAs: dynamicText) instead of a bare attribute — with the same +-- per-parameter FORMAT block as a listview dynamictext (ledger #75/#76). +-- +-- Two defects were fixed to make this work end-to-end: +-- +-- 1. The FORMAT block was dropped for column params. The shared +-- buildClientTemplateParams helper (object-list column path + ALTER PAGE +-- path) ignored the parsed `format (...)`, and the column-scoped +-- serializer hardcoded FormattingInfo. A `format` block on a column param +-- is now written and round-trips through DESCRIBE. +-- +-- 2. CE0463 "the definition of this widget has changed" on load. A +-- dynamic-text column (no attribute, no custom-content widgets) was +-- classified as the default item kind and serialized its `tooltip` as +-- TextTemplate:null. Studio Pro stores an empty Forms$ClientTemplate there +-- (as it does for an attribute column), so the widget failed to load. The +-- column is now classified as a dynamic-text kind and emits the empty +-- ClientTemplate. +-- +-- Verified: mxcli exec -> raw `mx check` (Mendix 11.12.1, no update-widgets) = +-- 0 errors; DESCRIBE round-trips the format block; the cell renders the +-- formatted value at runtime (run --local + Playwright). +-- ============================================================================ + +create entity Ledger.LineItem ( + Amount: Decimal, + DueOn: DateTime +); + +create page Ledger.LineItemsGrid ( + Title: 'Line items', + Layout: Atlas_Core.Atlas_Default +) +{ + layoutgrid grid { + row r { + column c (DesktopWidth: 12) { + datagrid dg (DataSource: DATABASE FROM Ledger.LineItem) { + -- Dynamic-text cell: Decimal with 2 decimals and a thousands separator. + column amount ( + Caption: 'Amount', + ShowContentAs: dynamicText, + Content: 'Amt: {1}', + ContentParams: [{1} = Amount format (decimalPrecision: 2, groupDigits: true)] + ) + -- Plain attribute column for contrast. + column due (Attribute: DueOn, Caption: 'Due') + } + } + } + } +} diff --git a/mdl/backend/widgetobj/builder.go b/mdl/backend/widgetobj/builder.go index d2febccd8..7e692ec90 100644 --- a/mdl/backend/widgetobj/builder.go +++ b/mdl/backend/widgetobj/builder.go @@ -317,6 +317,7 @@ type objectListItemKind string const ( itemKindAttribute objectListItemKind = "attribute" itemKindCustomContent objectListItemKind = "customcontent" + itemKindDynamicText objectListItemKind = "dynamictext" itemKindDefault objectListItemKind = "" ) @@ -331,6 +332,13 @@ func detectObjectListItemKind(specByKey map[string]backend.ObjectListItemPropert if len(childWidgets["content"]) > 0 { return itemKindCustomContent } + // A dynamic-text column (Show: dynamicText) has no attribute binding and no + // content widgets, so it would otherwise fall through to itemKindDefault and + // miss the tooltip empty-ClientTemplate convention Studio Pro applies to it + // (CE0463, ledger #77). Detect it from the showContentAs primitive. + if sca, ok := specByKey["showContentAs"]; ok && strings.EqualFold(sca.PrimitiveVal, "dynamicText") { + return itemKindDynamicText + } if attr, ok := specByKey["attribute"]; ok && attr.AttributePath != "" { return itemKindAttribute } @@ -345,16 +353,22 @@ func detectObjectListItemKind(specByKey map[string]backend.ObjectListItemPropert // Source: c3d61af1 in datagrid_builder.go — Studio Pro's per-column-kind // convention for DataGrid columns (verified against Cars_Overview): // -// property attribute column custom-content column -// tooltip empty CT null -// exportValue null empty CT -// dynamicText null null +// property attribute column dynamic-text column custom-content column +// tooltip empty CT empty CT null +// exportValue null null empty CT +// dynamicText null (the cell template) null +// +// The dynamic-text column matches the attribute column for tooltip/exportValue +// (verified against a Studio-Pro `mx update-widgets` reconciliation, ledger #77). var emptyClientTemplateRules = map[string]map[string]map[objectListItemKind]map[string]bool{ "com.mendix.widget.web.datagrid.Datagrid": { "columns": { itemKindAttribute: { "tooltip": true, }, + itemKindDynamicText: { + "tooltip": true, + }, itemKindCustomContent: { "exportValue": true, }, @@ -1503,14 +1517,34 @@ func SerializeColumnClientTemplateParameter(param *pages.ClientTemplateParameter } } + // Use the parameter's per-parameter formatting when present; a nil + // FormattingInfo reproduces the previous hardcoded defaults, so every + // unformatted column parameter is byte-identical to before. Mirrors + // sdk/mpr/writer_widgets.go:serializeClientTemplateParameter so a + // `format (...)` block authored on a DataGrid2 dynamic-text column param + // reaches the runtime instead of being silently dropped (ledger #77). + dateFormat, customDateFormat, enumFormat := "Date", "", "Text" + decimalPrecision := int64(2) + groupDigits := false + if fi := param.FormattingInfo; fi != nil { + if fi.DateFormat != "" { + dateFormat = fi.DateFormat + } + customDateFormat = fi.CustomDateFormat + if fi.EnumFormat != "" { + enumFormat = fi.EnumFormat + } + decimalPrecision = int64(fi.DecimalPrecision) + groupDigits = fi.GroupDigits + } formattingInfo := bson.D{ {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, {Key: "$Type", Value: "Forms$FormattingInfo"}, - {Key: "CustomDateFormat", Value: ""}, - {Key: "DateFormat", Value: "Date"}, - {Key: "DecimalPrecision", Value: int64(2)}, - {Key: "EnumFormat", Value: "Text"}, - {Key: "GroupDigits", Value: false}, + {Key: "CustomDateFormat", Value: customDateFormat}, + {Key: "DateFormat", Value: dateFormat}, + {Key: "DecimalPrecision", Value: decimalPrecision}, + {Key: "EnumFormat", Value: enumFormat}, + {Key: "GroupDigits", Value: groupDigits}, } var sourceVariable any diff --git a/mdl/backend/widgetobj/column_formatting_test.go b/mdl/backend/widgetobj/column_formatting_test.go new file mode 100644 index 000000000..2864fe86d --- /dev/null +++ b/mdl/backend/widgetobj/column_formatting_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 + +package widgetobj + +import ( + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// TestSerializeColumnClientTemplateParameterFormattingInfo locks in that a +// DataGrid2 dynamic-text column's ClientTemplateParameter serializes the +// parameter's own FormattingInfo — not a hardcoded default. Before ledger #77 +// this serializer ignored param.FormattingInfo and always wrote +// DecimalPrecision:2 / GroupDigits:false / DateFormat:Date, so a +// `format (decimalPrecision: 0, groupDigits: true)` block authored on a column +// param was silently dropped and the cell rendered unformatted. +func TestSerializeColumnClientTemplateParameterFormattingInfo(t *testing.T) { + t.Run("honors per-parameter FormattingInfo", func(t *testing.T) { + param := &pages.ClientTemplateParameter{ + AttributeRef: "MyModule.Item.Amount", + FormattingInfo: &pages.FormattingInfo{ + DecimalPrecision: 0, + GroupDigits: true, + DateFormat: "Date", + EnumFormat: "Text", + }, + } + got := SerializeColumnClientTemplateParameter(param) + fi := findField(t, got, "FormattingInfo").(bson.D) + + if dp := findField(t, fi, "DecimalPrecision"); dp != int64(0) { + t.Errorf("DecimalPrecision = %v, want 0 (param's own value, not hardcoded 2)", dp) + } + if gd := findField(t, fi, "GroupDigits"); gd != true { + t.Errorf("GroupDigits = %v, want true (param's own value, not hardcoded false)", gd) + } + }) + + t.Run("custom date format", func(t *testing.T) { + param := &pages.ClientTemplateParameter{ + AttributeRef: "MyModule.Item.DueOn", + FormattingInfo: &pages.FormattingInfo{ + DateFormat: "Custom", + CustomDateFormat: "yyyy-MM-dd", + DecimalPrecision: 2, + EnumFormat: "Text", + }, + } + got := SerializeColumnClientTemplateParameter(param) + fi := findField(t, got, "FormattingInfo").(bson.D) + + if df := findField(t, fi, "DateFormat"); df != "Custom" { + t.Errorf("DateFormat = %v, want Custom", df) + } + if cdf := findField(t, fi, "CustomDateFormat"); cdf != "yyyy-MM-dd" { + t.Errorf("CustomDateFormat = %v, want yyyy-MM-dd", cdf) + } + }) + + t.Run("nil FormattingInfo keeps byte-identical defaults", func(t *testing.T) { + param := &pages.ClientTemplateParameter{AttributeRef: "MyModule.Item.Name"} + got := SerializeColumnClientTemplateParameter(param) + fi := findField(t, got, "FormattingInfo").(bson.D) + + if dp := findField(t, fi, "DecimalPrecision"); dp != int64(2) { + t.Errorf("DecimalPrecision = %v, want 2 (default)", dp) + } + if df := findField(t, fi, "DateFormat"); df != "Date" { + t.Errorf("DateFormat = %v, want Date (default)", df) + } + if gd := findField(t, fi, "GroupDigits"); gd != false { + t.Errorf("GroupDigits = %v, want false (default)", gd) + } + if ef := findField(t, fi, "EnumFormat"); ef != "Text" { + t.Errorf("EnumFormat = %v, want Text (default)", ef) + } + }) +} + +// TestDynamicTextColumnKind locks in that a DataGrid2 dynamic-text column +// (showContentAs=dynamicText, no attribute, no content widgets) is classified +// as itemKindDynamicText and therefore gets the tooltip empty-ClientTemplate +// convention Studio Pro applies — without it the column's tooltip serialized as +// null and the whole widget failed to load with CE0463 (ledger #77). +func TestDynamicTextColumnKind(t *testing.T) { + spec := map[string]backend.ObjectListItemProperty{ + "showContentAs": {PropertyKey: "showContentAs", Operation: "primitive", PrimitiveVal: "dynamicText"}, + } + if k := detectObjectListItemKind(spec, nil); k != itemKindDynamicText { + t.Fatalf("kind = %q, want %q", k, itemKindDynamicText) + } + if !shouldEmitEmptyClientTemplate("com.mendix.widget.web.datagrid.Datagrid", "columns", "tooltip", itemKindDynamicText) { + t.Error("dynamic-text column tooltip must emit an empty ClientTemplate (Studio Pro convention), not null") + } + // exportValue stays null for a dynamic-text column (matches the attribute column). + if shouldEmitEmptyClientTemplate("com.mendix.widget.web.datagrid.Datagrid", "columns", "exportValue", itemKindDynamicText) { + t.Error("dynamic-text column exportValue must stay null, not an empty ClientTemplate") + } +} diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 9fd649cf0..c3c543f86 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -153,6 +153,12 @@ func (pb *pageBuilder) buildClientTemplateParams(astParams []ast.ParamAssignment TypeName: "Forms$ClientTemplateParameter", }, } + // A per-parameter `format (...)` block carries the same FormattingInfo the + // standalone dynamictext widget honors (buildDynamicTextV3). This shared + // helper feeds DataGrid2 dynamic-text columns (object-list path) and the + // ALTER PAGE column path, so a `format` block authored on a column param + // reaches the runtime instead of being silently dropped (ledger #77). + param.FormattingInfo = formattingInfoFromParamFormat(p.Format) strVal, ok := p.Value.(string) if !ok { out = append(out, param) diff --git a/mdl/executor/cmd_pages_describe_pluggable.go b/mdl/executor/cmd_pages_describe_pluggable.go index 6b714463f..0c0742c8d 100644 --- a/mdl/executor/cmd_pages_describe_pluggable.go +++ b/mdl/executor/cmd_pages_describe_pluggable.go @@ -600,11 +600,16 @@ func extractTextTemplateParameters(ctx *ExecContext, textTemplate map[string]any return nil } var result []string + var suffixes []string // per-param format block " (decimalPrecision: 0, …)", "" when default for _, p := range params { pMap, ok := p.(map[string]any) if !ok { continue } + // One suffix per emitted param, in the same order, zipped in after the + // value string is chosen so a DataGrid2 dynamic-text column's per-parameter + // formatting round-trips through DESCRIBE (ledger #77). + suffixes = append(suffixes, formatParamFormatSuffix(pMap)) // Check for Expression first (literal value) if expr, ok := pMap["Expression"].(string); ok && expr != "" { result = append(result, expr) @@ -651,6 +656,13 @@ func extractTextTemplateParameters(ctx *ExecContext, textTemplate map[string]any // Parameter exists but has no binding result = append(result, "") } + // Append each param's format block to its value string; result and suffixes + // are aligned (one of each per param that had a valid pMap). + for i := range result { + if i < len(suffixes) && suffixes[i] != "" { + result[i] += suffixes[i] + } + } return result }