From 12bdd7c412af35e435a60e9ee02154991381991c Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 28 Aug 2026 00:03:40 +0200 Subject: [PATCH 1/2] test: add schema-driven round-trip and fixture tests Serialization and deserialization have no regression coverage beyond a handful of write-only assertions, which makes any refactor of that code unsafe. This adds two suites that lock in current behaviour. RoundTripTest checks serialization stability: serialize, load, serialize again, and require the two documents to match as an unordered set of lines. Node order within an @graph is not stable across a save and load cycle, so raw text comparison would be flaky, and comparing documents avoids deep object comparison entirely. Instances are populated by ommtest.helper.synthesizeInstance, which chooses values by property kind and reads cardinality and numeric bounds from the property validators. It never refers to a property by name, so it needs no maintenance as the model adds, moves or removes types and properties. Candidates are lazy and have fallbacks, so a validator the synthesizer does not anticipate degrades to a skipped property rather than a broken test. It populates 2349 of 2363 properties across all 292 types; the remainder are constrained by regular expressions, which is out of scope and reported as such. FixtureTest pins the document format against golden files, which the round-trip check cannot do because it only compares the library against itself. Fixture content is defined once in code and the golden files are generated from it, so refreshing after a model version bump is one call to ommtest.helper.regenerateFixtures plus a review of the diff. Round tripping all 292 types takes about three minutes, which is too slow for every commit. The suite runs a fixed-stride sample of 30 types by default and the full model when OPENMINDS_TEST_ALL_TYPES is set. The workflow sets it for push to main, where the openMINDS pipeline lands regenerated type classes, and for a new weekly schedule; pull requests run the sample. Types with known library defects are listed in ommtest.helper.knownRoundTripGap and reported as incomplete rather than failed, so the suite is usable as a gate while those defects are open. The controlled term case is expressed as a predicate rather than a list of 112 names. That file is expected to shrink as the defects are fixed. Co-Authored-By: Claude Opus 5 --- .github/workflows/run_tests.yml | 10 + .../+ommtest/+helper/buildFixtureCollection.m | 56 +++ .../+ommtest/+helper/controlledInstance.m | 36 ++ .../+ommtest/+helper/fixtureNamespaceTag.m | 21 + tools/tests/+ommtest/+helper/fixturePath.m | 13 + .../+ommtest/+helper/knownRoundTripGap.m | 64 +++ .../+ommtest/+helper/regenerateFixtures.m | 42 ++ .../+ommtest/+helper/roundTripTypeSelection.m | 37 ++ .../+ommtest/+helper/synthesizeInstance.m | 392 ++++++++++++++++++ .../fixtures/collection_ebrains_legacy.jsonld | 13 + tools/tests/fixtures/collection_omi.jsonld | 92 ++++ tools/tests/unitTests/FixtureTest.m | 129 ++++++ tools/tests/unitTests/RoundTripTest.m | 147 +++++++ 13 files changed, 1052 insertions(+) create mode 100644 tools/tests/+ommtest/+helper/buildFixtureCollection.m create mode 100644 tools/tests/+ommtest/+helper/controlledInstance.m create mode 100644 tools/tests/+ommtest/+helper/fixtureNamespaceTag.m create mode 100644 tools/tests/+ommtest/+helper/fixturePath.m create mode 100644 tools/tests/+ommtest/+helper/knownRoundTripGap.m create mode 100644 tools/tests/+ommtest/+helper/regenerateFixtures.m create mode 100644 tools/tests/+ommtest/+helper/roundTripTypeSelection.m create mode 100644 tools/tests/+ommtest/+helper/synthesizeInstance.m create mode 100644 tools/tests/fixtures/collection_ebrains_legacy.jsonld create mode 100644 tools/tests/fixtures/collection_omi.jsonld create mode 100644 tools/tests/unitTests/FixtureTest.m create mode 100644 tools/tests/unitTests/RoundTripTest.m diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4491fd188..ed121a0f7 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -15,6 +15,10 @@ on: branches: [ "main" ] # Allows for manually running this workflow from the Actions tab workflow_dispatch: + # Weekly full-model run, so the exhaustive round-trip sweep is exercised + # even in weeks without a push to main + schedule: + - cron: '0 3 * * 1' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -61,11 +65,17 @@ jobs: sarif_file: docs/reports/code_issues.sarif # Run all tests in the project. + # The round-trip test covers every openMINDS type when + # OPENMINDS_TEST_ALL_TYPES is set, which takes several minutes. Pull + # requests run a representative sample instead. The full sweep runs + # on push to main, which is where the openMINDS pipeline lands + # regenerated type classes, and on the weekly schedule. - name: Run tests if: always() uses: matlab-actions/run-command@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENMINDS_TEST_ALL_TYPES: ${{ (github.event_name == 'pull_request') && '0' || '1' }} with: command: | doCreateBadge = "${{ matrix.MATLABVersion }}" == "${{ env.LatestMATLABVersion }}"; diff --git a/tools/tests/+ommtest/+helper/buildFixtureCollection.m b/tools/tests/+ommtest/+helper/buildFixtureCollection.m new file mode 100644 index 000000000..daa112f8e --- /dev/null +++ b/tools/tests/+ommtest/+helper/buildFixtureCollection.m @@ -0,0 +1,56 @@ +function collection = buildFixtureCollection() +%buildFixtureCollection Canonical instance graph used by the fixture tests +% +% collection = ommtest.helper.buildFixtureCollection() returns a +% collection covering the structural features that serialization has to +% get right: a scalar string, a string list, a linked instance, an +% embedded instance, a controlled instance reference, a date, and a +% number. +% +% Every instance is given an explicit identifier so the serialized +% document is byte-for-byte reproducible. Without that, blank node +% identifiers are random and no golden file could be compared. +% +% This function is the single definition of the fixture content. The +% golden files are generated from it by +% ommtest.helper.regenerateFixtures, so refreshing fixtures after a model +% version bump is one command plus a review of the diff. +% +% Output Arguments: +% collection - An openminds.Collection holding the fixture graph. +% +% See also ommtest.helper.regenerateFixtures + + baseIRI = "https://openminds.om-i.org/instances/matlabTestFixture/"; + + contactInformation = openminds.core.ContactInformation( ... + 'id', baseIRI + "contact-001"); + contactInformation.email = "ada@example.org"; + + person = openminds.core.Person('id', baseIRI + "person-001"); + person.givenName = "Ada"; + person.familyName = "Lovelace"; + person.alternateName = ["A. Lovelace", "Ada L."]; + person.contactInformation = contactInformation; + + quantitativeValue = openminds.core.QuantitativeValue(); + quantitativeValue.value = 42; + quantitativeValue.unit = ommtest.helper.controlledInstance( ... + "openminds.controlledterms.UnitOfMeasurement", "day"); + + specimenAge = openminds.core.SpecimenAge(); + specimenAge.age = quantitativeValue; + specimenAge.reference = ommtest.helper.controlledInstance( ... + "openminds.controlledterms.AgeReference", "birth"); + + subjectState = openminds.core.SubjectState('id', baseIRI + "subjectState-001"); + subjectState.age = specimenAge; + + subject = openminds.core.Subject('id', baseIRI + "subject-001"); + subject.lookupLabel = "fixtureSubject"; + subject.species = ommtest.helper.controlledInstance( ... + "openminds.controlledterms.Species", "Homo sapiens"); + subject.studiedState = subjectState; + + collection = openminds.Collection(person, subject); +end diff --git a/tools/tests/+ommtest/+helper/controlledInstance.m b/tools/tests/+ommtest/+helper/controlledInstance.m new file mode 100644 index 000000000..9aede1887 --- /dev/null +++ b/tools/tests/+ommtest/+helper/controlledInstance.m @@ -0,0 +1,36 @@ +function instance = controlledInstance(className, instanceName) +%controlledInstance Create a controlled instance by name, whichever API it uses +% +% instance = ommtest.helper.controlledInstance(className, instanceName) +% returns the named controlled instance of the given openMINDS type. +% +% openMINDS exposes two disjoint mechanisms for controlled instances and +% there is no common entry point. Subclasses of +% openminds.abstract.ControlledTerm take the instance name directly in +% their constructor, while types using the +% openminds.internal.mixin.HasControlledInstance mixin require the static +% fromName method and reject a string constructor argument. This function +% dispatches on the superclass so callers do not have to know which +% mechanism a given type uses. +% +% Input Arguments: +% className - Full MATLAB class name of the controlled type. +% instanceName - Name of the controlled instance, e.g. "Homo sapiens". +% +% Output Arguments: +% instance - The requested controlled instance. + + arguments + className (1,1) string + instanceName (1,1) string + end + + usesMixin = any(ismember(superclasses(className), ... + {'openminds.internal.mixin.HasControlledInstance'})); + + if usesMixin + instance = feval(sprintf("%s.fromName", className), instanceName); + else + instance = feval(className, instanceName); + end +end diff --git a/tools/tests/+ommtest/+helper/fixtureNamespaceTag.m b/tools/tests/+ommtest/+helper/fixtureNamespaceTag.m new file mode 100644 index 000000000..ea1de24e6 --- /dev/null +++ b/tools/tests/+ommtest/+helper/fixtureNamespaceTag.m @@ -0,0 +1,21 @@ +function tag = fixtureNamespaceTag() +%fixtureNamespaceTag Short tag naming the active openMINDS namespace +% +% tag = ommtest.helper.fixtureNamespaceTag() returns "omi" when the +% active model uses the https://openminds.om-i.org namespace and +% "ebrains" when it uses https://openminds.ebrains.eu. +% +% Fixtures are named by namespace rather than by version number because +% the namespace is what actually appears in the serialized document. + + baseIRI = openminds.constant.BaseURI(); + + if startsWith(baseIRI, "https://openminds.om-i.org") + tag = "omi"; + elseif startsWith(baseIRI, "https://openminds.ebrains.eu") + tag = "ebrains"; + else + error('ommtest:fixtureNamespaceTag:UnknownNamespace', ... + 'No fixture tag defined for base IRI "%s".', baseIRI) + end +end diff --git a/tools/tests/+ommtest/+helper/fixturePath.m b/tools/tests/+ommtest/+helper/fixturePath.m new file mode 100644 index 000000000..2b5d3ff93 --- /dev/null +++ b/tools/tests/+ommtest/+helper/fixturePath.m @@ -0,0 +1,13 @@ +function folderPath = fixturePath() +%fixturePath Folder holding the golden JSON-LD fixtures +% +% folderPath = ommtest.helper.fixturePath() returns the absolute path of +% the fixtures folder, resolved relative to this file so it does not +% depend on the current working folder. + + thisFile = mfilename('fullpath'); + helperFolder = fileparts(thisFile); % +helper + packageFolder = fileparts(helperFolder); % +ommtest + testsFolder = fileparts(packageFolder); % tools/tests + folderPath = string(fullfile(testsFolder, 'fixtures')); +end diff --git a/tools/tests/+ommtest/+helper/knownRoundTripGap.m b/tools/tests/+ommtest/+helper/knownRoundTripGap.m new file mode 100644 index 000000000..6040168ef --- /dev/null +++ b/tools/tests/+ommtest/+helper/knownRoundTripGap.m @@ -0,0 +1,64 @@ +function reason = knownRoundTripGap(typeName) +%knownRoundTripGap Reason a type is not expected to survive a JSON-LD round trip +% +% reason = ommtest.helper.knownRoundTripGap(typeName) returns a string +% explaining why the given openMINDS type currently fails to round trip +% through JSON-LD, or an empty string if the type is expected to succeed. +% +% Every entry here is a defect in the library, not in the test. This list +% is expected to shrink. When a fix lands, the corresponding entry must +% be removed so the round-trip test starts guarding the fixed behaviour. +% +% Input Arguments: +% typeName - Short name of an openMINDS type, e.g. "Person". +% +% Output Arguments: +% reason - Explanation of the gap, or "" if the type should round trip. +% +% See also ommtest.helper.synthesizeInstance + + arguments + typeName (1,1) string + end + + reason = ""; + + if isControlledTermType(typeName) + reason = "Controlled terms defined by the user lose every property " + ... + "on reload. ControlledTermBase/initializeControlledTerm discards " + ... + "the decoded struct and passes only the identifier to " + ... + "deserializeFromName, which finds no matching controlled instance " + ... + "and returns an empty object."; + return + end + + if ismember(typeName, residualGapTypes()) + reason = "Multi-valued properties linking to controlled instances " + ... + "lose all but the first entry on reload."; + end +end + +function tf = isControlledTermType(typeName) + className = openminds.enum.Types(typeName).ClassName; + tf = any(ismember(superclasses(className), {'openminds.abstract.ControlledTerm'})); +end + +function typeNames = residualGapTypes() +% Types that fail for reasons other than the controlled term defect. +% +% Unlike the controlled term case there is no clean structural predicate +% for these, so they are listed explicitly. Determined by sweeping every +% type through save and load; see the round-trip test for the procedure. + + typeNames = [ ... + "Accessibility", "AtlasAnnotation", "ChemicalSubstance", ... + "ContentType", "CustomAnnotation", "DataAnalysis", "DataCopy", ... + "DatasetVersion", "Dependency", "File", "FileBundle", ... + "FilePathPattern", "GenericComputation", "LocalFile", ... + "ModelValidation", "Optimization", "ParcellationTerminologyVersion", ... + "QuantitativeRelationAssessment", "Setup", "Simulation", ... + "SoftwareVersion", "SubjectGroup", "SubjectGroupState", ... + "SubjectState", "TissueSample", "TissueSampleCollection", ... + "TissueSampleCollectionState", "TissueSampleState", ... + "ValidationTest", "Visualization"]; +end diff --git a/tools/tests/+ommtest/+helper/regenerateFixtures.m b/tools/tests/+ommtest/+helper/regenerateFixtures.m new file mode 100644 index 000000000..41ee27bc6 --- /dev/null +++ b/tools/tests/+ommtest/+helper/regenerateFixtures.m @@ -0,0 +1,42 @@ +function outputPath = regenerateFixtures(options) +%regenerateFixtures Write the golden JSON-LD fixture for the active model +% +% ommtest.helper.regenerateFixtures() writes the golden fixture file for +% the currently active openMINDS model version, from the graph defined by +% ommtest.helper.buildFixtureCollection. +% +% Run this after a model version bump, then review the diff. A change in +% the golden file is a change in the serialized output of the library and +% should be understood before it is committed. +% +% Name-Value Arguments: +% FixtureFolder - Folder to write to. Defaults to the fixtures folder +% next to the tests. +% +% Output Arguments: +% outputPath - Path of the file that was written. +% +% See also ommtest.helper.buildFixtureCollection, ommtest.helper.fixturePath + + arguments + options.FixtureFolder (1,1) string = ommtest.helper.fixturePath() + end + + if ~isfolder(options.FixtureFolder) + mkdir(options.FixtureFolder) + end + + collection = ommtest.helper.buildFixtureCollection(); + outputPath = fullfile(options.FixtureFolder, currentFixtureName()); + collection.save(outputPath); + + fprintf('Wrote fixture: %s\n', outputPath); + + if ~nargout + clear outputPath + end +end + +function fileName = currentFixtureName() + fileName = "collection_" + ommtest.helper.fixtureNamespaceTag() + ".jsonld"; +end diff --git a/tools/tests/+ommtest/+helper/roundTripTypeSelection.m b/tools/tests/+ommtest/+helper/roundTripTypeSelection.m new file mode 100644 index 000000000..46fea9856 --- /dev/null +++ b/tools/tests/+ommtest/+helper/roundTripTypeSelection.m @@ -0,0 +1,37 @@ +function typeNames = roundTripTypeSelection() +%roundTripTypeSelection Types to exercise in the JSON-LD round-trip test +% +% typeNames = ommtest.helper.roundTripTypeSelection() returns a cell +% array of openMINDS type names to run the round-trip test against. +% +% Round tripping every type takes several minutes, which is too slow for +% a per-commit test run. By default this returns an evenly spaced sample +% of the model, which gives fast regression signal on every commit. Set +% the environment variable OPENMINDS_TEST_ALL_TYPES to "1" to return +% every type instead, for scheduled runs and for the schema rebuild +% pipeline. +% +% The sample is a fixed stride through the type list rather than a +% curated set, so it needs no maintenance as the model changes and it +% still spans the breadth of the model. +% +% Output Arguments: +% typeNames - Cell array of type names, for use as a TestParameter. + + allTypeNames = string(cellstr(enumeration('openminds.enum.Types'))); + allTypeNames(allTypeNames == "None") = []; + allTypeNames = sort(allTypeNames); + + if isFullSweepRequested() + typeNames = cellstr(allTypeNames); + return + end + + numSampled = 30; + stride = max(1, floor(numel(allTypeNames) / numSampled)); + typeNames = cellstr(allTypeNames(1:stride:end)); +end + +function tf = isFullSweepRequested() + tf = strcmp(getenv('OPENMINDS_TEST_ALL_TYPES'), '1'); +end diff --git a/tools/tests/+ommtest/+helper/synthesizeInstance.m b/tools/tests/+ommtest/+helper/synthesizeInstance.m new file mode 100644 index 000000000..3b1cf5dbd --- /dev/null +++ b/tools/tests/+ommtest/+helper/synthesizeInstance.m @@ -0,0 +1,392 @@ +function [instance, report] = synthesizeInstance(className, options) +%synthesizeInstance Create an openMINDS instance populated with test values +% +% instance = ommtest.helper.synthesizeInstance(className) creates an +% instance of the given openMINDS type and populates every public +% property with a deterministic, schema-valid value. Linked and embedded +% properties are populated with synthesized instances of an allowed type. +% +% [instance, report] = ommtest.helper.synthesizeInstance(className) +% additionally returns a report struct describing which properties were +% populated and which were skipped, with the reason for each skip. +% +% instance = ommtest.helper.synthesizeInstance(className, Name=Value) +% specifies additional options. +% +% Input Arguments: +% className - Full MATLAB class name of an openMINDS type, e.g. +% "openminds.core.actors.Person". +% +% Name-Value Arguments: +% LinkDepth - Number of levels of linked and embedded instances to +% populate. At depth 0 those properties are left empty. +% (Default: 1) +% +% Output Arguments: +% instance - A populated instance of the requested type. +% report - Struct with fields Populated and Skipped. Skipped is a +% struct array with fields Property, Reason and Category. +% Category is one of "PatternConstrained", "NoCandidate", +% "ValidatorRejected" or "LinkDepth". +% +% Values are chosen by property *kind* (string, datetime, numeric, +% linked, embedded) rather than by property name, so this function does +% not need updating when the openMINDS model adds, moves, or removes +% types and properties. Cardinality and numeric bounds are read from the +% property validators. A property whose validators reject every candidate +% value is skipped and recorded in the report rather than raising, so a +% newly introduced validator degrades coverage instead of breaking the +% test suite. +% +% Known gap: properties validated against a regular expression, such as +% the identifier of a DOI, are skipped. Generating a string to match an +% arbitrary pattern is out of scope; string round-tripping is covered by +% the many unconstrained string properties. +% +% See also openminds.internal.meta.fromClassName + + arguments + className (1,1) string + options.LinkDepth (1,1) double {mustBeNonnegative, mustBeInteger} = 1 + end + + instance = feval(className); + report = struct("Populated", string.empty, "Skipped", emptySkipStruct()); + report = populateProperties(instance, className, options.LinkDepth, report); +end + +function report = populateProperties(instance, className, linkDepth, report) +% Populate every public property of instance with a synthesized value. + + metaType = openminds.internal.meta.fromClassName(char(className)); + + for propertyName = metaType.PropertyNames + isLinked = metaType.isPropertyWithLinkedType(propertyName); + isEmbedded = metaType.isPropertyWithEmbeddedType(propertyName); + + if (isLinked || isEmbedded) && linkDepth == 0 + report = addSkip(report, propertyName, "Link depth exhausted", "LinkDepth"); + continue + end + + metaProperty = getMetaProperty(className, propertyName); + validatorText = getValidatorText(metaProperty); + numItems = requiredItemCount(metaType, propertyName, validatorText); + + if isLinked + allowedTypes = string(metaType.listLinkedTypesForProperty(propertyName)); + candidates = synthesizeInstanceValues(allowedTypes, linkDepth, numItems); + elseif isEmbedded + allowedTypes = string(metaType.listEmbeddedTypesForProperty(propertyName)); + candidates = synthesizeInstanceValues(allowedTypes, linkDepth, numItems); + else + candidates = synthesizePrimitiveValues(metaProperty, validatorText, propertyName, numItems); + end + + [wasAssigned, reason] = tryAssign(instance, propertyName, candidates); + if wasAssigned + report.Populated(end+1) = propertyName; + else + report = addSkip(report, propertyName, reason, ... + skipCategory(validatorText, candidates)); + end + end +end + +function values = synthesizePrimitiveValues(metaProperty, validatorText, propertyName, numItems) +% Return an ordered list of candidate values for a non-linked property. +% +% Each list candidate is followed by a scalar fallback, so a cardinality +% constraint this function did not anticipate degrades to a populated +% scalar rather than a skipped property. + + valueClass = ""; + if ~isempty(metaProperty.Validation) && ~isempty(metaProperty.Validation.Class) + valueClass = string(metaProperty.Validation.Class.Name); + end + + switch valueClass + case "string" + % Distinct values, because list properties commonly require + % unique items. + values = {@() propertyName + "_" + string(1:numItems), ... + @() propertyName + "_1"}; + + case "datetime" + if any(contains(validatorText, "mustBeValidTime")) + values = {@() repmat(datetime(-1, 1, 1, 12, 30, 0), 1, numItems), ... + @() datetime(-1, 1, 1, 12, 30, 0)}; + else + % A date-only datetime satisfies mustBeValidDate; offset + % each item so unique-item validators are satisfied too. + values = {@() datetime(2024, 1, 1) + caldays(0:numItems-1), ... + @() datetime(2024, 1, 1)}; + end + + case {"int64", "int32", "double", "single"} + firstValue = numericStartValue(validatorText); + values = {@() cast(firstValue:firstValue+numItems-1, valueClass), ... + @() cast(firstValue, valueClass)}; + + case "logical" + values = {@() true(1, numItems), @() true}; + + otherwise + % A property may be typed as an openMINDS type without being + % registered in LINKED_PROPERTIES or EMBEDDED_PROPERTIES, so + % fall back to synthesizing an instance of the declared class. + if valueClass ~= "" && isOpenMindsType(valueClass) + values = {@() synthesizeInstanceArray(valueClass, 1, numItems), ... + @() synthesizeInstanceArray(valueClass, 1, 1)}; + else + values = {}; + end + end +end + +function tf = isOpenMindsType(className) + metaClass = meta.class.fromName(className); + tf = ~isempty(metaClass) && ~metaClass.Abstract ... + && any(ismember(superclasses(className), {'openminds.abstract.Schema'})); +end + +function values = synthesizeInstanceValues(allowedTypes, linkDepth, numItems) +% Build candidate instance arrays from the first few allowed types. +% +% Trying more than one allowed type matters because the first candidate +% may be abstract or may itself fail to synthesize. Each candidate is an +% array of numItems instances, followed by a scalar fallback. + + values = {}; + numCandidateTypes = min(3, numel(allowedTypes)); + + for i = 1:numCandidateTypes + thisType = allowedTypes(i); + + metaClass = meta.class.fromName(thisType); + if isempty(metaClass) || metaClass.Abstract + continue + end + + values{end+1} = @() synthesizeInstanceArray(thisType, linkDepth, numItems); %#ok + if numItems > 1 + values{end+1} = @() synthesizeInstanceArray(thisType, linkDepth, 1); %#ok + end + end +end + +function instances = synthesizeInstanceArray(className, linkDepth, numItems) +% Create an array of distinct instances of the given type. + + if isControlledTerm(className) + instanceNames = cachedInstanceNames(className); + instances = arrayfun( ... + @(i) controlledTermInstance(className, instanceNames, i), ... + 1:numItems, "UniformOutput", false); + else + instances = arrayfun( ... + @(~) ommtest.helper.synthesizeInstance(className, "LinkDepth", linkDepth-1), ... + 1:numItems, "UniformOutput", false); + end + + instances = [instances{:}]; +end + +function instance = controlledTermInstance(className, instanceNames, index) +% Create a controlled term instance from one of its controlled instances. + + if isempty(instanceNames) + instance = feval(className); + return + end + + % Pick distinct names where possible, so unique-item validators pass. + name = instanceNames(min(index, numel(instanceNames))); + instance = ommtest.helper.controlledInstance(className, name); +end + +function instanceNames = cachedInstanceNames(className) +% Controlled instance names for a type, cached for the MATLAB session. +% +% listInstances reads the controlled instance library, which is far too +% expensive to repeat for every synthesized instance. + + persistent nameCache + if isempty(nameCache) + nameCache = dictionary(string.empty, cell.empty); + end + + if ~isKey(nameCache, className) + nameCache(className) = {feval(sprintf("%s.listInstances", className))}; + end + + instanceNames = nameCache{className}; +end + +function tf = isControlledTerm(className) + superclassNames = superclasses(className); + tf = any(ismember(superclassNames, ... + {'openminds.internal.mixin.HasControlledInstance', ... + 'openminds.abstract.ControlledTerm'})); +end + +function [wasAssigned, reason] = tryAssign(instance, propertyName, candidates) +% Assign the first candidate value the property's validators accept. +% +% Candidates are thunks rather than values, so only the candidates +% actually needed are constructed. Building every candidate up front +% would synthesize whole instance trees that are then discarded. + + wasAssigned = false; + reason = "No candidate value for this property type"; + + for i = 1:numel(candidates) + try + instance.(propertyName) = candidates{i}(); + wasAssigned = true; + reason = ""; + return + catch ME + reason = string(ME.message); + end + end +end + +function numItems = requiredItemCount(metaType, propertyName, validatorText) +% Determine how many items to synthesize for a property. +% +% Scalar properties always get one item. List properties get two, so +% that array handling is exercised, unless a min or max length validator +% requires otherwise. + + % Inspect the validators directly rather than relying on + % metaType.isPropertyValueScalar, which only consults + % mustBeScalarOrEmpty for linked and embedded properties and reports + % unrestricted-size primitive properties as non-scalar. + isScalarProperty = any(contains(validatorText, "mustBeScalarOrEmpty")) ... + || metaType.isPropertyValueScalar(propertyName); + + if isScalarProperty + numItems = 1; + return + end + + numItems = 2; + + minLength = extractValidatorBound(validatorText, "mustBeMinLength"); + if ~isnan(minLength) + numItems = max(numItems, minLength); + end + + maxLength = extractValidatorBound(validatorText, "mustBeMaxLength"); + if ~isnan(maxLength) + numItems = min(numItems, maxLength); + end + + numItems = max(numItems, 1); +end + +function startValue = numericStartValue(validatorText) +% First value satisfying the numeric range validators of a property. + + lowerBound = 1; + + inclusiveLower = extractValidatorBound(validatorText, "mustBeGreaterThanOrEqual"); + if ~isnan(inclusiveLower) + lowerBound = max(lowerBound, inclusiveLower); + end + + exclusiveLower = extractValidatorBound(validatorText, "mustBeGreaterThan"); + if ~isnan(exclusiveLower) + lowerBound = max(lowerBound, exclusiveLower + 1); + end + + upperBound = Inf; + + inclusiveUpper = extractValidatorBound(validatorText, "mustBeLessThanOrEqual"); + if ~isnan(inclusiveUpper) + upperBound = min(upperBound, inclusiveUpper); + end + + exclusiveUpper = extractValidatorBound(validatorText, "mustBeLessThan"); + if ~isnan(exclusiveUpper) + upperBound = min(upperBound, exclusiveUpper - eps(exclusiveUpper)); + end + + startValue = lowerBound; + + % A default lower bound of 1 can exceed a tight upper bound, as for a + % scale factor constrained to be less than 1. Fall back to the middle + % of the permitted range. + if startValue > upperBound + startValue = upperBound / 2; + end +end + +function bound = extractValidatorBound(validatorText, validatorName) +% Extract the numeric bound from a validator such as mustBeMinLength(x,2). +% +% mustBeGreaterThan is a prefix of mustBeGreaterThanOrEqual, so the +% pattern requires the argument separator immediately after the name. + + bound = NaN; + pattern = validatorName + "\([^,()]+,\s*(-?[\d.]+)\s*\)"; + + for i = 1:numel(validatorText) + token = regexp(validatorText(i), pattern, "tokens", "once"); + if ~isempty(token) + bound = str2double(token(1)); + return + end + end +end + +function validatorText = getValidatorText(metaProperty) +% Return the validator functions of a property as text, for inspection. + + validatorText = string.empty; + if isempty(metaProperty.Validation) + return + end + + validatorFunctions = metaProperty.Validation.ValidatorFunctions; + validatorText = strings(1, numel(validatorFunctions)); + for i = 1:numel(validatorFunctions) + validatorText(i) = string(func2str(validatorFunctions{i})); + end +end + +function metaProperty = getMetaProperty(className, propertyName) + metaClass = meta.class.fromName(className); + propertyNames = string({metaClass.PropertyList.Name}); + metaProperty = metaClass.PropertyList(propertyNames == propertyName); +end + +function category = skipCategory(validatorText, candidates) +% Classify why a property could not be populated. +% +% Distinguishing a property this function is not designed to satisfy +% from one that failed unexpectedly lets callers tell a genuine +% regression from a documented limitation. + + if any(contains(validatorText, "mustMatchPattern")) + % Generating a string to satisfy an arbitrary regular expression + % is out of scope for this synthesizer. + category = "PatternConstrained"; + elseif isempty(candidates) + category = "NoCandidate"; + else + category = "ValidatorRejected"; + end +end + +function report = addSkip(report, propertyName, reason, category) + report.Skipped(end+1) = struct( ... + "Property", propertyName, ... + "Reason", reason, ... + "Category", category); +end + +function S = emptySkipStruct() + S = struct("Property", {}, "Reason", {}, "Category", {}); +end diff --git a/tools/tests/fixtures/collection_ebrains_legacy.jsonld b/tools/tests/fixtures/collection_ebrains_legacy.jsonld new file mode 100644 index 000000000..3a8f83931 --- /dev/null +++ b/tools/tests/fixtures/collection_ebrains_legacy.jsonld @@ -0,0 +1,13 @@ +{ + "@context": { + "@vocab": "https://openminds.ebrains.eu/vocab/" + }, + "@graph": [ + { + "@id": "https://openminds.ebrains.eu/instances/matlabTestFixture/person-001", + "@type": "https://openminds.ebrains.eu/core/Person", + "familyName": "Lovelace", + "givenName": "Ada" + } + ] +} diff --git a/tools/tests/fixtures/collection_omi.jsonld b/tools/tests/fixtures/collection_omi.jsonld new file mode 100644 index 000000000..b710c967a --- /dev/null +++ b/tools/tests/fixtures/collection_omi.jsonld @@ -0,0 +1,92 @@ +{ + "@context": { + "@vocab": "https://openminds.om-i.org/props/" + }, + "@graph": [ + { + "@id": "https://openminds.om-i.org/instances/ageReference/birth", + "@type": "https://openminds.om-i.org/types/AgeReference", + "definition": "An age reference point defined by the complete expulsion or extraction of the developing offspring from the gestational parent, marking the end of gestation and the beginning of postnatal development.", + "name": "birth", + "synonym": [ + "start of postnatal stage", + "end of gestation" + ] + }, + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/contact-001", + "@type": "https://openminds.om-i.org/types/ContactInformation", + "email": "ada@example.org" + }, + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/person-001", + "@type": "https://openminds.om-i.org/types/Person", + "alternateName": [ + "A. Lovelace", + "Ada L." + ], + "contactInformation": [ + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/contact-001" + } + ], + "familyName": "Lovelace", + "givenName": "Ada" + }, + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/subject-001", + "@type": "https://openminds.om-i.org/types/Subject", + "lookupLabel": "fixtureSubject", + "species": [ + { + "@id": "https://openminds.om-i.org/instances/species/homoSapiens" + } + ], + "studiedState": [ + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/subjectState-001" + } + ] + }, + { + "@id": "https://openminds.om-i.org/instances/matlabTestFixture/subjectState-001", + "@type": "https://openminds.om-i.org/types/SubjectState", + "age": { + "age": { + "unit": [ + { + "@id": "https://openminds.om-i.org/instances/unitOfMeasurement/day" + } + ], + "value": 42, + "@type": "https://openminds.om-i.org/types/QuantitativeValue" + }, + "reference": [ + { + "@id": "https://openminds.om-i.org/instances/ageReference/birth" + } + ], + "@type": "https://openminds.om-i.org/types/SpecimenAge" + } + }, + { + "@id": "https://openminds.om-i.org/instances/species/homoSapiens", + "@type": "https://openminds.om-i.org/types/Species", + "definition": "The species *Homo sapiens* (humans) belongs to the family of *hominidae* (great apes).", + "name": "Homo sapiens", + "otherOntologyIdentifier": "http://uri.interlex.org/base/ilx_0105114", + "preferredCrossReference": "https://knowledge-space.org/wiki/NCBITaxon:9606#human", + "preferredOntologyIdentifier": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "synonym": [ + "homo sapien", + "human", + "man" + ] + }, + { + "@id": "https://openminds.om-i.org/instances/unitOfMeasurement/day", + "@type": "https://openminds.om-i.org/types/UnitOfMeasurement", + "name": "day" + } + ] +} diff --git a/tools/tests/unitTests/FixtureTest.m b/tools/tests/unitTests/FixtureTest.m new file mode 100644 index 000000000..eb016a324 --- /dev/null +++ b/tools/tests/unitTests/FixtureTest.m @@ -0,0 +1,129 @@ +classdef FixtureTest < matlab.unittest.TestCase +% FixtureTest - Verify openMINDS reads and writes stable JSON-LD documents +% +% These tests compare against golden JSON-LD files checked into +% tools/tests/fixtures. Unlike the round-trip test, which only checks +% that the library agrees with itself, these pin the actual document +% format, so a change in serialized output has to be reviewed rather +% than silently accepted. +% +% The fixture content is defined once in +% ommtest.helper.buildFixtureCollection and the golden files are +% generated from it by ommtest.helper.regenerateFixtures. Refreshing +% fixtures after a model version bump is one command plus a diff review. +% +% See also ommtest.helper.buildFixtureCollection, ommtest.helper.regenerateFixtures + + properties (Access = private) + TemporaryFolder (1,1) string + end + + methods (TestClassSetup) + function warmInstanceLibrary(~) + % Build the controlled instance library once, with warnings off. + % + % Loading the library emits warnings for instance folders that + % are not mapped to a type. They have no identifier, so they + % cannot be filtered selectively, and they would otherwise be + % repeated across the parameterized tests. The library is a + % session singleton, so warming it here keeps warnings enabled + % while the tests themselves run. + + warnState = warning('off', 'all'); + cleanupObj = onCleanup(@() warning(warnState)); + openminds.internal.InstanceLibrary.getSingleton(); + end + end + + methods (TestMethodSetup) + function createTemporaryFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + fixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.TemporaryFolder = string(fixture.Folder); + end + end + + methods (Test) + function testGoldenFixtureExists(testCase) + testCase.assertTrue(isfile(testCase.goldenFixturePath()), ... + sprintf(['Golden fixture is missing: %s\n', ... + 'Generate it with ommtest.helper.regenerateFixtures.'], ... + testCase.goldenFixturePath())) + end + + function testSerializedOutputMatchesGoldenFixture(testCase) + % The document the library produces today must match the document + % that was reviewed and committed. + + testCase.assumeTrue(isfile(testCase.goldenFixturePath())) + + collection = ommtest.helper.buildFixtureCollection(); + producedPath = fullfile(testCase.TemporaryFolder, "produced.jsonld"); + collection.save(producedPath); + + produced = string(fileread(producedPath)); + golden = string(fileread(testCase.goldenFixturePath())); + + testCase.verifyEqual(produced, golden, ... + ['Serialized output no longer matches the golden fixture. ', ... + 'If the change is intended, regenerate the fixture with ', ... + 'ommtest.helper.regenerateFixtures and review the diff.']) + end + + function testGoldenFixtureLoadsWithValuesIntact(testCase) + % Loading the golden document must reproduce the original values, + % including a linked instance and a controlled instance reference. + + testCase.assumeTrue(isfile(testCase.goldenFixturePath())) + + collection = openminds.Collection(testCase.goldenFixturePath()); + + person = collection.list(openminds.enum.Types("Person")); + testCase.assertNumElements(person, 1, ... + 'Expected exactly one Person in the fixture collection.') + testCase.verifyEqual(person.givenName, "Ada") + testCase.verifyEqual(person.familyName, "Lovelace") + testCase.verifyEqual(person.alternateName, ["A. Lovelace", "Ada L."]) + + contactInformation = person.contactInformation; + testCase.assertNumElements(contactInformation, 1, ... + 'The linked ContactInformation was not resolved on load.') + testCase.verifyEqual(contactInformation.email, "ada@example.org") + + subject = collection.list(openminds.enum.Types("Subject")); + testCase.assertNumElements(subject, 1, ... + 'Expected exactly one Subject in the fixture collection.') + testCase.verifyEqual(subject.lookupLabel, "fixtureSubject") + testCase.verifyEqual(string(subject.species.id), ... + "https://openminds.om-i.org/instances/species/homoSapiens") + end + + function testLegacyNamespaceDocumentIsRejectedClearly(testCase) + % A document written with the pre-v4 EBRAINS namespace cannot be + % loaded while a v4 model is active. + % + % This pins current behaviour: the failure is a clear, identified + % error rather than silent data loss. Supporting cross-namespace + % loading would be an improvement, and this test must then be + % changed to assert that the document loads. + + legacyPath = fullfile(ommtest.helper.fixturePath(), ... + "collection_ebrains_legacy.jsonld"); + testCase.assumeTrue(isfile(legacyPath)) + testCase.assumeEqual(ommtest.helper.fixtureNamespaceTag(), "omi", ... + 'This test only applies while a v4 or later model is active.') + + testCase.verifyError(@() openminds.Collection(legacyPath), ... + 'OPENMINDS_MATLAB:Types:InvalidAtType', ... + ['Loading a legacy namespace document should fail with a ', ... + 'clear error identifying the namespace mismatch.']) + end + end + + methods (Access = private) + function fixtureFilePath = goldenFixturePath(~) + fileName = "collection_" + ommtest.helper.fixtureNamespaceTag() + ".jsonld"; + fixtureFilePath = fullfile(ommtest.helper.fixturePath(), fileName); + end + end +end diff --git a/tools/tests/unitTests/RoundTripTest.m b/tools/tests/unitTests/RoundTripTest.m new file mode 100644 index 000000000..9f43a37bb --- /dev/null +++ b/tools/tests/unitTests/RoundTripTest.m @@ -0,0 +1,147 @@ +classdef RoundTripTest < matlab.unittest.TestCase +% RoundTripTest - Verify openMINDS instances survive a JSON-LD round trip +% +% Each type is populated with synthesized values, saved to JSON-LD, +% loaded back, and saved again. The two documents must be equivalent. +% +% Serialization stability, serialize(load(serialize(x))) == serialize(x), +% is used as the round-trip property. It detects data loss without +% requiring deep object comparison, and it fails loudly when a property +% is silently dropped on either leg of the trip. +% +% The type list is generated from openminds.enum.Types, so types added +% to or removed from the model are covered without editing this file. +% By default an evenly spaced sample of types runs; set the environment +% variable OPENMINDS_TEST_ALL_TYPES to "1" for the full sweep. +% +% Types with known library defects are listed in +% ommtest.helper.knownRoundTripGap and are reported as incomplete rather +% than failed, so this suite stays a usable regression gate while those +% defects are outstanding. The synthesizer is still exercised for those +% types, so a regression in the helper is caught regardless. +% +% See also ommtest.helper.synthesizeInstance, ommtest.helper.knownRoundTripGap + + properties (TestParameter) + % Automatically generate a test case for each metadata type + MetadataType = ommtest.helper.roundTripTypeSelection(); + end + + properties (Access = private) + TemporaryFolder (1,1) string + end + + methods (TestClassSetup) + function warmInstanceLibrary(~) + % Build the controlled instance library once, with warnings off. + % + % Loading the library emits warnings for instance folders that + % are not mapped to a type. They have no identifier, so they + % cannot be filtered selectively, and they would otherwise be + % repeated across the parameterized tests. The library is a + % session singleton, so warming it here keeps warnings enabled + % while the tests themselves run. + + warnState = warning('off', 'all'); + cleanupObj = onCleanup(@() warning(warnState)); + openminds.internal.InstanceLibrary.getSingleton(); + end + end + + methods (TestMethodSetup) + function createTemporaryFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + fixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.TemporaryFolder = string(fixture.Folder); + end + end + + methods (Test) + function testJsonLdRoundTrip(testCase, MetadataType) + className = string(openminds.enum.Types(MetadataType).ClassName); + [instance, report] = ommtest.helper.synthesizeInstance(className); + + % Guard the test helper itself. A synthesizer that silently + % stopped populating properties would make the round-trip + % assertion below pass against empty instances. A handful of + % types consist solely of properties constrained by a regular + % expression, which the synthesizer does not attempt, so those + % are skipped rather than treated as a regression. + if testCase.hasOnlyPatternConstrainedProperties(report) + testCase.assumeFail(sprintf( ... + ['Every property of "%s" is constrained by a regular ', ... + 'expression, which the synthesizer does not generate ', ... + 'values for.'], MetadataType)); + end + + testCase.verifyNotEmpty(report.Populated, ... + sprintf('No property of "%s" could be populated. Skipped: %s', ... + MetadataType, testCase.describeSkipped(report))); + + gapReason = ommtest.helper.knownRoundTripGap(MetadataType); + testCase.assumeEqual(gapReason, "", ... + sprintf('Known round-trip gap for "%s": %s', MetadataType, gapReason)); + + firstDocument = testCase.saveToJsonLd(instance, "first.jsonld"); + + reloaded = openminds.Collection(testCase.filePath("first.jsonld")); + secondDocument = testCase.saveCollection(reloaded, "second.jsonld"); + + testCase.verifyEqual( ... + testCase.normalizeDocument(secondDocument), ... + testCase.normalizeDocument(firstDocument), ... + sprintf(['Round trip changed the serialized document for "%s". ', ... + 'A property was dropped or altered by save or load.'], MetadataType)); + end + end + + methods (Access = private) + function jsonText = saveToJsonLd(testCase, instance, fileName) + collection = openminds.Collection(instance); + jsonText = testCase.saveCollection(collection, fileName); + end + + function jsonText = saveCollection(testCase, collection, fileName) + targetPath = testCase.filePath(fileName); + collection.save(targetPath); + testCase.assertTrue(isfile(targetPath), ... + sprintf('Saving the collection did not produce %s', targetPath)) + jsonText = string(fileread(targetPath)); + end + + function targetPath = filePath(testCase, fileName) + targetPath = fullfile(testCase.TemporaryFolder, fileName); + end + end + + methods (Static, Access = private) + function tf = hasOnlyPatternConstrainedProperties(report) + % True when nothing was populated and every skip was a pattern. + + tf = isempty(report.Populated) ... + && ~isempty(report.Skipped) ... + && all(string({report.Skipped.Category}) == "PatternConstrained"); + end + + function description = describeSkipped(report) + % Names of the properties the synthesizer could not populate. + + if isempty(report.Skipped) + description = ""; + else + description = strjoin(string({report.Skipped.Property}), ", "); + end + end + + function lines = normalizeDocument(jsonText) + % Compare document content independent of node order. + % + % Nodes in a JSON-LD @graph are unordered, and the collection + % does not guarantee a stable order across a save and load + % cycle, so compare the set of lines rather than the raw text. + + lines = sort(string(splitlines(jsonText))); + lines(strlength(strtrim(lines)) == 0) = []; + end + end +end From 5fb026ae11e22a1efd8d3519a3230ed7b38d67d5 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Mon, 31 Aug 2026 12:39:02 +0200 Subject: [PATCH 2/2] ci: only commit test badges on push, and ignore the tutorial output file The badge step force-pushed "Update code issues and tests badges" commits onto the head branch of every pull request whose badges changed. For a stack of dependent PRs that makes each child report a non-linear history against its base after every CI round, and the commits reappear as soon as they are removed. Badges are derived artifacts that belong to the default branch, so the step now runs only for push events. The basicNeuroscienceDataset tutorial writes example_metadata.jsonld into code/livescripts when the tutorial tests run. It is generated output that keeps showing up untracked, and it was accidentally swept into a commit once, so it is now ignored. Co-Authored-By: Claude Opus 5 --- .github/workflows/run_tests.yml | 7 +++++-- .gitignore | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ed121a0f7..6bd2c842a 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -81,9 +81,12 @@ jobs: doCreateBadge = "${{ matrix.MATLABVersion }}" == "${{ env.LatestMATLABVersion }}"; addpath(genpath("tools")); testToolbox("CreateBadge", doCreateBadge) - # Commit updated SVG badges for the issues and tests (if changed) + # Commit updated SVG badges for the issues and tests (if changed). + # Push events only, so badge commits land on main and nowhere else: + # committing them on a pull_request event rewrites the PR head branch + # and breaks any stack built on top of it. - name: Commit svg badges if updated - if: matrix.MATLABVersion == env.LatestMATLABVersion + if: matrix.MATLABVersion == env.LatestMATLABVersion && github.event_name == 'push' continue-on-error: true run: | git config user.name "${{ github.workflow }} by ${{ github.actor }}" diff --git a/.gitignore b/.gitignore index 974f216e9..0d1b85bcd 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,6 @@ codegen/ .MATLABDriveTag # Python cache -__pycache__ \ No newline at end of file +__pycache__ +# Generated by the tutorial livescripts during test runs +code/livescripts/example_metadata.jsonld