Online migrations of checksums

alembic/versions/c3a1f0b9d4e2_widen_checksum_column.py

@@ -0,0 +1,48 @@
+"""Widen files.checksum column for algorithm-prefixed checksums
+
+Checksums are now stored as ``<algorithm>:<hexdigest>`` (e.g. ``sha1:...``).
+Widen the column from 64 to 128 characters so the prefix fits today and leaves
+room for longer digests (e.g. ``sha256:``) in the future.
+
+The actual re-hashing of existing values is done by the online (data) migration
+``recalculate_checksums`` in :mod:`simdb.workers.migrations`, not here.
+
+Revision ID: c3a1f0b9d4e2
+Revises: 6fb9b8fbac38
+Create Date: 2026-07-17 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c3a1f0b9d4e2"
+down_revision: Union[str, Sequence[str], None] = "6fb9b8fbac38"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.batch_alter_table("files", schema=None) as batch_op:
+        batch_op.alter_column(
+            "checksum",
+            existing_type=sa.String(length=64),
+            type_=sa.String(length=128),
+            existing_nullable=True,
+        )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    with op.batch_alter_table("files", schema=None) as batch_op:
+        batch_op.alter_column(
+            "checksum",
+            existing_type=sa.String(length=128),
+            type_=sa.String(length=64),
+            existing_nullable=True,
+        )

alembic/versions/d4b2e6f1a7c3_add_online_migrations_table.py

@@ -0,0 +1,37 @@
+"""Add online_migrations tracking table
+
+Records which online (data) migrations have been applied, so the runner in
+:mod:`simdb.workers.migrations` can skip migrations that have already run.
+
+Revision ID: d4b2e6f1a7c3
+Revises: c3a1f0b9d4e2
+Create Date: 2026-07-17 00:00:01.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d4b2e6f1a7c3"
+down_revision: Union[str, Sequence[str], None] = "c3a1f0b9d4e2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    op.create_table(
+        "online_migrations",
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("applied_at", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("name"),
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_table("online_migrations")

src/simdb/checksum.py

@@ -3,12 +3,52 @@
 
 from simdb.imas.utils import SimDBUrl
 
+#: Algorithm used to generate checksums. Prepended to every stored checksum as a
+#: ``<algorithm>:<hexdigest>`` prefix so the encoding is self-describing.
+CHECKSUM_ALGORITHM = "sha1"
+
+
+def format_checksum(hexdigest: str, algorithm: str = CHECKSUM_ALGORITHM) -> str:
+    """Prefix a raw hex digest with its algorithm, e.g. ``sha1:2fd4e1c6...``.
+
+    :param hexdigest: the hex representation of the digest
+    :param algorithm: the algorithm that produced the digest
+    :return: the algorithm-prefixed checksum string
+    """
+    return f"{algorithm}:{hexdigest}"
+
+
+def is_prefixed(checksum: str) -> bool:
+    """Return whether a checksum already carries an ``<algorithm>:`` prefix."""
+    return bool(checksum) and ":" in checksum
+
+
+def strip_checksum(checksum: str) -> str:
+    """Return the bare hex digest, dropping any ``<algorithm>:`` prefix.
+
+    Legacy (pre-prefix) checksums are returned unchanged. Used to serialize
+    checksums on the wire for API versions that predate the prefix.
+    """
+    if not checksum:
+        return checksum
+    return checksum.split(":", 1)[1] if ":" in checksum else checksum
+
+
+def checksums_match(a: str, b: str) -> bool:
+    """Compare two checksums ignoring any algorithm prefix on either side.
+
+    This keeps validation working across the prefix change: a legacy bare-hex
+    checksum (e.g. from an older client) is considered equal to its prefixed
+    form (``sha1:<hex>``).
+    """
+    return strip_checksum(a) == strip_checksum(b)
+
 
 def sha1_checksum(uri: SimDBUrl) -> str:
     """Generate a SHA1 checksum from the given file.
 
     :param uri: the URI of the file to checksum
-    :return: a string containing the hex representation of the computed SHA1 checksum
+    :return: the algorithm-prefixed checksum (``sha1:<hexdigest>``)
     """
     if uri.scheme != "file":
         raise ValueError(f"invalid scheme for file checksum: {uri.scheme}")
@@ -25,4 +65,4 @@ def sha1_checksum(uri: SimDBUrl) -> str:
     with path.open("rb") as file:
         for chunk in iter(lambda: file.read(4096), b""):
             sha1.update(chunk)
-    return sha1.hexdigest()
+    return format_checksum(sha1.hexdigest())

src/simdb/cli/commands/simulation.py

@@ -9,6 +9,7 @@
 import click
 from rich.prompt import Confirm
 
+from simdb.checksum import checksums_match
 from simdb.cli.manifest import Manifest
 from simdb.cli.remote_api import RemoteAPI, RemoteError
 from simdb.config.config import Config
@@ -479,7 +480,7 @@ def simulation_validate(
             # Pass config and ids_list parameters
             current_checksum = file.generate_checksum(config, ids_list)
 
-            if current_checksum != file.checksum:
+            if not checksums_match(current_checksum, file.checksum):
                 raise ValidationError(
                     f"Checksum mismatch for file {file.uri}. "
                     f"Expected: {file.checksum}, Got: {current_checksum}"

src/simdb/cli/remote_api.py

@@ -33,6 +33,7 @@
 from requests.auth import AuthBase
 from semantic_version import Version
 
+from simdb.checksum import checksums_match
 from simdb.config import Config
 from simdb.database.models import Simulation
 from simdb.imas.utils import SimDBUrl, imas_files
@@ -1084,7 +1085,7 @@ def _pull_file(
                     )
                 print("\r", file=out_stream, end="", flush=True)
 
-        if sha1.hexdigest() != checksum:
+        if not checksums_match(sha1.hexdigest(), checksum):
             raise APIError(f"Checksum failed for file {from_path}")
 
     @versioned_method("v1.2", "v1.3")

src/simdb/database/database.py

@@ -933,9 +933,27 @@ def get_local_db(config: Config) -> Database:
             run_migrations(database.engine)
         else:
             raise e
+    # With the schema at head, apply any pending online (data) migrations. This
+    # runs on every open, but is a cheap no-op once the data is up to date.
+    _run_online_migrations(database, config)
     return database
 
 
+def _run_online_migrations(database: Database, config: Config) -> None:
+    """Apply online (data) migrations to the local database.
+
+    Recalculates any checksums still stored in the legacy (bare-hex) format.
+    Imported lazily to avoid a circular import: ``simdb.workers`` imports
+    ``tasks``, which imports ``get_db`` from this module.
+    """
+    from simdb.workers.migrations import run_online_migrations  # noqa: PLC0415
+
+    results = run_online_migrations(database, config)
+    changed = sum(results.values())
+    if changed:
+        print(f"Recalculated {changed} local checksum(s).")
+
+
 def get_db(config: Config) -> Database:
     db_type = config.get_option("database.type")
     if db_type == "postgres":

src/simdb/database/models/__init__.py

@@ -1,7 +1,15 @@
 from .base import Base
 from .file import File
 from .metadata import MetaData
+from .online_migration import OnlineMigrationHistory
 from .simulation import Simulation
 from .watcher import Watcher
 
-__all__ = ["Base", "File", "MetaData", "Simulation", "Watcher"]
+__all__ = [
+    "Base",
+    "File",
+    "MetaData",
+    "OnlineMigrationHistory",
+    "Simulation",
+    "Watcher",
+]

src/simdb/database/models/file.py

@@ -30,7 +30,7 @@ class File(Base):
     id = Column(sql_types.Integer, primary_key=True)
     uuid = Column(UUID, nullable=False, unique=True, index=True)
     uri: SimDBUrl = Column(URI(1024), nullable=True)
-    checksum = Column(sql_types.String(64), nullable=True)
+    checksum = Column(sql_types.String(128), nullable=True)
     type = Column(sql_types.Enum(DataType), nullable=True)
     datetime = Column(sql_types.DateTime, nullable=False)
 

src/simdb/database/models/online_migration.py

@@ -0,0 +1,26 @@
+from datetime import datetime
+
+from sqlalchemy import Column
+from sqlalchemy import types as sql_types
+
+from .base import Base
+
+
+class OnlineMigrationHistory(Base):
+    """Record of an applied online (data) migration.
+
+    Online migrations transform live data (see :mod:`simdb.workers.migrations`).
+    One row is written per migration once it has completed successfully, which
+    lets the runner skip migrations that have already run.
+    """
+
+    __tablename__ = "online_migrations"
+    name = Column(sql_types.String(255), primary_key=True)
+    applied_at = Column(sql_types.DateTime, nullable=False, default=datetime.now)
+
+    def __init__(self, name: str, applied_at: datetime) -> None:
+        self.name = name
+        self.applied_at = applied_at
+
+    def __str__(self) -> str:
+        return f"{self.name} (applied {self.applied_at})"

src/simdb/imas/checksum.py

@@ -1,6 +1,7 @@
 import hashlib
 from pathlib import Path
 
+from simdb.checksum import format_checksum
 from simdb.imas.utils import SimDBUrl
 
 from .utils import imas_files, list_idss, open_imas
@@ -27,4 +28,4 @@ def checksum(uri: SimDBUrl, ids_list: list) -> str:
                 continue
             for chunk in iter(lambda: file.read(4096), b""):
                 sha1.update(chunk)
-    return sha1.hexdigest()
+    return format_checksum(sha1.hexdigest())

src/simdb/imas/utils.py

@@ -332,15 +332,15 @@ def imas_files(uri: SimDBUrl) -> List[Path]:
     path = _get_path(uri)
 
     if backend == "hdf5":
-        return [p.absolute() for p in path.glob("*.h5")]
+        return [p.absolute() for p in sorted(path.glob("*.h5"), key=lambda p: p.name)]
     elif backend == "mdsplus":
         return [
             path / "ids_001.characteristics",
             path / "ids_001.datafile",
             path / "ids_001.tree",
         ]
     elif backend == "ascii":
-        return [p.absolute() for p in path.glob("*.ids")]
+        return [p.absolute() for p in sorted(path.glob("*.ids"), key=lambda p: p.name)]
     else:
         raise ValueError(f"Unknown IMAS backend {backend}")
 

src/simdb/remote/apis/files.py

@@ -1,4 +1,5 @@
 import gzip
+import re
 import uuid
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional
@@ -8,7 +9,7 @@
 from flask_restx import Namespace, Resource
 from werkzeug.datastructures import FileStorage
 
-from simdb.checksum import sha1_checksum
+from simdb.checksum import checksums_match, sha1_checksum, strip_checksum
 from simdb.cli.manifest import DataType
 from simdb.database import DatabaseError, models
 from simdb.imas.checksum import checksum as imas_checksum
@@ -28,6 +29,30 @@
 
 api = Namespace("files", path="/")
 
+#: First API version whose wire format carries the ``<algorithm>:`` checksum
+#: prefix. Older clients expect a bare hex digest and compare it exactly, so we
+#: strip the prefix from responses served under earlier versions.
+_PREFIX_WIRE_MIN_VERSION = (1, 3)
+
+
+def _request_api_version() -> tuple:
+    """Return the API version of the current request as a ``(major, minor)`` tuple.
+
+    Derived from the Flask blueprint name (e.g. ``api_v1_2`` -> ``(1, 2)``).
+    Defaults to the newest behaviour when it cannot be determined.
+    """
+    match = re.match(r"api_v(\d+)(?:_(\d+))?", request.blueprint or "")
+    if not match:
+        return _PREFIX_WIRE_MIN_VERSION
+    return (int(match.group(1)), int(match.group(2) or 0))
+
+
+def _wire_checksum(checksum: str) -> str:
+    """Serialize a stored checksum for the wire, stripping the prefix pre-v1.3."""
+    if checksum and _request_api_version() < _PREFIX_WIRE_MIN_VERSION:
+        return strip_checksum(checksum)
+    return checksum
+
 
 def _verify_file(
     sim_uuid: uuid.UUID,
@@ -50,7 +75,7 @@ def _verify_file(
         if not path.exists():
             raise ValueError(f"file {path} does not exist")
         checksum = sha1_checksum(SimDBUrl.build(scheme="file", path=path.as_posix()))
-        if sim_file.checksum != checksum:
+        if not checksums_match(sim_file.checksum, checksum):
             raise ValueError(f"checksum failed for file {sim_file!r}")
     elif sim_file.type == DataType.IMAS:
         uri = sim_file.uri
@@ -69,7 +94,7 @@ def _verify_file(
             scheme=uri.scheme, path=uri.path, query=f"path={path_value}"
         )
         checksum = imas_checksum(new_uri, ids_list or [])
-        if sim_file.checksum != checksum:
+        if not checksums_match(sim_file.checksum, checksum):
             raise ValueError(f"checksum failed for simulation {sim_file.uri}")
 
 
@@ -178,7 +203,12 @@ class FileList(Resource):
     @pydantic_validate(api)
     def get(self, user: User) -> FileDataList:
         files = current_app.db.list_files()
-        return FileDataList.model_validate([file.to_model() for file in files])
+        models_ = []
+        for file in files:
+            model = file.to_model()
+            model.checksum = _wire_checksum(model.checksum)
+            models_.append(model)
+        return FileDataList.model_validate(models_)
 
     @requires_auth()
     def post(self, user: User):
@@ -198,7 +228,11 @@ class File(Resource):
     @pydantic_validate(api)
     def get(self, file_uuid: str, user: Optional[User] = None) -> FileGetDataResponse:
         file = current_app.db.get_file(file_uuid)
-        return file.to_model_with_path()
+        response = file.to_model_with_path()
+        response.checksum = _wire_checksum(response.checksum)
+        for file_info in response.files:
+            file_info.checksum = _wire_checksum(file_info.checksum)
+        return response
 
 
 @api.route("/file/download/<string:file_uuid>")