From 0186e0164b618e223ce18b55bd6da40bbbe31fb7 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 08:30:25 -0400 Subject: [PATCH 01/12] feat: initial saxswaxs server --- CHAP/saxswaxs/server/__init__.py | 31 + CHAP/saxswaxs/server/chap.py | 462 +++++++++++++ CHAP/saxswaxs/server/saxswaxs_to_chap.py | 820 +++++++++++++++++++++++ CHAP/saxswaxs/server/scan_to_map.py | 143 ++++ CHAP/saxswaxs/server/server.py | 196 ++++++ CHAP/saxswaxs/server/task_queue.py | 51 ++ 6 files changed, 1703 insertions(+) create mode 100644 CHAP/saxswaxs/server/__init__.py create mode 100644 CHAP/saxswaxs/server/chap.py create mode 100755 CHAP/saxswaxs/server/saxswaxs_to_chap.py create mode 100644 CHAP/saxswaxs/server/scan_to_map.py create mode 100644 CHAP/saxswaxs/server/server.py create mode 100644 CHAP/saxswaxs/server/task_queue.py diff --git a/CHAP/saxswaxs/server/__init__.py b/CHAP/saxswaxs/server/__init__.py new file mode 100644 index 0000000..7540a6c --- /dev/null +++ b/CHAP/saxswaxs/server/__init__.py @@ -0,0 +1,31 @@ +"""Daemon-like application for efficient automated SAXS/WAXS data +processing.""" + +def get_logger(name=__name__, log_level="DEBUG"): + """Create and return a :class:`logging.Logger` with a stream handler. + + Configures the logger with a formatted :class:`logging.StreamHandler` + that writes to stderr. Re-assigning ``logger.handlers`` ensures no + duplicate handlers accumulate on repeated calls with the same *name*. + + :param name: Logger name, typically the calling module's ``__name__``. + :type name: str + :param log_level: Case-insensitive logging level string + (e.g. ``"DEBUG"``, ``"INFO"``, ``"WARNING"``). + :type log_level: str + :returns: Configured logger instance. + :rtype: logging.Logger + """ + import logging + + logger = logging.getLogger(name) + log_level = getattr(logging, log_level.upper()) + logger.setLevel(log_level) + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter( + '{asctime}: {name:20} (L{lineno}): {levelname}: {message}', + datefmt='%Y-%m-%d %H:%M:%S', style='{')) + logger.addHandler(log_handler) + logger.handlers = [log_handler] + logger.propagate = False + return logger diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py new file mode 100644 index 0000000..132042f --- /dev/null +++ b/CHAP/saxswaxs/server/chap.py @@ -0,0 +1,462 @@ +"""CHAP processing code using "cached" CHAP ``PipelineItem``s for +better performance.""" + +from CHAP.common.reader import YAMLReader +from CHAP.common.writer import ZarrWriter, ZarrValuesWriter +from CHAP.pipeline import PipelineData +from CHAP.saxswaxs.processor import SetupProcessor, UpdateValuesProcessor +from functools import cache +from pathlib import Path +import subprocess +from typing import Optional + +from pydantic import BaseModel, ConfigDict + +from chap_daemon import get_logger +from chap_daemon.scan_to_map import scan_to_map +from chap_daemon.saxswaxs_to_chap import ( + saxswaxs_to_chap, + make_pipeline as _make_pipeline, + convert_configs as _convert_configs, +) + +# functions whose caches will need to be cleared regularly to work +# with live data processing +from CHAP.common.models.map import get_scanparser +from chess_scanparsers.scanparsers import ( + filespec, + list_fmb_saxswaxs_detector_files, +) + +logger = get_logger('chap') + +def cache_clear(): + """Clear scan parser and file listing caches before processing new data.""" + get_scanparser.cache_clear() + filespec.cache_clear() + list_fmb_saxswaxs_detector_files.cache_clear() + + +def init(): + """No-op placeholder for module initialization.""" + pass + + +@cache +def _read_yaml(filename, schema): + """Read a YAML config file and return it wrapped in a :class:`PipelineData` object. + + Results are cached so the same file is only read once per process lifetime. + Call :func:`cache_clear` before processing new data to invalidate stale entries. + + :param filename: Path to the YAML file to read. + :type filename: str or Path + :param schema: CHAP schema string used to validate and parse the YAML contents. + :type schema: str + :returns: Parsed config wrapped in a PipelineData container. + :rtype: PipelineData + """ + return PipelineData( + name='YAMLReader', + data=YAMLReader.run( + filename=str(filename), + schema=schema, + ), + schema=schema, + ) + + +@cache +def read_configs(detectors_yaml, map_yaml, pyfai_yaml, corrections_yaml, fits_yaml): + """Read the four config YAML files required for SAXS/WAXS processing. + + Results are cached; call :func:`cache_clear` before processing a new scan. + + :param detectors_yaml: Path to the detector config YAML file. + :type detectors_yaml: Path + :param map_yaml: Path to the map config YAML file. + :type map_yaml: Path + :param pyfai_yaml: Path to the pyFAI integration processor config YAML file. + :type pyfai_yaml: Path + :param corrections_yaml: Path to the corrections config YAML file. + :type corrections_yaml: Path + :returns: List of four PipelineData objects for detector, map, pyFAI, and + corrections configs respectively. + :rtype: list[PipelineData] + """ + return [ + _read_yaml( + detectors_yaml, + 'common.models.map.DetectorConfig' + ), + _read_yaml( + map_yaml, + 'common.models.map.MapConfig' + ), + _read_yaml( + pyfai_yaml, + 'common.models.integration.PyfaiIntegrationConfig' + ), + _read_yaml( + corrections_yaml, + 'saxswaxs.models.CorrectionsConfig' + ), + _read_yaml( + fits_yaml, + 'saxswaxs.models.FitsConfig' + ), + ] + + +def setup(cfg): + """Run the CHAP setup pipeline to create the Zarr dataset structure. + + Generates map and pipeline config files from the spec scan, reads all + config files, runs :class:`SetupProcessor` to create the Zarr dataset + structure, and writes the result to disk. + + :param cfg: Configuration for the setup task. + :type cfg: SetupCfg + """ + cache_clear() + setup_configs(cfg) + logger.info('Reading') + data = read_configs( + cfg.detectors_yaml, cfg.map_yaml, cfg.pyfai_yaml, cfg.corrections_yaml, cfg.fits_yaml, + ) + logger.info('Processing') + zarr_tree = [ + PipelineData( + data=SetupProcessor.run( + data=data, + dataset_chunks=cfg.dataset_chunks, + raw_data=False, + ), + name='saxswaxs.processor.SetupProcessor.run', + ) + ] + logger.info('Writing') + ZarrWriter.run( + data=zarr_tree, + filename=str(cfg.data_zarr), + force_overwrite=True + ) + + +def update(cfg): + """Run the CHAP update pipeline to process a scan's data into the Zarr dataset. + + Reads all config files, runs :class:`UpdateValuesProcessor` for the specified + scan and index slice, and writes the resulting values into the existing Zarr + dataset. + + :param cfg: Configuration for the update task. + :type cfg: UpdateCfg + """ + cache_clear() + logger.info('Reading') + data = read_configs( + cfg.detectors_yaml, cfg.map_yaml, cfg.pyfai_yaml, cfg.corrections_yaml, cfg.fits_yaml, + ) + logger.info('Processing') + values = [ + PipelineData( + data=UpdateValuesProcessor.run( + data=data, + filename=str(cfg.data_zarr), + spec_file=cfg.spec_file, + scan_number=cfg.scan_number, + idx_slice=dict( + start=cfg.idx_slice_start, + stop=cfg.idx_slice_stop, + step=cfg.idx_slice_step, + ), + raw_data=True, + ), + name='UpdateValuesProcessor.run', + ) + ] + logger.info('Writing') + ZarrValuesWriter.run( + data=values, + filename=str(cfg.data_zarr), + resize_axis=0, + idx_slice=dict( + start=cfg.idx_slice_start, + stop=cfg.idx_slice_stop, + step=cfg.idx_slice_step, + ), + force_overwrite=True, + ) + + +def convert(cfg): + """Run the CHAP convert pipeline to convert the Zarr dataset to NeXus format. + + Launches CHAP as a subprocess with the ``convert`` pipeline defined in + ``cfg.outputdir/pipeline.yaml``. Subprocess output is written to + ``cfg.outputdir/chap_convert.log``. + + :param cfg: Configuration for the convert task. + :type cfg: ConvertCfg + """ + logger.info("CHAP convert starting") + + logname = cfg.outputdir / "chap_convert.log" + with open(logname, "w") as logfile: + process = subprocess.Popen( + [ + "CHAP", + cfg.outputdir / "pipeline.yaml", + "-p", + "convert", + ], + stdout=logfile, + stderr=subprocess.STDOUT, + ) + process.wait() + logger.info(f"CHAP convert logging to {logname}") + + +def setup_configs(cfg): + """Write map config and CHAP pipeline config YAML files for a spec scan. + + Calls :func:`scan_to_map` to generate the map config from the spec scan, + then :func:`saxswaxs_to_chap` to generate the corresponding CHAP pipeline + config files in the output directory. + + :param cfg: Configuration containing spec file, scan number, counter names, + and output file paths. + :type cfg: SetupCfg + """ + logger.info( + "scan_to_map(" + f"{cfg.spec_file}, {cfg.scan_number}, 'id3b', 'SAXSWAXS', " + f"{cfg.dwell_time_actual_counter_name}, " + f"{cfg.presample_intensity_counter_name}, " + f"{cfg.postsample_intensity_counter_name}, " + f"{cfg.map_yaml})" + ) + scan_to_map( + str(cfg.spec_file), cfg.scan_number, "id3b", "SAXSWAXS", + cfg.dwell_time_actual_counter_name, + cfg.presample_intensity_counter_name, + cfg.postsample_intensity_counter_name, + str(cfg.map_yaml), + ) + logger.info( + f"saxswaxs_to_chap({cfg.map_yaml}, {cfg.tool_yamls}, {cfg.outputdir})" + ) + saxswaxs_to_chap( + str(cfg.map_yaml), [str(t_y) for t_y in cfg.tool_yamls], str(cfg.outputdir), + detector_filename=str(cfg.detectors_yaml), + pyfai_filename=str(cfg.pyfai_yaml), + correction_filename=str(cfg.corrections_yaml), + fits_filename=str(cfg.fits_yaml), + ) + + +class SaxswaxsCfg(BaseModel): + """Base configuration shared by all SAXS/WAXS processing tasks. + + Contains paths to the spec file, all four YAML config files (detector, + map, pyFAI integration, corrections), and the output Zarr dataset. + + :ivar spec_file: Path to the SPEC file containing the scan. + :vartype spec_file: Path + :ivar scan_number: Number of the scan within the SPEC file. + :vartype scan_number: int + :ivar detectors_yaml: Path to the detector config YAML file. + :vartype detectors_yaml: Path + :ivar map_yaml: Path to the map config YAML file. + :vartype map_yaml: Path + :ivar pyfai_yaml: Path to the pyFAI integration processor config YAML file. + :vartype pyfai_yaml: Path + :ivar corrections_yaml: Path to the corrections config YAML file. + :vartype corrections_yaml: Path + :ivar data_zarr: Path to the output Zarr dataset. + :vartype data_zarr: Path + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + spec_file: Path + scan_number: int + + detectors_yaml: Path + map_yaml: Path + pyfai_yaml: Path + corrections_yaml: Path + fits_yaml: Path + + data_zarr: Path + + +class SetupCfg(SaxswaxsCfg): + """Configuration for the setup task, which creates the Zarr dataset structure. + + Extends :class:`SaxswaxsCfg` with the output directory, tool config files, + SPEC counter names for intensity and dwell time, and dataset chunk sizes. + + :ivar outputdir: Directory for output CHAP config files and the Zarr dataset. + :vartype outputdir: Path + :ivar tool_yamls: List of tool config YAML file paths. + :vartype tool_yamls: list[Path] + :ivar dwell_time_actual_counter_name: SPEC counter column name for actual dwell times. + :vartype dwell_time_actual_counter_name: str + :ivar presample_intensity_counter_name: SPEC counter column name for presample intensity. + :vartype presample_intensity_counter_name: str + :ivar postsample_intensity_counter_name: SPEC counter column name for postsample + intensity, or ``None`` if not recorded. + :vartype postsample_intensity_counter_name: str or None + :ivar dataset_chunks: Chunk sizes for the Zarr dataset dimensions. + :vartype dataset_chunks: list[int] + """ + + outputdir: Path + + tool_yamls: list[Path] + + dwell_time_actual_counter_name: str + presample_intensity_counter_name: str + postsample_intensity_counter_name: Optional[str] = None + + dataset_chunks: list[int] + + +class UpdateCfg(SaxswaxsCfg): + """Configuration for the update task, which fills data into the Zarr dataset. + + Extends :class:`SaxswaxsCfg` with an index slice identifying which rows + of the dataset to process in this update. + + :ivar idx_slice_start: Start index of the row slice to process, defaults to ``0``. + :vartype idx_slice_start: int + :ivar idx_slice_stop: Stop index of the row slice to process, defaults to ``-1``. + :vartype idx_slice_stop: int + :ivar idx_slice_step: Step of the row slice to process, defaults to ``1``. + :vartype idx_slice_step: int + """ + + idx_slice_start: int = 0 + idx_slice_stop: int = -1 + idx_slice_step: int = 1 + + +class ConvertCfg(BaseModel): + """Configuration for the convert task, which converts the Zarr dataset to NeXus. + + Only requires the output directory containing the ``pipeline.yaml`` written + by the setup task. + + :ivar outputdir: Directory containing the CHAP ``pipeline.yaml`` and Zarr dataset. + :vartype outputdir: Path + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + outputdir: Path + + +class ConfigFilesCfg(BaseModel): + """Base configuration shared by tasks that read and write CHAP config files + in a common output directory. + + :ivar outputdir: Directory to which output config files will be written, and + against which relative filenames are resolved. + :vartype outputdir: Path + :ivar detector_filename: Path to the detector config YAML file. If relative, + resolved against ``outputdir``. Defaults to ``'detector_config.yaml'``. + :vartype detector_filename: str + :ivar pyfai_filename: Path to the pyFAI integration processor config YAML + file. If relative, resolved against ``outputdir``. Defaults to + ``'pyfai_integration_processor_config.yaml'``. + :vartype pyfai_filename: str + :ivar correction_filename: Path to the corrections config YAML file. If + relative, resolved against ``outputdir``. Defaults to + ``'corrections_config.yaml'``. + :vartype correction_filename: str + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + outputdir: Path + detector_filename: str = 'detector_config.yaml' + pyfai_filename: str = 'pyfai_integration_processor_config.yaml' + correction_filename: str = 'corrections_config.yaml' + + +class MakePipelineCfg(ConfigFilesCfg): + """Configuration for the make_pipeline task, which writes a ``pipeline.yaml`` + from pre-existing config files without requiring the old workflow library. + + Extends :class:`ConfigFilesCfg` with the map, fits, and pipeline filenames. + + :ivar map_filename: Path to the map config YAML file. If relative, resolved + against ``outputdir``. Defaults to ``'map_config.yaml'``. + :vartype map_filename: str + :ivar fits_filename: Path to the fits config YAML file. If relative, resolved + against ``outputdir``. Defaults to ``'fits_config.yaml'``. + :vartype fits_filename: str + :ivar pipeline_filename: Output filename for the pipeline YAML. Defaults to + ``'pipeline.yaml'``. + :vartype pipeline_filename: str + """ + + map_filename: str = 'map_config.yaml' + fits_filename: str = 'fits_config.yaml' + pipeline_filename: str = 'pipeline.yaml' + + +class ConvertConfigsCfg(ConfigFilesCfg): + """Configuration for the convert_configs task, which writes detector, + pyFAI integration, and corrections config YAML files from old-style + saxswaxs workflow tool config files. + + Extends :class:`ConfigFilesCfg` with the list of tool YAML files to convert. + + :ivar tool_yamls: List of tool config YAML file paths to convert. + :vartype tool_yamls: list[Path] + """ + + tool_yamls: list[Path] + + +def make_pipeline(cfg): + """Run the make_pipeline task to write a ``pipeline.yaml`` from pre-existing + config files. + + Calls :func:`chap_daemon.saxswaxs_to_chap.make_pipeline` with the paths + and filenames from ``cfg``. + + :param cfg: Configuration for the make_pipeline task. + :type cfg: MakePipelineCfg + """ + _make_pipeline( + str(cfg.outputdir), + map_filename=cfg.map_filename, + detector_filename=cfg.detector_filename, + pyfai_filename=cfg.pyfai_filename, + correction_filename=cfg.correction_filename, + fits_filename=cfg.fits_filename, + pipeline_filename=cfg.pipeline_filename, + ) + + +def convert_configs(cfg): + """Run the convert_configs task to write detector, pyFAI integration, and + corrections config YAML files from old-style saxswaxs workflow tool configs. + + Calls :func:`chap_daemon.saxswaxs_to_chap.convert_configs` with the paths + and filenames from ``cfg``. + + :param cfg: Configuration for the convert_configs task. + :type cfg: ConvertConfigsCfg + """ + _convert_configs( + str(cfg.outputdir), + [str(t) for t in cfg.tool_yamls], + detector_filename=cfg.detector_filename, + pyfai_filename=cfg.pyfai_filename, + correction_filename=cfg.correction_filename, + ) diff --git a/CHAP/saxswaxs/server/saxswaxs_to_chap.py b/CHAP/saxswaxs/server/saxswaxs_to_chap.py new file mode 100755 index 0000000..b89522f --- /dev/null +++ b/CHAP/saxswaxs/server/saxswaxs_to_chap.py @@ -0,0 +1,820 @@ + +"""Script to act as a drop-in replacement for the old saxswaxsworkflow +CLI tool. +Instead of running the old workflow code, a new CHAP pipeline will be +constructed & run from the given tool & map configs. +""" + +import argparse +import logging +import os +import sys + +import yaml + + +class VerboseSafeDumper(yaml.SafeDumper): + """YAML SafeDumper subclass that writes all nodes explicitly without aliases.""" + + def ignore_aliases(self, data): + """Return True to prevent YAML from using anchor/alias references.""" + return True + + +def saxswaxs_to_chap( + map_config_file, tool_config_files, outputdir, + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml', + fits_filename='fits_config.yaml', + pipeline_filename='pipeline.yaml'): + """Build CHAP pipeline config files from old-style saxswaxs workflow configs. + + Loads the map and tool config files using the old ``workflow`` library, then + calls :func:`wf_to_chap` to write the corresponding CHAP pipeline YAML files + to ``outputdir``. + + :param map_config_file: Path to the map config YAML file. + :type map_config_file: str + :param tool_config_files: List of tool config YAML file paths. + :type tool_config_files: list[str] + :param outputdir: Directory in which to write the output CHAP config files. + :type outputdir: str + :param detector_filename: Output filename for the detector config YAML, + defaults to ``'detector_config.yaml'``. + :type detector_filename: str, optional + :param pyfai_filename: Output filename for the pyFAI integration processor + config YAML, defaults to ``'pyfai_integration_processor_config.yaml'``. + :type pyfai_filename: str, optional + :param correction_filename: Output filename for the corrections config YAML, + defaults to ``'corrections_config.yaml'``. + :type correction_filename: str, optional + :param pipeline_filename: Output filename for the CHAP pipeline config YAML, + defaults to ``'pipeline.yaml'``. + :type pipeline_filename: str, optional + """ + # Initialize old-style saxswaxs workflow configuration objects + from workflow.map import MapConfig + from workflow.basemodel import BaseModel + from workflow.workflow import Workflow + + logger = logging.getLogger(__name__) + map_config = MapConfig.construct_from_file( + map_config_file, + logger=logger, validate_data_present=False) + tools = [ + BaseModel.construct_from_file( + tool_config_file, logger=logger) + for tool_config_file in tool_config_files + ] + wf = Workflow(map_config=map_config, tools=tools, + validate_data_present=False) + + # Compose chap pipeline config form old-style saxswaxs workflow + chap_config = wf_to_chap( + wf, outputdir, + map_filename=map_config_file, + detector_filename=detector_filename, + pyfai_filename=pyfai_filename, + correction_filename=correction_filename, + fits_filename=fits_filename, + pipeline_filename=pipeline_filename, + ) + + +def make_pipeline(outputdir, + map_filename='map_config.yaml', + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml', + fits_filename='fits_config.yaml', + pipeline_filename='pipeline.yaml'): + """Compose a pipeline file for a complete saxswaxs workflow based + on the config files provided, and asssuming they all already + exist. Sort of a lightweight version of wf_to_chap.""" + from CHAP.common.reader import YAMLReader + + outputdir = os.path.abspath(outputdir) + + if not os.path.isabs(map_filename): + map_filename = os.path.join(outputdir, map_filename) + if not os.path.isabs(detector_filename): + detector_filename = os.path.join(outputdir, detector_filename) + if not os.path.isabs(pyfai_filename): + pyfai_filename = os.path.join(outputdir, pyfai_filename) + if not os.path.isabs(correction_filename): + correction_filename = os.path.join(outputdir, correction_filename) + if not os.path.isabs(fits_filename): + fits_filename = os.path.join(outputdir, fits_filename) + + map_config = YAMLReader.run( + filename=map_filename, schema='common.models.map.MapConfig') + zarr_filename = f'{map_config.title}.zarr' + nxs_filename = f'{map_config.title}.nxs' + + readers = [ + { + 'common.reader.YAMLReader': { + 'filename': detector_filename, + 'schema': 'common.models.map.DetectorConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': map_filename, + 'schema': 'common.models.map.MapConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': pyfai_filename, + 'schema': 'common.models.integration.PyfaiIntegrationConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': correction_filename, + 'schema': 'saxswaxs.models.CorrectionsConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': fits_filename, + 'schema': 'saxswaxs.models.FitsConfig' + } + }, + ] + + update_pipelines = {} + npts = 0 + nrows = 0 + row_npts = 1 + for scans in map_config.spec_scans: + for scan_number in scans.scan_numbers: + sp = scans.get_scanparser(scan_number) + _npts = int(sp.spec_scan_npts) + if len(sp.spec_scan_shape) > 1: + _nrows = sp.spec_scan_shape[1] + row_npts = sp.spec_scan_shape[0] + else: + _nrows = 1 + row_npts = _npts + for i in range(_nrows): + idx_slice = { + 'start': npts + (i * row_npts), + 'stop': npts + (i * row_npts) + row_npts, + 'step': 1, + } + update_pipelines[f'update_{nrows + i}'] = [ + *readers, + { + 'saxswaxs.processor.UpdateValuesProcessor': { + 'raw_data': False, + 'filename': zarr_filename, + 'spec_file': scans.spec_file, + 'scan_number': scan_number, + 'idx_slice': idx_slice, + } + }, + { + 'common.ZarrValuesWriter': { + 'filename': zarr_filename, + 'resize_axis': 0, + 'idx_slice': idx_slice, + 'force_overwrite': True, + } + } + ] + nrows += _nrows + npts += _npts + + chap_config = { + 'config': { + 'root': outputdir, + 'log_level': 'debug', + }, + 'setup': [ + *readers, + { + 'saxswaxs.processor.SetupProcessor': { + 'raw_data': False, + 'dataset_chunks': [row_npts], + } + }, + { + 'common.writer.ZarrWriter': { + 'filename': zarr_filename, + 'force_overwrite': True, + } + } + ], + **update_pipelines, + 'convert': [ + { + 'common.processor.ZarrToNexusProcessor': { + 'zarr_filename': zarr_filename, + 'nexus_filename': nxs_filename, + } + } + ] + } + pipeline_path = os.path.join(outputdir, pipeline_filename) + print(f'Writing to {pipeline_path}') + os.makedirs(os.path.dirname(pipeline_path), exist_ok=True) + with open(pipeline_path, 'w') as outf: + yaml.dump(chap_config, outf, sort_keys=False, Dumper=VerboseSafeDumper) + return chap_config + + +def convert_configs(outputdir, tool_config_files, + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml'): + """Write the new CHAP.saxswaxs-formatted pyfai and corrections + config files based on the old workflow tool files provided. Should + be independent from any map configuration or + saxswaxsworkflow.workflow.Workflow object -- use the dictionaries + in the tool_config_files only.""" + + outputdir = os.path.abspath(outputdir) + + pyfai_integration_processor_config = { + 'azimuthal_integrators': [], + 'integrations': [], + } + detectors = [] + corrections = [] + for tool_config_file in tool_config_files: + with open(tool_config_file) as f: + t = yaml.safe_load(f) + if t.get('tool_type') == 'integration': + # Build azimuthal_integrators entries + for det in t.get('detectors', []): + prefix = det['prefix'] + already_added = any( + ai['id'] == prefix + and ai['poni_file'] == str(det['poni_file']) + and ai['mask_file'] == str(det['mask_file']) + for ai in pyfai_integration_processor_config['azimuthal_integrators'] + ) + if not already_added: + pyfai_integration_processor_config['azimuthal_integrators'].append( + { + 'id': prefix, + 'poni_file': str(det['poni_file']), + 'mask_file': str(det['mask_file']), + } + ) + # Build detector_config entries + visited = any(prefix == d['id'] for d in detectors) + if not visited: + placeholder_shape = (1, 1) + if prefix == 'PIL5': + shape = [619, 487] + elif prefix in ('PIL9', 'PIL11'): + shape = [407, 487] + else: + print( + f'WARNING: unrecognized detector prefix {prefix}; ' + + f'using placeholder shape {placeholder_shape}' + ) + shape = list(placeholder_shape) + detectors.append({'id': prefix, 'shape': shape}) + + # Build integrations entry + integration_config = {'name': t['title']} + if t.get('integration_type') == 'radial': + integration_config['integration_method'] = 'integrate_radial' + integration_config['integration_params'] = { + 'ais': [det['prefix'] for det in t.get('detectors', [])], + 'npt': t['azimuthal_npt'], + 'npt_rad': t['radial_npt'], + 'radial_range': [t['radial_min'], t['radial_max']], + 'azimuth_range': [t['azimuthal_min'], t['azimuthal_max']], + 'unit': t['azimuthal_units'], + 'radial_unit': t['radial_units'], + 'method': 'bbox_csr_cython', + } + else: + integration_config['multi_geometry'] = { + 'ais': [det['prefix'] for det in t.get('detectors', [])], + 'unit': t['radial_units'], + 'radial_range': [t['radial_min'], t['radial_max']], + 'azimuth_range': [t['azimuthal_min'], t['azimuthal_max']], + } + if t.get('integration_type') == 'azimuthal': + integration_config['integration_method'] = 'integrate1d' + integration_config['integration_params'] = { + 'npt': t['radial_npt'], + 'method': 'bbox_csr_cython', + } + elif t.get('integration_type') == 'cake': + integration_config['integration_method'] = 'integrate2d' + integration_config['integration_params'] = { + 'npt_rad': t['radial_npt'], + 'npt_azim': t['azimuthal_npt'], + 'method': 'bbox_csr_cython', + } + pyfai_integration_processor_config['integrations'].append(integration_config) + else: + # It's a corrections tool — include all fields except tool_type + correction = {k: v for k, v in t.items() + if k not in ('tool_type', 'validate_data_present')} + corrections.append(correction) + + # Write detector config .yaml + detector_config = {'detectors': detectors} + if not os.path.isabs(detector_filename): + detector_filename = os.path.join(outputdir, detector_filename) + print(f'Writing to {detector_filename}') + os.makedirs(os.path.dirname(detector_filename), exist_ok=True) + with open(detector_filename, 'w') as outf: + yaml.dump(detector_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write pyfai config .yaml + if not os.path.isabs(pyfai_filename): + pyfai_filename = os.path.join(outputdir, pyfai_filename) + print(f'Writing to {pyfai_filename}') + os.makedirs(os.path.dirname(pyfai_filename), exist_ok=True) + with open(pyfai_filename, 'w') as outf: + yaml.dump(pyfai_integration_processor_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write corrections config .yaml + correction_config = {'corrections': corrections} + if not os.path.isabs(correction_filename): + correction_filename = os.path.join(outputdir, correction_filename) + print(f'Writing to {correction_filename}') + os.makedirs(os.path.dirname(correction_filename), exist_ok=True) + with open(correction_filename, 'w') as outf: + yaml.dump(correction_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + +def wf_to_chap(wf, outputdir, + map_filename='map_config.yaml', + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml', + fits_filename='fits_config.yaml', + pipeline_filename='pipeline.yaml'): + """Convert an old-style SAXSWAXS Workflow configuration into the + analogous CHAP pipeline configuration. + + Writes detector, pyFAI integration, corrections, map, and pipeline YAML + config files to ``outputdir`` and returns the pipeline config dict. + + :param wf: Workflow configuration to convert. + :type wf: workflow.Workflow + :param outputdir: Directory to which all output config .yaml files will + be written. + :type outputdir: str + :param map_filename: Filename for the map config .yaml file, defaults to + ``'map_config.yaml'``. + :type map_filename: str, optional + :param detector_filename: Filename for the detector config .yaml file, + defaults to ``'detector_config.yaml'``. + :type detector_filename: str, optional + :param pyfai_filename: Filename for the PyfaiIntegrationProcessorConfig + .yaml file, defaults to ``'pyfai_integration_processor_config.yaml'``. + :type pyfai_filename: str, optional + :param correction_filename: Filename for the corrections config .yaml file, + defaults to ``'corrections_config.yaml'``. + :type correction_filename: str, optional + :param pipeline_filename: Filename for the CHAP pipeline config .yaml file, + defaults to ``'pipeline.yaml'``. + :type pipeline_filename: str, optional + :returns: Full CHAP pipeline config dict (setup, per-row update, and convert + pipelines). + :rtype: dict + """ + outputdir = os.path.abspath(outputdir) + + # Write map configuration to file + map_config = wf.map_config + if not os.path.isabs(map_filename): + map_filename = os.path.join(outputdir, map_filename) + + # Write pyfai integration processor config to file + pyfai_integration_processor_config = { + 'azimuthal_integrators': [], + 'integrations': [], + } + detectors = [] + corrections = [] + for t in wf.tools: + if hasattr(t, 'detectors'): + # It's an integration tool. + add_integration(pyfai_integration_processor_config, t) + for det in t.detectors: + visited = any([det.prefix == d['id'] for d in detectors]) + if not visited: + # Decide on detector shape (use list not tuple for + # yaml file) + placeholder_shape = (1, 1) + if det.prefix == 'PIL5': + shape = [619, 487] + elif det.prefix in ('PIL9', 'PIL11'): + shape = [407, 487] + else: + print( + f'WARNING: unrecorgnized detector prefix {det.prefix}; ' + + f'using placeholder shape {placeholder_shape}' + ) + shape = placeholder_shape + detectors.append( + { + 'id': det.prefix, + 'shape': shape, + } + ) + else: + # It's a corrections tool + corrections.append( + t.model_dump( + mode='json', + exclude_unset=True, + exclude_defaults=True, + exclude=['validate_data_present'], + ) + ) + + # Write detector config .yaml + detector_config = {'detectors': detectors} + if not os.path.isabs(detector_filename): + detector_filename = os.path.join(outputdir, detector_filename) + print(f'Writing to {detector_filename}') + os.makedirs(os.path.dirname(detector_filename), exist_ok=True) + with open(detector_filename, 'w') as outf: + yaml.dump(detector_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write pyfai config .yaml + if not os.path.isabs(pyfai_filename): + pyfai_filename = os.path.join(outputdir, pyfai_filename) + print(f'Writing to {pyfai_filename}') + os.makedirs(os.path.dirname(pyfai_filename), exist_ok=True) + with open(pyfai_filename, 'w') as outf: + yaml.dump(pyfai_integration_processor_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write corrections config .yaml + correction_config = {'corrections': corrections} + if not os.path.isabs(correction_filename): + correction_filename = os.path.join(outputdir, correction_filename) + print(f'Writing to {correction_filename}') + os.makedirs(os.path.dirname(correction_filename), exist_ok=True) + with open(correction_filename, 'w') as outf: + yaml.dump(correction_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Iterate through each scan in the map and compose the individual + # scan-row-wise update jobs so the dataset can be processed in + # parallel by running all update pipelines at once, or + # incrementally by running each update pipeline one at a time. + update_pipelines = {} + npts = 0 + nrows = 0 + zarr_filename = f'{wf.map_config.title}.zarr' + nxs_filename = f'{wf.map_config.title}.nxs' + for scans in map_config.spec_scans: + for scan_number in scans.scan_numbers: + sp = scans.get_scanparser(scan_number) + _npts = int(sp.spec_scan_npts) + if len(sp.spec_scan_shape) > 1: + _nrows = sp.spec_scan_shape[1] + row_npts = sp.spec_scan_shape[0] + else: + _nrows = 1 + row_npts = _npts + for i in range(_nrows): + idx_slice = { + 'start': npts + (i * row_npts), + 'stop': npts + (i * row_npts) + row_npts, + 'step': 1, + } + update_pipelines[f'update_{nrows + i}'] = [ + { + 'common.reader.YAMLReader': { + 'filename': detector_filename, + 'schema': 'common.models.map.DetectorConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': map_filename, + 'schema': 'common.models.map.MapConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': pyfai_filename, + 'schema': 'common.models.integration.PyfaiIntegrationConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': correction_filename, + 'schema': 'saxswaxs.models.CorrectionsConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': fits_filename, + 'schema': 'saxswaxs.models.FitsConfig' + } + }, + { + 'saxswaxs.processor.UpdateValuesProcessor': { + 'raw_data': False, + 'filename': zarr_filename, + 'spec_file': scans.spec_file, + 'scan_number': scan_number, + 'idx_slice': idx_slice, + } + }, + { + 'common.ZarrValuesWriter': { + 'filename': zarr_filename, + 'resize_axis': 0, + 'idx_slice': idx_slice, + 'force_overwrite': True, + } + } + ] + nrows += _nrows + npts += _npts + + # Compose final CHAP pipeline config and write to file. + # _restructure_pipeline = restructure_pipeline(wf) + chap_config = { + 'config': { + 'root': outputdir, + 'log_level': 'debug', + }, + 'setup': [ + { + 'common.reader.YAMLReader': { + 'filename': detector_filename, + 'schema': 'common.models.map.DetectorConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': map_filename, + 'schema': 'common.models.map.MapConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': pyfai_filename, + 'schema': 'common.models.integration.PyfaiIntegrationConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': correction_filename, + 'schema': 'saxswaxs.models.CorrectionsConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': fits_filename, + 'schema': 'saxswaxs.models.FitsConfig' + } + }, + { + 'saxswaxs.processor.SetupProcessor': { + 'raw_data': False, + 'dataset_chunks': [row_npts], + } + }, + { + 'common.writer.ZarrWriter': { + 'filename': f'{wf.map_config.title}.zarr', + 'force_overwrite': True + } + } + ], + **update_pipelines, + 'convert': [ + { + 'common.processor.ZarrToNexusProcessor': { + 'zarr_filename': zarr_filename, + 'nexus_filename': nxs_filename, + } + } + ], + # 'struct': _restructure_pipeline, + } + pipeline_filename = os.path.join(outputdir, pipeline_filename) + print(f'Writing to {pipeline_filename}') + os.makedirs(os.path.dirname(pipeline_filename), exist_ok=True) + with open(pipeline_filename, 'w') as outf: + yaml.dump(chap_config, outf, sort_keys=False, Dumper=VerboseSafeDumper) + return chap_config + + +def restructure_pipeline(wf): + """Build a CHAP pipeline config for restructuring data from unstructured to structured form. + + Reads independent dimension axes and all tool output signals from an existing + NeXus file and passes them to + ``saxswaxs.processor.UnstructuredToStructuredProcessor``, writing a structured + dataset back to the same NeXus file. + + .. note:: This function is not currently used; see the commented-out lines in + :func:`wf_to_chap`. + + :param wf: Old-style saxswaxs workflow configuration. + :type wf: workflow.Workflow + :returns: List of CHAP pipeline step dicts for the restructure pipeline. + :rtype: list[dict] + """ + axes = [a.label for a in wf.map_config.independent_dimensions] + readers = [ + { + 'common.reader.NexusReader': { + 'filename': f'{wf.map_config.title}.nxs', + 'nxpath': f'{wf.map_config.title}/independent_dimensions/{a}', + 'nxmemory': 100000, + 'name': a + } + } + for a in axes] + fields = [ + { + 'name': a, + 'type': 'axis' + } + for a in axes] + tools = {t.title: t for t in wf.tools} + for title, tool in tools.items(): + if hasattr(tool, 'integration_type'): + # Integration tool; only signal is intensity "I" + signal = 'I' + _axes = tool.integrated_data_dims + elif hasattr(tool, 'correction_type'): + # Corrections tool; varying signals + signal = 'result' + _axes = tools[tool.uncorrected_data_title].integrated_data_dims + readers.extend( + [ + { + 'common.reader.NexusReader': { + 'filename': f'{wf.map_config.title}.nxs', + 'nxpath': f'{title}/data/{x}', + 'nxmemory': 100000, + 'name': f'{title}_{x}' + } + } + for x in [signal] + _axes] + ) + fields.extend( + [ + { + 'name': f'{title}_{signal}', + 'type': 'signal', + 'axes': axes + [f'{title}_{a}' for a in _axes] + }, + *[ + { + 'name': f'{title}_{a}', + 'type': 'axis' + } + for a in _axes + ] + ] + ) + pipeline = [ + *readers, + { + 'saxswaxs.processor.UnstructuredToStructuredProcessor': { + 'fields': fields + } + }, + { + 'common.writer.NexusWriter': { + 'filename': f'{wf.map_config.title}.nxs', + 'nxpath': '/structured_data', + 'force_overwrite': True, + } + } + ] + return pipeline + + +def add_integration(pyfai_integration_processor_config, tool_config): + """Add the necessary components from a `workflow.IntegrationTool` + to an existing config for a + `saxswaxs.PyfaiIntegrationProcessor`. + + :param pyfai_integration_processor_config: Partial config for + `saxswaxs.PyfaiIntegrationProcessor` to which the given tool's + integration will be added. + :type pyfai_integration_processor_config: dict + :param tool_config: Old-style workflow integration tool object. + :type tool_config: workflow.integration.IntegrationConfig + :returns: Updated `pyfai_integration_processor_config` + :rtype: dict + """ + def detector_in_config(det): + """Convenience function for determining if the given detector + is already in `pyfai_integration_processor_config`. + """ + for _det in pyfai_integration_processor_config['azimuthal_integrators']: + if (_det['id'] == det.prefix + and _det['poni_file'] == str(det.poni_file) + and _det['mask_file'] == str(det.mask_file)): + return True + return False + + for detector in tool_config.detectors: + if not detector_in_config(detector): + pyfai_integration_processor_config['azimuthal_integrators'].append( + { + 'id': str(detector.prefix), + 'poni_file': str(detector.poni_file), + 'mask_file': str(detector.mask_file), + } + ) + + integration_config = { + 'name': tool_config.title, + } + if tool_config.integration_type == 'radial': + integration_config['integration_method'] = 'integrate_radial' + integration_config['integration_params'] = { + 'ais': [det.prefix for det in tool_config.detectors], + 'npt': tool_config.azimuthal_npt, + 'npt_rad': tool_config.radial_npt, + 'radial_range': [tool_config.radial_min, tool_config.radial_max], + 'azimuth_range': [tool_config.azimuthal_min, tool_config.azimuthal_max], + 'unit': tool_config.azimuthal_units, + 'radial_unit': tool_config.radial_units, + 'method': 'bbox_csr_cython', + } + else: + integration_config['multi_geometry'] = { + 'ais': [det.prefix for det in tool_config.detectors], + 'unit': tool_config.radial_units, + 'radial_range': [tool_config.radial_min, tool_config.radial_max], + 'azimuth_range': [tool_config.azimuthal_min, tool_config.azimuthal_max], + } + if tool_config.integration_type == 'azimuthal': + integration_config['integration_method'] = 'integrate1d' + integration_config['integration_params'] = { + 'npt': tool_config.radial_npt, + 'method': 'bbox_csr_cython', + } + elif tool_config.integration_type == 'cake': + integration_config['integration_method'] = 'integrate2d' + integration_config['integration_params'] = { + 'npt_rad': tool_config.radial_npt, + 'npt_azim': tool_config.azimuthal_npt, + 'method': 'bbox_csr_cython', + } + pyfai_integration_processor_config['integrations'].append(integration_config) + + return pyfai_integration_processor_config + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', '--map_config_file', required=True, metavar='map.yaml', + help='''Map configuration .yaml file to use for constructing + the workflow.''' + ) + parser.add_argument( + '-t', '--tool_config_files', required=True, action='extend', + nargs='+', metavar='tool.yaml', help='''List of .yaml files + containing tool configurations to apply to the map + configuration provided.''' + ) + parser.add_argument( + '-f', '--force_overwrite', action='store_true', help='''Use + this flag to overwrite the output file if it already + exists.''' + ) + parser.add_argument( + '-i', '--inputdir', default='.', help='''Directory containing + all input files''' + ) + parser.add_argument( + '-o', '--outputdir', default='.', help='''Directory in which + to place output file.''' + ) + parser.add_argument( + '-l', '--log', choices=logging._nameToLevel.keys(), + default='INFO', help='''Specify a preferred logging level.''' + ) + args = parser.parse_args(sys.argv[1:]) + + map_config_file = os.path.join(args.inputdir, args.map_config_file) + tool_config_files = [ + os.path.join(args.inputdir, tool_config_file) + for tool_config_file in args.tool_config_files + ] + saxswaxs_to_chap(map_config_file, tool_config_files, args.outputdir) diff --git a/CHAP/saxswaxs/server/scan_to_map.py b/CHAP/saxswaxs/server/scan_to_map.py new file mode 100644 index 0000000..ee9acde --- /dev/null +++ b/CHAP/saxswaxs/server/scan_to_map.py @@ -0,0 +1,143 @@ +#!/nfs/chess/sw/miniforge3_chap/envs/CHAP_saxswaxs/bin/python +"""Script to convert a single SPEC scan into its +MapConfigRepresentation using CHAP tools +""" + +def spec_scan_to_map_config( + spec_file, scan_number, station, experiment, + dwell_time_actual_counter_name, + presample_intensity_counter_name, + postsample_intensity_counter_name): + """Convert a single SPEC scan into a CHAP map config data structure. + + :param spec_file: Path to the SPEC file containing the scan. + :type spec_file: str + :param scan_number: Number of the scan within the SPEC file. + :type scan_number: int + :param station: Station identifier (e.g. ``'id3b'``). + :type station: str + :param experiment: Experiment type (e.g. ``'SAXSWAXS'``). + :type experiment: str + :param dwell_time_actual_counter_name: SPEC counter column name for actual + dwell times. + :type dwell_time_actual_counter_name: str + :param presample_intensity_counter_name: SPEC counter column name for + presample intensity. + :type presample_intensity_counter_name: str + :param postsample_intensity_counter_name: SPEC counter column name for + postsample intensity, or ``None`` if not recorded. + :type postsample_intensity_counter_name: str or None + :returns: Map config data structure. + :rtype: dict + """ + from CHAP.common.map_utils import SpecScanToMapConfigProcessor + + proc = SpecScanToMapConfigProcessor() + map_config = proc.process( + None, + spec_file, scan_number, station, experiment, + dwell_time_actual_counter_name, + presample_intensity_counter_name, + postsample_intensity_counter_name, + validate_data_present=False, + ) + return map_config + +def write_yaml(data, filename): + """Write data to a YAML file, creating parent directories as needed. + + :param data: Data to serialize to YAML. + :param filename: Output file path. + :type filename: str + """ + import os + import yaml + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w') as outf: + yaml.dump(data, outf, sort_keys=False) + +def scan_to_map( + spec_file, scan_number, station, experiment, + dwell_time_actual_counter_name, + presample_intensity_counter_name, + postsample_intensity_counter_name, + map_config_filename): + """Convert a SPEC scan to a map config YAML file. + + :param spec_file: Path to the SPEC file containing the scan. + :type spec_file: str + :param scan_number: Number of the scan within the SPEC file. + :type scan_number: int + :param station: Station identifier (e.g. ``'id3b'``). + :type station: str + :param experiment: Experiment type (e.g. ``'SAXSWAXS'``). + :type experiment: str + :param dwell_time_actual_counter_name: SPEC counter column name for actual + dwell times. + :type dwell_time_actual_counter_name: str + :param presample_intensity_counter_name: SPEC counter column name for + presample intensity. + :type presample_intensity_counter_name: str + :param postsample_intensity_counter_name: SPEC counter column name for + postsample intensity, or ``None`` if not recorded. + :type postsample_intensity_counter_name: str or None + :param map_config_filename: Output path for the map config YAML file. + :type map_config_filename: str + """ + map_config_data = spec_scan_to_map_config( + spec_file, scan_number, station, experiment, + dwell_time_actual_counter_name, + presample_intensity_counter_name, + postsample_intensity_counter_name, + ) + write_yaml(map_config_data, map_config_filename) + + +if __name__ == '__main__': + import argparse + import os + import sys + + parser = argparse.ArgumentParser( + description='''Create a map_config.yaml representing the given + SPEC scan''') + parser.add_argument( + '--spec_file', required=True, + help='Name of the SPEC file containing the scan of interest.') + parser.add_argument( + '--scan_number', required=True, type=int, + help='Number of the scan of interest within the given SPEC file.') + parser.add_argument( + '--outputdir', required=True, help='''Path to the output + analysis directory for this scan. The map configuration YAML + and reduced data NeXus files will be written here.''') + parser.add_argument( + '--dwell_time_actual_counter_name', required=True, + help='''Name of the SPEC counter column representing actual + dwell times''') + parser.add_argument( + '--presample_intensity_counter_name', required=True, + help='''Name of the SPEC counter column representing presample + intensity values''') + parser.add_argument( + '--postsample_intensity_counter_name', + help='''Name of the SPEC counter column representing postsample + intensity values''') + parser.add_argument( + '--station', choices=['id3b'], default='id3b', + help='''Name of the station at which the scan was collected + ("id3b" is currently the only supported value)''') + parser.add_argument( + '--experiment', choices=['SAXSWAXS'], default='SAXSWAXS', + help='''Name of the scan\'s experiment type ("SAXSWAXS" is + currently the only supported value).''') + args = parser.parse_args(sys.argv[1:]) + + scan_to_map( + args.spec_file, args.scan_number, args.station, args.experiment, + args.dwell_time_actual_counter_name, + args.presample_intensity_counter_name, + args.postsample_intensity_counter_name, + os.path.join(args.outputdir, 'map_config.yaml') + ) diff --git a/CHAP/saxswaxs/server/server.py b/CHAP/saxswaxs/server/server.py new file mode 100644 index 0000000..6706489 --- /dev/null +++ b/CHAP/saxswaxs/server/server.py @@ -0,0 +1,196 @@ +"""Server for handling HTTP requests.""" + +from flask import Flask, jsonify, request +import logging +import time +from traceback import print_exc + +from chap_daemon import get_logger +from chap_daemon.task_queue import put +from chap_daemon.chap import ( + setup, update, convert, make_pipeline, convert_configs, + SetupCfg, UpdateCfg, ConvertCfg, MakePipelineCfg, ConvertConfigsCfg, +) + +app = Flask(__name__) +app.logger = get_logger('server') +app.logger.propagate = False + +# Logging middleware +@app.before_request +def start_timer(): + """Record the request start time for duration logging.""" + request.start_time = time.time() + +@app.after_request +def log_request(response): + """Log the HTTP method, path, status code, and duration for the request.""" + now = time.time() + duration = round(now - request.start_time, 4) + app.logger.info( + f"{request.method} {request.path} - {response.status_code} ({duration}s)" + ) + return response + +# API endpoints +@app.route('/setup', methods=['POST']) +def setup_handler(): + """Handle POST /setup — parse JSON body and queue a setup task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = SetupCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + setup_args = (cfg,) + setup_kwargs = {} + put(setup, setup_args, setup_kwargs) + return jsonify({'status': 'queued'}), 202 + +@app.route('/update', methods=['POST']) +def update_handler(): + """Handle POST /update — parse JSON body and queue an update task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = UpdateCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + update_args = (cfg,) + update_kwargs = {} + put(update, update_args, update_kwargs) + return jsonify({'status': 'queued'}), 202 + +@app.route('/convert', methods=['POST']) +def convert_handler(): + """Handle POST /convert — parse JSON body and queue a convert task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = ConvertCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + convert_args = (cfg,) + convert_kwargs = {} + put(convert, convert_args, convert_kwargs) + return jsonify({'status': 'queued'}), 202 + + +@app.route('/make_pipeline', methods=['POST']) +def make_pipeline_handler(): + """Handle POST /make_pipeline — parse JSON body and queue a make_pipeline task. + + Constructs a :class:`~chap_daemon.chap.MakePipelineCfg` from the request + body and queues :func:`~chap_daemon.chap.make_pipeline` to write a + ``pipeline.yaml`` from the pre-existing config files in ``outputdir``. + """ + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = MakePipelineCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'status': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + put(make_pipeline, (cfg,), {}) + return jsonify({'status': 'queued'}), 202 + + +@app.route('/convert_configs', methods=['POST']) +def convert_configs_handler(): + """Handle POST /convert_configs — parse JSON body and queue a convert_configs task. + + Constructs a :class:`~chap_daemon.chap.ConvertConfigsCfg` from the request + body and queues :func:`~chap_daemon.chap.convert_configs` to write detector, + pyFAI integration, and corrections config YAML files from the provided + tool config files. + """ + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = ConvertConfigsCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'status': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + put(convert_configs, (cfg,), {}) + return jsonify({'status': 'queued'}), 202 + + +def run(): + """Start the Flask development server.""" + app.run(debug=False) + +if __name__ == '__main__': + run() diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py new file mode 100644 index 0000000..793dcea --- /dev/null +++ b/CHAP/saxswaxs/server/task_queue.py @@ -0,0 +1,51 @@ +"""Queueing system for data processing tasks""" + +import queue +import threading +from traceback import print_exc +from time import sleep, time + +from chap_daemon import get_logger + +logger = get_logger('task_queue') + +_task_queue = queue.Queue() + +def _worker(): + """Continuously dequeue and execute tasks from the task queue. + + Runs in a background daemon thread. On task failure the error is logged + and the task is marked done. + """ + while True: + task, args, kwargs = _task_queue.get() + logger.info( + f'Starting task: {str(task)}, args: {args}, kwargs: {kwargs}') + t0 = time() + success = False + while not success: + # Handle race conditions from missing data + try: + task(*args, **kwargs) + success = True + except Exception as exc: + logger.error(f'Task failed: {exc}') + print_exc() + success = True # FIXME temporary for debugging + sleep(5) + _task_queue.task_done() + tf = time() + logger.info(f'Task done. ({tf-t0:.5f} seconds)') + +def put(task, args, kwargs): + """Enqueue a task for execution by the background worker thread. + + :param task: Callable to execute. + :param args: Positional arguments to pass to ``task``. + :type args: tuple + :param kwargs: Keyword arguments to pass to ``task``. + :type kwargs: dict + """ + _task_queue.put((task, args, kwargs)) + +threading.Thread(target=_worker, daemon=True).start() From e9685fddf9f416929cba22cb325cba2216d0534a Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 08:46:42 -0400 Subject: [PATCH 02/12] build: add saxswaxs-server installable executable --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0149670..381e8cc 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,10 @@ def datafiles(idir, pattern=None): 'examples': data_files, }, entry_points={ - 'console_scripts': ['CHAP = CHAP.runner:main'], + 'console_scripts': [ + 'CHAP = CHAP.runner:main', + 'saxswaxs-server = CHAP.saxswaxs.server.server:run', + ], }, classifiers=[ "Programming Language :: Python :: 3", From 117bf4126d147e998459f30b12b7cac088c135bc Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 09:49:25 -0400 Subject: [PATCH 03/12] test: initial CHAP.saxswaxs.server pytest facility --- CHAP/test/data/saxswaxs/1/convert.json | 1 + CHAP/test/data/saxswaxs/1/setup.json | 17 ++ CHAP/test/data/saxswaxs/1/update.json | 12 ++ CHAP/test/data/saxswaxs/2/convert.json | 1 + CHAP/test/data/saxswaxs/2/setup.json | 17 ++ CHAP/test/data/saxswaxs/2/update.json | 9 + CHAP/test/saxswaxs/server/__init__.py | 0 CHAP/test/saxswaxs/server/conftest.py | 48 +++++ CHAP/test/saxswaxs/server/test_chap.py | 206 +++++++++++++++++++ CHAP/test/saxswaxs/server/test_models.py | 105 ++++++++++ CHAP/test/saxswaxs/server/test_server.py | 133 ++++++++++++ CHAP/test/saxswaxs/server/test_task_queue.py | 73 +++++++ 12 files changed, 622 insertions(+) create mode 100644 CHAP/test/data/saxswaxs/1/convert.json create mode 100644 CHAP/test/data/saxswaxs/1/setup.json create mode 100644 CHAP/test/data/saxswaxs/1/update.json create mode 100644 CHAP/test/data/saxswaxs/2/convert.json create mode 100644 CHAP/test/data/saxswaxs/2/setup.json create mode 100644 CHAP/test/data/saxswaxs/2/update.json create mode 100644 CHAP/test/saxswaxs/server/__init__.py create mode 100644 CHAP/test/saxswaxs/server/conftest.py create mode 100644 CHAP/test/saxswaxs/server/test_chap.py create mode 100644 CHAP/test/saxswaxs/server/test_models.py create mode 100644 CHAP/test/saxswaxs/server/test_server.py create mode 100644 CHAP/test/saxswaxs/server/test_task_queue.py diff --git a/CHAP/test/data/saxswaxs/1/convert.json b/CHAP/test/data/saxswaxs/1/convert.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/convert.json @@ -0,0 +1 @@ +{} diff --git a/CHAP/test/data/saxswaxs/1/setup.json b/CHAP/test/data/saxswaxs/1/setup.json new file mode 100644 index 0000000..f9c44a5 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/setup.json @@ -0,0 +1,17 @@ +{ + "spec_file": "/data/spec/scan.spec", + "scan_number": 1, + "detectors_yaml": "/data/config/detectors.yaml", + "map_yaml": "/data/config/map.yaml", + "pyfai_yaml": "/data/config/pyfai.yaml", + "corrections_yaml": "/data/config/corrections.yaml", + "fits_yaml": "/data/config/fits.yaml", + "tool_yamls": [ + "/data/tools/tool1.yaml", + "/data/tools/tool2.yaml" + ], + "dwell_time_actual_counter_name": "dwell_counter", + "presample_intensity_counter_name": "presample_counter", + "postsample_intensity_counter_name": "postsample_counter", + "dataset_chunks": [10] +} diff --git a/CHAP/test/data/saxswaxs/1/update.json b/CHAP/test/data/saxswaxs/1/update.json new file mode 100644 index 0000000..7486066 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/update.json @@ -0,0 +1,12 @@ +{ + "spec_file": "/data/spec/scan.spec", + "scan_number": 1, + "detectors_yaml": "/data/config/detectors.yaml", + "map_yaml": "/data/config/map.yaml", + "pyfai_yaml": "/data/config/pyfai.yaml", + "corrections_yaml": "/data/config/corrections.yaml", + "fits_yaml": "/data/config/fits.yaml", + "idx_slice_start": 10, + "idx_slice_stop": 20, + "idx_slice_step": 1 +} diff --git a/CHAP/test/data/saxswaxs/2/convert.json b/CHAP/test/data/saxswaxs/2/convert.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/convert.json @@ -0,0 +1 @@ +{} diff --git a/CHAP/test/data/saxswaxs/2/setup.json b/CHAP/test/data/saxswaxs/2/setup.json new file mode 100644 index 0000000..396270e --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/setup.json @@ -0,0 +1,17 @@ +{ + "spec_file": "/experiment/spec/run2.spec", + "scan_number": 5, + "detectors_yaml": "/experiment/config/det.yaml", + "map_yaml": "/experiment/config/mapping.yaml", + "pyfai_yaml": "/experiment/config/azimuthal.yaml", + "corrections_yaml": "/experiment/config/corr.yaml", + "fits_yaml": "/experiment/config/fit_params.yaml", + "tool_yamls": [ + "/experiment/tools/a.yaml", + "/experiment/tools/b.yaml", + "/experiment/tools/c.yaml" + ], + "dwell_time_actual_counter_name": "timer", + "presample_intensity_counter_name": "mon", + "dataset_chunks": [50, 50] +} diff --git a/CHAP/test/data/saxswaxs/2/update.json b/CHAP/test/data/saxswaxs/2/update.json new file mode 100644 index 0000000..27f4667 --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/update.json @@ -0,0 +1,9 @@ +{ + "spec_file": "/experiment/spec/run2.spec", + "scan_number": 5, + "detectors_yaml": "/experiment/config/det.yaml", + "map_yaml": "/experiment/config/mapping.yaml", + "pyfai_yaml": "/experiment/config/azimuthal.yaml", + "corrections_yaml": "/experiment/config/corr.yaml", + "fits_yaml": "/experiment/config/fit_params.yaml" +} diff --git a/CHAP/test/saxswaxs/server/__init__.py b/CHAP/test/saxswaxs/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CHAP/test/saxswaxs/server/conftest.py b/CHAP/test/saxswaxs/server/conftest.py new file mode 100644 index 0000000..fa1f0bc --- /dev/null +++ b/CHAP/test/saxswaxs/server/conftest.py @@ -0,0 +1,48 @@ +"""Shared fixtures for CHAP.saxswaxs.server tests.""" + +import json +from pathlib import Path + +import pytest + +_DATA_ROOT = Path(__file__).parent.parent.parent / "data" / "saxswaxs" + +# Table of fixture directories — add a new entry here to add a new test case set. +FIXTURE_DIRS = [ + _DATA_ROOT / "1", + _DATA_ROOT / "2", +] + + +@pytest.fixture(params=FIXTURE_DIRS, ids=[d.name for d in FIXTURE_DIRS]) +def fixture_dir(request): + return request.param + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +@pytest.fixture +def setup_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import SetupCfg + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + return SetupCfg(**data) + + +@pytest.fixture +def update_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import UpdateCfg + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + return UpdateCfg(**data) + + +@pytest.fixture +def convert_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import ConvertCfg + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + return ConvertCfg(**data) diff --git a/CHAP/test/saxswaxs/server/test_chap.py b/CHAP/test/saxswaxs/server/test_chap.py new file mode 100644 index 0000000..5018781 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_chap.py @@ -0,0 +1,206 @@ +"""Unit tests for CHAP.saxswaxs.server.chap with all CHAP processors mocked.""" + +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +import pytest + +# Fixtures setup_cfg, update_cfg, convert_cfg come from conftest.py + +CHAP_MOCKS = { + "CHAP.saxswaxs.server.chap.YAMLReader": MagicMock, + "CHAP.saxswaxs.server.chap.ZarrWriter": MagicMock, + "CHAP.saxswaxs.server.chap.ZarrValuesWriter": MagicMock, + "CHAP.saxswaxs.server.chap.SetupProcessor": MagicMock, + "CHAP.saxswaxs.server.chap.UpdateValuesProcessor": MagicMock, + "CHAP.saxswaxs.server.chap.PipelineData": MagicMock, + "CHAP.saxswaxs.server.chap.setup_configs": MagicMock, + "CHAP.saxswaxs.server.chap.read_configs": MagicMock, + "CHAP.saxswaxs.server.chap.cache_clear": MagicMock, +} + + +def _patch_all(): + """Return a single patch.multiple context manager covering all CHAP seams.""" + return patch.multiple("CHAP.saxswaxs.server.chap", **{k.split(".")[-1]: MagicMock() for k in CHAP_MOCKS}) + + +class TestCacheClear: + def test_cache_clear_called_in_setup(self, setup_cfg): + with patch("CHAP.saxswaxs.server.chap.cache_clear") as mock_cc, \ + patch("CHAP.saxswaxs.server.chap.setup_configs"), \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=[MagicMock()] * 5), \ + patch("CHAP.saxswaxs.server.chap.SetupProcessor"), \ + patch("CHAP.saxswaxs.server.chap.ZarrWriter"), \ + patch("CHAP.saxswaxs.server.chap.PipelineData"): + from CHAP.saxswaxs.server.chap import setup + setup(setup_cfg) + mock_cc.assert_called_once() + + def test_cache_clear_called_in_update(self, update_cfg): + with patch("CHAP.saxswaxs.server.chap.cache_clear") as mock_cc, \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=[MagicMock()] * 5), \ + patch("CHAP.saxswaxs.server.chap.UpdateValuesProcessor"), \ + patch("CHAP.saxswaxs.server.chap.ZarrValuesWriter"), \ + patch("CHAP.saxswaxs.server.chap.PipelineData"): + from CHAP.saxswaxs.server.chap import update + update(update_cfg) + mock_cc.assert_called_once() + + +class TestSetup: + def _run(self, cfg): + mock_data = [MagicMock()] * 5 + mock_zarr_tree = MagicMock() + mock_pipeline_data = MagicMock(return_value=mock_zarr_tree) + mock_setup_proc = MagicMock() + mock_setup_proc.run = MagicMock(return_value=MagicMock()) + mock_zarr_writer = MagicMock() + + with patch("CHAP.saxswaxs.server.chap.cache_clear"), \ + patch("CHAP.saxswaxs.server.chap.setup_configs") as mock_sc, \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=mock_data) as mock_rc, \ + patch("CHAP.saxswaxs.server.chap.SetupProcessor", mock_setup_proc), \ + patch("CHAP.saxswaxs.server.chap.ZarrWriter", mock_zarr_writer), \ + patch("CHAP.saxswaxs.server.chap.PipelineData", mock_pipeline_data): + from CHAP.saxswaxs.server.chap import setup + setup(cfg) + return mock_sc, mock_rc, mock_setup_proc, mock_zarr_writer, mock_pipeline_data + + def test_setup_configs_called(self, setup_cfg): + mock_sc, *_ = self._run(setup_cfg) + mock_sc.assert_called_once_with(setup_cfg) + + def test_read_configs_called_with_correct_paths(self, setup_cfg): + _, mock_rc, *_ = self._run(setup_cfg) + mock_rc.assert_called_once_with( + setup_cfg.detectors_yaml, + setup_cfg.map_yaml, + setup_cfg.pyfai_yaml, + setup_cfg.corrections_yaml, + setup_cfg.fits_yaml, + ) + + def test_setup_processor_called_with_dataset_chunks(self, setup_cfg): + _, _, mock_setup_proc, *_ = self._run(setup_cfg) + mock_setup_proc.run.assert_called_once() + _, call_kwargs = mock_setup_proc.run.call_args + assert call_kwargs["dataset_chunks"] == setup_cfg.dataset_chunks + assert call_kwargs["raw_data"] is False + + def test_zarr_writer_called_with_correct_args(self, setup_cfg): + _, _, _, mock_zarr_writer, _ = self._run(setup_cfg) + mock_zarr_writer.run.assert_called_once() + _, call_kwargs = mock_zarr_writer.run.call_args + assert call_kwargs["filename"] == str(setup_cfg.data_zarr) + assert call_kwargs["force_overwrite"] is True + + +class TestUpdate: + def _run(self, cfg): + mock_data = [MagicMock()] * 5 + mock_update_proc = MagicMock() + mock_update_proc.run = MagicMock(return_value=MagicMock()) + mock_zarr_values_writer = MagicMock() + mock_pipeline_data = MagicMock() + + with patch("CHAP.saxswaxs.server.chap.cache_clear"), \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=mock_data), \ + patch("CHAP.saxswaxs.server.chap.UpdateValuesProcessor", mock_update_proc), \ + patch("CHAP.saxswaxs.server.chap.ZarrValuesWriter", mock_zarr_values_writer), \ + patch("CHAP.saxswaxs.server.chap.PipelineData", mock_pipeline_data): + from CHAP.saxswaxs.server.chap import update + update(cfg) + return mock_update_proc, mock_zarr_values_writer + + def test_update_processor_called_with_correct_slice(self, update_cfg): + mock_proc, _ = self._run(update_cfg) + mock_proc.run.assert_called_once() + _, call_kwargs = mock_proc.run.call_args + assert call_kwargs["idx_slice"] == { + "start": update_cfg.idx_slice_start, + "stop": update_cfg.idx_slice_stop, + "step": update_cfg.idx_slice_step, + } + + def test_update_processor_called_with_spec_and_zarr(self, update_cfg): + mock_proc, _ = self._run(update_cfg) + _, call_kwargs = mock_proc.run.call_args + assert call_kwargs["spec_file"] == update_cfg.spec_file + assert call_kwargs["scan_number"] == update_cfg.scan_number + assert call_kwargs["filename"] == str(update_cfg.data_zarr) + assert call_kwargs["raw_data"] is True + + def test_zarr_values_writer_called_with_correct_args(self, update_cfg): + _, mock_writer = self._run(update_cfg) + mock_writer.run.assert_called_once() + _, call_kwargs = mock_writer.run.call_args + assert call_kwargs["filename"] == str(update_cfg.data_zarr) + assert call_kwargs["resize_axis"] == 0 + assert call_kwargs["idx_slice"] == { + "start": update_cfg.idx_slice_start, + "stop": update_cfg.idx_slice_stop, + "step": update_cfg.idx_slice_step, + } + assert call_kwargs["force_overwrite"] is True + + +class TestConvert: + def _run(self, cfg): + mock_process = MagicMock() + mock_popen = MagicMock(return_value=mock_process) + + with patch("CHAP.saxswaxs.server.chap.subprocess.Popen", mock_popen): + from CHAP.saxswaxs.server.chap import convert + convert(cfg) + return mock_popen, mock_process + + def test_popen_called_with_chap_convert(self, convert_cfg): + mock_popen, _ = self._run(convert_cfg) + mock_popen.assert_called_once() + args, kwargs = mock_popen.call_args + cmd = args[0] + assert cmd[0] == "CHAP" + assert cmd[1] == convert_cfg.outputdir / "pipeline.yaml" + assert cmd[2] == "-p" + assert cmd[3] == "convert" + + def test_popen_stdout_is_logfile(self, convert_cfg, tmp_path): + import subprocess as _subprocess + mock_process = MagicMock() + + captured = {} + + def fake_popen(cmd, stdout, stderr): + captured["stdout"] = stdout + captured["stderr"] = stderr + return mock_process + + with patch("CHAP.saxswaxs.server.chap.subprocess.Popen", fake_popen): + from CHAP.saxswaxs.server.chap import convert + convert(convert_cfg) + + assert hasattr(captured["stdout"], "write"), "stdout should be a file object" + assert captured["stderr"] == _subprocess.STDOUT + + def test_logfile_path(self, convert_cfg): + mock_process = MagicMock() + opened_paths = [] + + original_open = open + + def fake_open(path, mode="r", *args, **kwargs): + if mode == "w": + opened_paths.append(Path(path)) + return original_open(path, mode, *args, **kwargs) + + with patch("builtins.open", fake_open), \ + patch("CHAP.saxswaxs.server.chap.subprocess.Popen", return_value=mock_process): + from CHAP.saxswaxs.server.chap import convert + convert(convert_cfg) + + assert any(p == convert_cfg.outputdir / "chap_convert.log" for p in opened_paths) + + def test_process_wait_called(self, convert_cfg): + _, mock_process = self._run(convert_cfg) + mock_process.wait.assert_called_once() diff --git a/CHAP/test/saxswaxs/server/test_models.py b/CHAP/test/saxswaxs/server/test_models.py new file mode 100644 index 0000000..b3abe1a --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_models.py @@ -0,0 +1,105 @@ +"""Unit tests for Pydantic config models in CHAP.saxswaxs.server.chap.""" + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from CHAP.saxswaxs.server.chap import ConvertCfg, SetupCfg, UpdateCfg + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +class TestSetupCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = SetupCfg(**data) + assert cfg.scan_number == data["scan_number"] + assert cfg.dwell_time_actual_counter_name == data["dwell_time_actual_counter_name"] + assert cfg.presample_intensity_counter_name == data["presample_intensity_counter_name"] + assert isinstance(cfg.spec_file, Path) + assert isinstance(cfg.detectors_yaml, Path) + assert isinstance(cfg.map_yaml, Path) + assert isinstance(cfg.pyfai_yaml, Path) + assert isinstance(cfg.corrections_yaml, Path) + assert isinstance(cfg.fits_yaml, Path) + assert isinstance(cfg.data_zarr, Path) + assert isinstance(cfg.outputdir, Path) + assert len(cfg.tool_yamls) == len(data["tool_yamls"]) + assert all(isinstance(p, Path) for p in cfg.tool_yamls) + assert cfg.dataset_chunks == data["dataset_chunks"] + + def test_missing_required_field_raises(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + del data["spec_file"] + with pytest.raises(ValidationError): + SetupCfg(**data) + + def test_postsample_intensity_optional(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + data.pop("postsample_intensity_counter_name", None) + cfg = SetupCfg(**data) + assert cfg.postsample_intensity_counter_name is None + + def test_fits_yaml_present(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = SetupCfg(**data) + assert cfg.fits_yaml is not None + assert isinstance(cfg.fits_yaml, Path) + + +class TestUpdateCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = UpdateCfg(**data) + assert cfg.scan_number == data["scan_number"] + assert isinstance(cfg.spec_file, Path) + assert isinstance(cfg.data_zarr, Path) + if "idx_slice_start" in data: + assert cfg.idx_slice_start == data["idx_slice_start"] + if "idx_slice_stop" in data: + assert cfg.idx_slice_stop == data["idx_slice_stop"] + if "idx_slice_step" in data: + assert cfg.idx_slice_step == data["idx_slice_step"] + + def test_slice_defaults(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + data.pop("idx_slice_start", None) + data.pop("idx_slice_stop", None) + data.pop("idx_slice_step", None) + cfg = UpdateCfg(**data) + assert cfg.idx_slice_start == 0 + assert cfg.idx_slice_stop == -1 + assert cfg.idx_slice_step == 1 + + def test_missing_required_field_raises(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + del data["scan_number"] + with pytest.raises(ValidationError): + UpdateCfg(**data) + + +class TestConvertCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + cfg = ConvertCfg(**data) + assert isinstance(cfg.outputdir, Path) + + def test_missing_outputdir_raises(self): + with pytest.raises(ValidationError): + ConvertCfg() diff --git a/CHAP/test/saxswaxs/server/test_server.py b/CHAP/test/saxswaxs/server/test_server.py new file mode 100644 index 0000000..97c6bf3 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_server.py @@ -0,0 +1,133 @@ +"""Tests for the Flask HTTP interface in CHAP.saxswaxs.server.server.""" + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +@pytest.fixture +def client(): + with patch("CHAP.saxswaxs.server.server.put"): + from CHAP.saxswaxs.server.server import app + app.config["TESTING"] = True + with app.test_client() as c: + yield c + + +def _setup_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + return data + + +def _update_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + return data + + +def _convert_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + return data + + +class TestSetupEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_setup_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + mock_put.assert_called_once() + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "setup" + assert type(args[0]).__name__ == "SetupCfg" + assert kwargs == {} + + def test_setup_cfg_fields_correct(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + body = _setup_body(fixture_dir, tmp_path) + client.post("/setup", json=body) + _, args, _ = mock_put.call_args[0] + cfg = args[0] + assert cfg.scan_number == body["scan_number"] + assert cfg.dwell_time_actual_counter_name == body["dwell_time_actual_counter_name"] + assert cfg.dataset_chunks == body["dataset_chunks"] + + def test_missing_required_field_returns_error(self, client, fixture_dir, tmp_path): + body = _setup_body(fixture_dir, tmp_path) + del body["spec_file"] + resp = client.post("/setup", json=body) + assert resp.status_code >= 400 + + def test_malformed_json_returns_error(self, client): + resp = client.post("/setup", data="not json", content_type="application/json") + assert resp.status_code >= 400 + + +class TestUpdateEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_update_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + mock_put.assert_called_once() + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "update" + assert type(args[0]).__name__ == "UpdateCfg" + + def test_update_cfg_slice_correct(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + body = _update_body(fixture_dir, tmp_path) + client.post("/update", json=body) + _, args, _ = mock_put.call_args[0] + cfg = args[0] + assert cfg.idx_slice_start == body.get("idx_slice_start", 0) + assert cfg.idx_slice_stop == body.get("idx_slice_stop", -1) + assert cfg.idx_slice_step == body.get("idx_slice_step", 1) + + +class TestConvertEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_convert_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "convert" + assert type(args[0]).__name__ == "ConvertCfg" + + +class TestRequestLogging: + def test_request_completes_without_logging_error(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put"): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 diff --git a/CHAP/test/saxswaxs/server/test_task_queue.py b/CHAP/test/saxswaxs/server/test_task_queue.py new file mode 100644 index 0000000..d4f53d2 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_task_queue.py @@ -0,0 +1,73 @@ +"""Unit tests for CHAP.saxswaxs.server.task_queue.""" + +import queue +import threading +from unittest.mock import MagicMock, call, patch + +import pytest + +import CHAP.saxswaxs.server.task_queue as tq + + +def drain(q, timeout=2): + """Block until q is empty or timeout expires.""" + q.join() + + +class TestPut: + def test_put_enqueues_tuple(self): + task = MagicMock() + args = (1, 2) + kwargs = {"a": 3} + tq.put(task, args, kwargs) + item = tq._task_queue.get_nowait() + tq._task_queue.task_done() + assert item == (task, args, kwargs) + + +class TestWorker: + def _run_worker_once(self, task, args=(), kwargs={}): + """Put one task, wait for it to complete.""" + tq.put(task, args, kwargs) + tq._task_queue.join() + + def test_worker_calls_task(self): + called = threading.Event() + task = MagicMock(side_effect=lambda: called.set()) + self._run_worker_once(task) + assert called.wait(timeout=5), "worker never called the task" + task.assert_called_once() + + def test_worker_passes_args_and_kwargs(self): + received = {} + + def capture(*args, **kwargs): + received["args"] = args + received["kwargs"] = kwargs + + tq.put(capture, (10, 20), {"x": 99}) + tq._task_queue.join() + assert received["args"] == (10, 20) + assert received["kwargs"] == {"x": 99} + + def test_worker_retries_on_exception(self): + """Worker should retry a failing task until it succeeds.""" + attempt_count = [0] + done = threading.Event() + + def flaky(): + attempt_count[0] += 1 + if attempt_count[0] < 2: + raise RuntimeError("transient failure") + done.set() + + tq.put(flaky, (), {}) + assert done.wait(timeout=15), "worker never retried the task" + assert attempt_count[0] == 2 + + def test_worker_marks_task_done_after_success(self): + sentinel = threading.Event() + tq.put(lambda: sentinel.set(), (), {}) + # join() blocks until task_done() is called; it would hang on failure + tq._task_queue.join() + assert sentinel.is_set() From 29cf973a275331250f6155fd0cfd3293ab1177a1 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 09:50:09 -0400 Subject: [PATCH 04/12] fix: remove temporary debugging measure --- CHAP/saxswaxs/server/task_queue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py index 793dcea..1f34476 100644 --- a/CHAP/saxswaxs/server/task_queue.py +++ b/CHAP/saxswaxs/server/task_queue.py @@ -31,7 +31,6 @@ def _worker(): except Exception as exc: logger.error(f'Task failed: {exc}') print_exc() - success = True # FIXME temporary for debugging sleep(5) _task_queue.task_done() tf = time() From fbbc09da07ddd17669afb07dc2a24cc0b039bdc9 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 10:04:46 -0400 Subject: [PATCH 05/12] rm: replace scan_to_map.py with direct use of SpecScanToMapConfigProcessor --- CHAP/saxswaxs/server/chap.py | 32 ++++--- CHAP/saxswaxs/server/scan_to_map.py | 143 ---------------------------- 2 files changed, 17 insertions(+), 158 deletions(-) delete mode 100644 CHAP/saxswaxs/server/scan_to_map.py diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py index 132042f..eaf973b 100644 --- a/CHAP/saxswaxs/server/chap.py +++ b/CHAP/saxswaxs/server/chap.py @@ -1,8 +1,9 @@ """CHAP processing code using "cached" CHAP ``PipelineItem``s for better performance.""" +from CHAP.common.map_utils import SpecScanToMapConfigProcessor from CHAP.common.reader import YAMLReader -from CHAP.common.writer import ZarrWriter, ZarrValuesWriter +from CHAP.common.writer import YAMLWriter, ZarrWriter, ZarrValuesWriter from CHAP.pipeline import PipelineData from CHAP.saxswaxs.processor import SetupProcessor, UpdateValuesProcessor from functools import cache @@ -13,7 +14,6 @@ from pydantic import BaseModel, ConfigDict from chap_daemon import get_logger -from chap_daemon.scan_to_map import scan_to_map from chap_daemon.saxswaxs_to_chap import ( saxswaxs_to_chap, make_pipeline as _make_pipeline, @@ -229,21 +229,23 @@ def setup_configs(cfg): and output file paths. :type cfg: SetupCfg """ - logger.info( - "scan_to_map(" - f"{cfg.spec_file}, {cfg.scan_number}, 'id3b', 'SAXSWAXS', " - f"{cfg.dwell_time_actual_counter_name}, " - f"{cfg.presample_intensity_counter_name}, " - f"{cfg.postsample_intensity_counter_name}, " - f"{cfg.map_yaml})" + map_config = PipelineData( + data=SpecScanToMapConfigProcessor.run( + spec_file=cfg.spec_file, + scan_number=cfg.scan_number, + station="id3b", + experiment="SAXSWAXS", + dwell_time_actual_counter_name=cfg.dwell_time_actual_counter_name, + presample_intensity_counter_name=cfg.presample_intensity_counter_name, + postsample_intensity_counter_name=cfg.postsample_intensity_counter_name, + validate_data_present=False, + ), ) - scan_to_map( - str(cfg.spec_file), cfg.scan_number, "id3b", "SAXSWAXS", - cfg.dwell_time_actual_counter_name, - cfg.presample_intensity_counter_name, - cfg.postsample_intensity_counter_name, - str(cfg.map_yaml), + YAMLWriter.run( + data=map_config, + filename=cfg.map_yaml, ) + logger.info( f"saxswaxs_to_chap({cfg.map_yaml}, {cfg.tool_yamls}, {cfg.outputdir})" ) diff --git a/CHAP/saxswaxs/server/scan_to_map.py b/CHAP/saxswaxs/server/scan_to_map.py deleted file mode 100644 index ee9acde..0000000 --- a/CHAP/saxswaxs/server/scan_to_map.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/nfs/chess/sw/miniforge3_chap/envs/CHAP_saxswaxs/bin/python -"""Script to convert a single SPEC scan into its -MapConfigRepresentation using CHAP tools -""" - -def spec_scan_to_map_config( - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name): - """Convert a single SPEC scan into a CHAP map config data structure. - - :param spec_file: Path to the SPEC file containing the scan. - :type spec_file: str - :param scan_number: Number of the scan within the SPEC file. - :type scan_number: int - :param station: Station identifier (e.g. ``'id3b'``). - :type station: str - :param experiment: Experiment type (e.g. ``'SAXSWAXS'``). - :type experiment: str - :param dwell_time_actual_counter_name: SPEC counter column name for actual - dwell times. - :type dwell_time_actual_counter_name: str - :param presample_intensity_counter_name: SPEC counter column name for - presample intensity. - :type presample_intensity_counter_name: str - :param postsample_intensity_counter_name: SPEC counter column name for - postsample intensity, or ``None`` if not recorded. - :type postsample_intensity_counter_name: str or None - :returns: Map config data structure. - :rtype: dict - """ - from CHAP.common.map_utils import SpecScanToMapConfigProcessor - - proc = SpecScanToMapConfigProcessor() - map_config = proc.process( - None, - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name, - validate_data_present=False, - ) - return map_config - -def write_yaml(data, filename): - """Write data to a YAML file, creating parent directories as needed. - - :param data: Data to serialize to YAML. - :param filename: Output file path. - :type filename: str - """ - import os - import yaml - - os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, 'w') as outf: - yaml.dump(data, outf, sort_keys=False) - -def scan_to_map( - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name, - map_config_filename): - """Convert a SPEC scan to a map config YAML file. - - :param spec_file: Path to the SPEC file containing the scan. - :type spec_file: str - :param scan_number: Number of the scan within the SPEC file. - :type scan_number: int - :param station: Station identifier (e.g. ``'id3b'``). - :type station: str - :param experiment: Experiment type (e.g. ``'SAXSWAXS'``). - :type experiment: str - :param dwell_time_actual_counter_name: SPEC counter column name for actual - dwell times. - :type dwell_time_actual_counter_name: str - :param presample_intensity_counter_name: SPEC counter column name for - presample intensity. - :type presample_intensity_counter_name: str - :param postsample_intensity_counter_name: SPEC counter column name for - postsample intensity, or ``None`` if not recorded. - :type postsample_intensity_counter_name: str or None - :param map_config_filename: Output path for the map config YAML file. - :type map_config_filename: str - """ - map_config_data = spec_scan_to_map_config( - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name, - ) - write_yaml(map_config_data, map_config_filename) - - -if __name__ == '__main__': - import argparse - import os - import sys - - parser = argparse.ArgumentParser( - description='''Create a map_config.yaml representing the given - SPEC scan''') - parser.add_argument( - '--spec_file', required=True, - help='Name of the SPEC file containing the scan of interest.') - parser.add_argument( - '--scan_number', required=True, type=int, - help='Number of the scan of interest within the given SPEC file.') - parser.add_argument( - '--outputdir', required=True, help='''Path to the output - analysis directory for this scan. The map configuration YAML - and reduced data NeXus files will be written here.''') - parser.add_argument( - '--dwell_time_actual_counter_name', required=True, - help='''Name of the SPEC counter column representing actual - dwell times''') - parser.add_argument( - '--presample_intensity_counter_name', required=True, - help='''Name of the SPEC counter column representing presample - intensity values''') - parser.add_argument( - '--postsample_intensity_counter_name', - help='''Name of the SPEC counter column representing postsample - intensity values''') - parser.add_argument( - '--station', choices=['id3b'], default='id3b', - help='''Name of the station at which the scan was collected - ("id3b" is currently the only supported value)''') - parser.add_argument( - '--experiment', choices=['SAXSWAXS'], default='SAXSWAXS', - help='''Name of the scan\'s experiment type ("SAXSWAXS" is - currently the only supported value).''') - args = parser.parse_args(sys.argv[1:]) - - scan_to_map( - args.spec_file, args.scan_number, args.station, args.experiment, - args.dwell_time_actual_counter_name, - args.presample_intensity_counter_name, - args.postsample_intensity_counter_name, - os.path.join(args.outputdir, 'map_config.yaml') - ) From d743d47cda343a0cd6a05d78f3b1e9e1ebd8af55 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 10:06:46 -0400 Subject: [PATCH 06/12] fix: chap_daemon -> CHAP.saxswaxs.server --- CHAP/saxswaxs/server/chap.py | 8 ++++---- CHAP/saxswaxs/server/server.py | 14 +++++++------- CHAP/saxswaxs/server/task_queue.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py index eaf973b..1ce9abf 100644 --- a/CHAP/saxswaxs/server/chap.py +++ b/CHAP/saxswaxs/server/chap.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, ConfigDict -from chap_daemon import get_logger -from chap_daemon.saxswaxs_to_chap import ( +from CHAP.saxswaxs.server import get_logger +from CHAP.saxswaxs.server.saxswaxs_to_chap import ( saxswaxs_to_chap, make_pipeline as _make_pipeline, convert_configs as _convert_configs, @@ -428,7 +428,7 @@ def make_pipeline(cfg): """Run the make_pipeline task to write a ``pipeline.yaml`` from pre-existing config files. - Calls :func:`chap_daemon.saxswaxs_to_chap.make_pipeline` with the paths + Calls :func:`CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline` with the paths and filenames from ``cfg``. :param cfg: Configuration for the make_pipeline task. @@ -449,7 +449,7 @@ def convert_configs(cfg): """Run the convert_configs task to write detector, pyFAI integration, and corrections config YAML files from old-style saxswaxs workflow tool configs. - Calls :func:`chap_daemon.saxswaxs_to_chap.convert_configs` with the paths + Calls :func:`CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs` with the paths and filenames from ``cfg``. :param cfg: Configuration for the convert_configs task. diff --git a/CHAP/saxswaxs/server/server.py b/CHAP/saxswaxs/server/server.py index 6706489..25ea775 100644 --- a/CHAP/saxswaxs/server/server.py +++ b/CHAP/saxswaxs/server/server.py @@ -5,9 +5,9 @@ import time from traceback import print_exc -from chap_daemon import get_logger -from chap_daemon.task_queue import put -from chap_daemon.chap import ( +from CHAP.saxswaxs.server import get_logger +from CHAP.saxswaxs.server.task_queue import put +from CHAP.saxswaxs.server.chap import ( setup, update, convert, make_pipeline, convert_configs, SetupCfg, UpdateCfg, ConvertCfg, MakePipelineCfg, ConvertConfigsCfg, ) @@ -125,8 +125,8 @@ def convert_handler(): def make_pipeline_handler(): """Handle POST /make_pipeline — parse JSON body and queue a make_pipeline task. - Constructs a :class:`~chap_daemon.chap.MakePipelineCfg` from the request - body and queues :func:`~chap_daemon.chap.make_pipeline` to write a + Constructs a :class:`~CHAP.saxswaxs.server.chap.MakePipelineCfg` from the request + body and queues :func:`~CHAP.saxswaxs.server.chap.make_pipeline` to write a ``pipeline.yaml`` from the pre-existing config files in ``outputdir``. """ body = request.get_json(force=True, silent=True) @@ -158,8 +158,8 @@ def make_pipeline_handler(): def convert_configs_handler(): """Handle POST /convert_configs — parse JSON body and queue a convert_configs task. - Constructs a :class:`~chap_daemon.chap.ConvertConfigsCfg` from the request - body and queues :func:`~chap_daemon.chap.convert_configs` to write detector, + Constructs a :class:`~CHAP.saxswaxs.server.chap.ConvertConfigsCfg` from the request + body and queues :func:`~CHAP.saxswaxs.server.chap.convert_configs` to write detector, pyFAI integration, and corrections config YAML files from the provided tool config files. """ diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py index 1f34476..52dbc44 100644 --- a/CHAP/saxswaxs/server/task_queue.py +++ b/CHAP/saxswaxs/server/task_queue.py @@ -5,7 +5,7 @@ from traceback import print_exc from time import sleep, time -from chap_daemon import get_logger +from CHAP.saxswaxs.server import get_logger logger = get_logger('task_queue') From 012fdf4609f7da96b99b4cf575114ebabf9c3575 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 10:56:49 -0400 Subject: [PATCH 07/12] refactor: eliminate saxswaxsw server dependency on old saxswaxsworkflow package installation --- CHAP/saxswaxs/server/saxswaxs_to_chap.py | 465 +--------------- .../saxswaxs/server/test_saxswaxs_to_chap.py | 525 ++++++++++++++++++ 2 files changed, 539 insertions(+), 451 deletions(-) create mode 100644 CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py diff --git a/CHAP/saxswaxs/server/saxswaxs_to_chap.py b/CHAP/saxswaxs/server/saxswaxs_to_chap.py index b89522f..4b3f236 100755 --- a/CHAP/saxswaxs/server/saxswaxs_to_chap.py +++ b/CHAP/saxswaxs/server/saxswaxs_to_chap.py @@ -30,9 +30,9 @@ def saxswaxs_to_chap( pipeline_filename='pipeline.yaml'): """Build CHAP pipeline config files from old-style saxswaxs workflow configs. - Loads the map and tool config files using the old ``workflow`` library, then - calls :func:`wf_to_chap` to write the corresponding CHAP pipeline YAML files - to ``outputdir``. + Converts old-style tool config files to CHAP-format detector, pyFAI + integration, and corrections config files via :func:`convert_configs`, + then composes the CHAP pipeline config YAML via :func:`make_pipeline`. :param map_config_file: Path to the map config YAML file. :type map_config_file: str @@ -49,30 +49,21 @@ def saxswaxs_to_chap( :param correction_filename: Output filename for the corrections config YAML, defaults to ``'corrections_config.yaml'``. :type correction_filename: str, optional + :param fits_filename: Output filename for the fits config YAML, + defaults to ``'fits_config.yaml'``. + :type fits_filename: str, optional :param pipeline_filename: Output filename for the CHAP pipeline config YAML, defaults to ``'pipeline.yaml'``. :type pipeline_filename: str, optional """ - # Initialize old-style saxswaxs workflow configuration objects - from workflow.map import MapConfig - from workflow.basemodel import BaseModel - from workflow.workflow import Workflow - - logger = logging.getLogger(__name__) - map_config = MapConfig.construct_from_file( - map_config_file, - logger=logger, validate_data_present=False) - tools = [ - BaseModel.construct_from_file( - tool_config_file, logger=logger) - for tool_config_file in tool_config_files - ] - wf = Workflow(map_config=map_config, tools=tools, - validate_data_present=False) - - # Compose chap pipeline config form old-style saxswaxs workflow - chap_config = wf_to_chap( - wf, outputdir, + convert_configs( + outputdir, tool_config_files, + detector_filename=detector_filename, + pyfai_filename=pyfai_filename, + correction_filename=correction_filename, + ) + make_pipeline( + outputdir, map_filename=map_config_file, detector_filename=detector_filename, pyfai_filename=pyfai_filename, @@ -352,434 +343,6 @@ def convert_configs(outputdir, tool_config_files, Dumper=VerboseSafeDumper) -def wf_to_chap(wf, outputdir, - map_filename='map_config.yaml', - detector_filename='detector_config.yaml', - pyfai_filename='pyfai_integration_processor_config.yaml', - correction_filename='corrections_config.yaml', - fits_filename='fits_config.yaml', - pipeline_filename='pipeline.yaml'): - """Convert an old-style SAXSWAXS Workflow configuration into the - analogous CHAP pipeline configuration. - - Writes detector, pyFAI integration, corrections, map, and pipeline YAML - config files to ``outputdir`` and returns the pipeline config dict. - - :param wf: Workflow configuration to convert. - :type wf: workflow.Workflow - :param outputdir: Directory to which all output config .yaml files will - be written. - :type outputdir: str - :param map_filename: Filename for the map config .yaml file, defaults to - ``'map_config.yaml'``. - :type map_filename: str, optional - :param detector_filename: Filename for the detector config .yaml file, - defaults to ``'detector_config.yaml'``. - :type detector_filename: str, optional - :param pyfai_filename: Filename for the PyfaiIntegrationProcessorConfig - .yaml file, defaults to ``'pyfai_integration_processor_config.yaml'``. - :type pyfai_filename: str, optional - :param correction_filename: Filename for the corrections config .yaml file, - defaults to ``'corrections_config.yaml'``. - :type correction_filename: str, optional - :param pipeline_filename: Filename for the CHAP pipeline config .yaml file, - defaults to ``'pipeline.yaml'``. - :type pipeline_filename: str, optional - :returns: Full CHAP pipeline config dict (setup, per-row update, and convert - pipelines). - :rtype: dict - """ - outputdir = os.path.abspath(outputdir) - - # Write map configuration to file - map_config = wf.map_config - if not os.path.isabs(map_filename): - map_filename = os.path.join(outputdir, map_filename) - - # Write pyfai integration processor config to file - pyfai_integration_processor_config = { - 'azimuthal_integrators': [], - 'integrations': [], - } - detectors = [] - corrections = [] - for t in wf.tools: - if hasattr(t, 'detectors'): - # It's an integration tool. - add_integration(pyfai_integration_processor_config, t) - for det in t.detectors: - visited = any([det.prefix == d['id'] for d in detectors]) - if not visited: - # Decide on detector shape (use list not tuple for - # yaml file) - placeholder_shape = (1, 1) - if det.prefix == 'PIL5': - shape = [619, 487] - elif det.prefix in ('PIL9', 'PIL11'): - shape = [407, 487] - else: - print( - f'WARNING: unrecorgnized detector prefix {det.prefix}; ' - + f'using placeholder shape {placeholder_shape}' - ) - shape = placeholder_shape - detectors.append( - { - 'id': det.prefix, - 'shape': shape, - } - ) - else: - # It's a corrections tool - corrections.append( - t.model_dump( - mode='json', - exclude_unset=True, - exclude_defaults=True, - exclude=['validate_data_present'], - ) - ) - - # Write detector config .yaml - detector_config = {'detectors': detectors} - if not os.path.isabs(detector_filename): - detector_filename = os.path.join(outputdir, detector_filename) - print(f'Writing to {detector_filename}') - os.makedirs(os.path.dirname(detector_filename), exist_ok=True) - with open(detector_filename, 'w') as outf: - yaml.dump(detector_config, outf, sort_keys=False, - Dumper=VerboseSafeDumper) - - # Write pyfai config .yaml - if not os.path.isabs(pyfai_filename): - pyfai_filename = os.path.join(outputdir, pyfai_filename) - print(f'Writing to {pyfai_filename}') - os.makedirs(os.path.dirname(pyfai_filename), exist_ok=True) - with open(pyfai_filename, 'w') as outf: - yaml.dump(pyfai_integration_processor_config, outf, sort_keys=False, - Dumper=VerboseSafeDumper) - - # Write corrections config .yaml - correction_config = {'corrections': corrections} - if not os.path.isabs(correction_filename): - correction_filename = os.path.join(outputdir, correction_filename) - print(f'Writing to {correction_filename}') - os.makedirs(os.path.dirname(correction_filename), exist_ok=True) - with open(correction_filename, 'w') as outf: - yaml.dump(correction_config, outf, sort_keys=False, - Dumper=VerboseSafeDumper) - - # Iterate through each scan in the map and compose the individual - # scan-row-wise update jobs so the dataset can be processed in - # parallel by running all update pipelines at once, or - # incrementally by running each update pipeline one at a time. - update_pipelines = {} - npts = 0 - nrows = 0 - zarr_filename = f'{wf.map_config.title}.zarr' - nxs_filename = f'{wf.map_config.title}.nxs' - for scans in map_config.spec_scans: - for scan_number in scans.scan_numbers: - sp = scans.get_scanparser(scan_number) - _npts = int(sp.spec_scan_npts) - if len(sp.spec_scan_shape) > 1: - _nrows = sp.spec_scan_shape[1] - row_npts = sp.spec_scan_shape[0] - else: - _nrows = 1 - row_npts = _npts - for i in range(_nrows): - idx_slice = { - 'start': npts + (i * row_npts), - 'stop': npts + (i * row_npts) + row_npts, - 'step': 1, - } - update_pipelines[f'update_{nrows + i}'] = [ - { - 'common.reader.YAMLReader': { - 'filename': detector_filename, - 'schema': 'common.models.map.DetectorConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': map_filename, - 'schema': 'common.models.map.MapConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': pyfai_filename, - 'schema': 'common.models.integration.PyfaiIntegrationConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': correction_filename, - 'schema': 'saxswaxs.models.CorrectionsConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': fits_filename, - 'schema': 'saxswaxs.models.FitsConfig' - } - }, - { - 'saxswaxs.processor.UpdateValuesProcessor': { - 'raw_data': False, - 'filename': zarr_filename, - 'spec_file': scans.spec_file, - 'scan_number': scan_number, - 'idx_slice': idx_slice, - } - }, - { - 'common.ZarrValuesWriter': { - 'filename': zarr_filename, - 'resize_axis': 0, - 'idx_slice': idx_slice, - 'force_overwrite': True, - } - } - ] - nrows += _nrows - npts += _npts - - # Compose final CHAP pipeline config and write to file. - # _restructure_pipeline = restructure_pipeline(wf) - chap_config = { - 'config': { - 'root': outputdir, - 'log_level': 'debug', - }, - 'setup': [ - { - 'common.reader.YAMLReader': { - 'filename': detector_filename, - 'schema': 'common.models.map.DetectorConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': map_filename, - 'schema': 'common.models.map.MapConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': pyfai_filename, - 'schema': 'common.models.integration.PyfaiIntegrationConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': correction_filename, - 'schema': 'saxswaxs.models.CorrectionsConfig' - } - }, - { - 'common.reader.YAMLReader': { - 'filename': fits_filename, - 'schema': 'saxswaxs.models.FitsConfig' - } - }, - { - 'saxswaxs.processor.SetupProcessor': { - 'raw_data': False, - 'dataset_chunks': [row_npts], - } - }, - { - 'common.writer.ZarrWriter': { - 'filename': f'{wf.map_config.title}.zarr', - 'force_overwrite': True - } - } - ], - **update_pipelines, - 'convert': [ - { - 'common.processor.ZarrToNexusProcessor': { - 'zarr_filename': zarr_filename, - 'nexus_filename': nxs_filename, - } - } - ], - # 'struct': _restructure_pipeline, - } - pipeline_filename = os.path.join(outputdir, pipeline_filename) - print(f'Writing to {pipeline_filename}') - os.makedirs(os.path.dirname(pipeline_filename), exist_ok=True) - with open(pipeline_filename, 'w') as outf: - yaml.dump(chap_config, outf, sort_keys=False, Dumper=VerboseSafeDumper) - return chap_config - - -def restructure_pipeline(wf): - """Build a CHAP pipeline config for restructuring data from unstructured to structured form. - - Reads independent dimension axes and all tool output signals from an existing - NeXus file and passes them to - ``saxswaxs.processor.UnstructuredToStructuredProcessor``, writing a structured - dataset back to the same NeXus file. - - .. note:: This function is not currently used; see the commented-out lines in - :func:`wf_to_chap`. - - :param wf: Old-style saxswaxs workflow configuration. - :type wf: workflow.Workflow - :returns: List of CHAP pipeline step dicts for the restructure pipeline. - :rtype: list[dict] - """ - axes = [a.label for a in wf.map_config.independent_dimensions] - readers = [ - { - 'common.reader.NexusReader': { - 'filename': f'{wf.map_config.title}.nxs', - 'nxpath': f'{wf.map_config.title}/independent_dimensions/{a}', - 'nxmemory': 100000, - 'name': a - } - } - for a in axes] - fields = [ - { - 'name': a, - 'type': 'axis' - } - for a in axes] - tools = {t.title: t for t in wf.tools} - for title, tool in tools.items(): - if hasattr(tool, 'integration_type'): - # Integration tool; only signal is intensity "I" - signal = 'I' - _axes = tool.integrated_data_dims - elif hasattr(tool, 'correction_type'): - # Corrections tool; varying signals - signal = 'result' - _axes = tools[tool.uncorrected_data_title].integrated_data_dims - readers.extend( - [ - { - 'common.reader.NexusReader': { - 'filename': f'{wf.map_config.title}.nxs', - 'nxpath': f'{title}/data/{x}', - 'nxmemory': 100000, - 'name': f'{title}_{x}' - } - } - for x in [signal] + _axes] - ) - fields.extend( - [ - { - 'name': f'{title}_{signal}', - 'type': 'signal', - 'axes': axes + [f'{title}_{a}' for a in _axes] - }, - *[ - { - 'name': f'{title}_{a}', - 'type': 'axis' - } - for a in _axes - ] - ] - ) - pipeline = [ - *readers, - { - 'saxswaxs.processor.UnstructuredToStructuredProcessor': { - 'fields': fields - } - }, - { - 'common.writer.NexusWriter': { - 'filename': f'{wf.map_config.title}.nxs', - 'nxpath': '/structured_data', - 'force_overwrite': True, - } - } - ] - return pipeline - - -def add_integration(pyfai_integration_processor_config, tool_config): - """Add the necessary components from a `workflow.IntegrationTool` - to an existing config for a - `saxswaxs.PyfaiIntegrationProcessor`. - - :param pyfai_integration_processor_config: Partial config for - `saxswaxs.PyfaiIntegrationProcessor` to which the given tool's - integration will be added. - :type pyfai_integration_processor_config: dict - :param tool_config: Old-style workflow integration tool object. - :type tool_config: workflow.integration.IntegrationConfig - :returns: Updated `pyfai_integration_processor_config` - :rtype: dict - """ - def detector_in_config(det): - """Convenience function for determining if the given detector - is already in `pyfai_integration_processor_config`. - """ - for _det in pyfai_integration_processor_config['azimuthal_integrators']: - if (_det['id'] == det.prefix - and _det['poni_file'] == str(det.poni_file) - and _det['mask_file'] == str(det.mask_file)): - return True - return False - - for detector in tool_config.detectors: - if not detector_in_config(detector): - pyfai_integration_processor_config['azimuthal_integrators'].append( - { - 'id': str(detector.prefix), - 'poni_file': str(detector.poni_file), - 'mask_file': str(detector.mask_file), - } - ) - - integration_config = { - 'name': tool_config.title, - } - if tool_config.integration_type == 'radial': - integration_config['integration_method'] = 'integrate_radial' - integration_config['integration_params'] = { - 'ais': [det.prefix for det in tool_config.detectors], - 'npt': tool_config.azimuthal_npt, - 'npt_rad': tool_config.radial_npt, - 'radial_range': [tool_config.radial_min, tool_config.radial_max], - 'azimuth_range': [tool_config.azimuthal_min, tool_config.azimuthal_max], - 'unit': tool_config.azimuthal_units, - 'radial_unit': tool_config.radial_units, - 'method': 'bbox_csr_cython', - } - else: - integration_config['multi_geometry'] = { - 'ais': [det.prefix for det in tool_config.detectors], - 'unit': tool_config.radial_units, - 'radial_range': [tool_config.radial_min, tool_config.radial_max], - 'azimuth_range': [tool_config.azimuthal_min, tool_config.azimuthal_max], - } - if tool_config.integration_type == 'azimuthal': - integration_config['integration_method'] = 'integrate1d' - integration_config['integration_params'] = { - 'npt': tool_config.radial_npt, - 'method': 'bbox_csr_cython', - } - elif tool_config.integration_type == 'cake': - integration_config['integration_method'] = 'integrate2d' - integration_config['integration_params'] = { - 'npt_rad': tool_config.radial_npt, - 'npt_azim': tool_config.azimuthal_npt, - 'method': 'bbox_csr_cython', - } - pyfai_integration_processor_config['integrations'].append(integration_config) - - return pyfai_integration_processor_config - - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( diff --git a/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py b/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py new file mode 100644 index 0000000..a23a0a1 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py @@ -0,0 +1,525 @@ +"""Tests for CHAP.saxswaxs.server.saxswaxs_to_chap.""" + +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +import pytest +import yaml + +from CHAP.saxswaxs.server.saxswaxs_to_chap import ( + VerboseSafeDumper, + convert_configs, + make_pipeline, + saxswaxs_to_chap, +) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +# Minimal integration tool payload; override individual keys per test. +_INTEGRATION_DEFAULTS = { + 'tool_type': 'integration', + 'title': 'my_integration', + 'integration_type': 'azimuthal', + 'detectors': [ + {'prefix': 'PIL5', 'poni_file': '/cal/det.poni', 'mask_file': '/cal/mask.tif'} + ], + 'radial_npt': 100, + 'azimuthal_npt': 72, + 'radial_min': 0.1, + 'radial_max': 5.0, + 'azimuthal_min': -180.0, + 'azimuthal_max': 180.0, + 'radial_units': 'q_A^-1', + 'azimuthal_units': 'chi_deg', +} + +_CORRECTIONS_DEFAULTS = { + 'tool_type': 'corrections', + 'title': 'my_correction', + 'correction_type': 'transmission', + 'validate_data_present': False, +} + + +def _write_tool(path, base, **overrides): + """Write a tool YAML to path, merging overrides into base.""" + data = {**base, **overrides} + path.write_text(yaml.dump(data)) + return data + + +def _write_integration(path, **overrides): + return _write_tool(path, _INTEGRATION_DEFAULTS, **overrides) + + +def _write_corrections(path, **overrides): + return _write_tool(path, _CORRECTIONS_DEFAULTS, **overrides) + + +def _read_outputs(tmp_path, det_name='detector_config.yaml', + pyfai_name='pyfai_integration_processor_config.yaml', + corr_name='corrections_config.yaml'): + with open(tmp_path / det_name) as f: + det = yaml.safe_load(f) + with open(tmp_path / pyfai_name) as f: + pyfai = yaml.safe_load(f) + with open(tmp_path / corr_name) as f: + corr = yaml.safe_load(f) + return det, pyfai, corr + + +def _make_mock_map(title='scan', scans=None): + """Return a mock MapConfig for make_pipeline tests. + + scans: list of (spec_file, scan_number, npts, shape) tuples. + """ + if scans is None: + scans = [('/spec/scan.spec', 1, 10, [10])] + mock_spec_scans = [] + for spec_file, scan_number, npts, shape in scans: + sp = MagicMock() + sp.spec_scan_npts = npts + sp.spec_scan_shape = shape + sg = MagicMock() + sg.spec_file = spec_file + sg.scan_numbers = [scan_number] + sg.get_scanparser.return_value = sp + mock_spec_scans.append(sg) + mc = MagicMock() + mc.title = title + mc.spec_scans = mock_spec_scans + return mc + + +# ── VerboseSafeDumper ───────────────────────────────────────────────────────── + +class TestVerboseSafeDumper: + def test_shared_reference_produces_no_alias(self): + shared = {'key': 'value'} + out = yaml.dump({'a': shared, 'b': shared}, Dumper=VerboseSafeDumper) + assert '*' not in out + assert '&' not in out + + def test_output_roundtrips(self): + data = {'x': [1, 2, 3], 'y': {'nested': True}} + out = yaml.dump(data, Dumper=VerboseSafeDumper) + assert yaml.safe_load(out) == data + + def test_nested_shared_reference(self): + node = [1, 2, 3] + out = yaml.dump({'p': node, 'q': node, 'r': node}, Dumper=VerboseSafeDumper) + assert out.count('- 1') == 3 + + +# ── convert_configs ─────────────────────────────────────────────────────────── + +class TestConvertConfigsOutputFiles: + def test_all_three_files_created(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + assert (tmp_path / 'detector_config.yaml').exists() + assert (tmp_path / 'pyfai_integration_processor_config.yaml').exists() + assert (tmp_path / 'corrections_config.yaml').exists() + + def test_absolute_output_filenames_respected(self, tmp_path): + sub = tmp_path / 'sub' + sub.mkdir() + _write_integration(tmp_path / 'tool.yaml') + det = str(sub / 'det.yaml') + pyfai = str(sub / 'pyfai.yaml') + corr = str(sub / 'corr.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')], + detector_filename=det, + pyfai_filename=pyfai, + correction_filename=corr) + assert Path(det).exists() + assert Path(pyfai).exists() + assert Path(corr).exists() + + def test_relative_output_filenames_resolved_against_outputdir(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')], + detector_filename='custom_det.yaml') + assert (tmp_path / 'custom_det.yaml').exists() + + +class TestConvertConfigsIntegrationMethod: + @pytest.mark.parametrize('integration_type,expected_method', [ + ('azimuthal', 'integrate1d'), + ('radial', 'integrate_radial'), + ('cake', 'integrate2d'), + ]) + def test_integration_method(self, tmp_path, integration_type, expected_method): + _write_integration(tmp_path / 'tool.yaml', integration_type=integration_type) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['integration_method'] == expected_method + + def test_radial_uses_azimuthal_npt_as_npt(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='radial', + radial_npt=50, azimuthal_npt=36) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + params = pyfai['integrations'][0]['integration_params'] + assert params['npt'] == 36 + assert params['npt_rad'] == 50 + + def test_azimuthal_uses_radial_npt_as_npt(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='azimuthal', + radial_npt=200) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['integration_params']['npt'] == 200 + + def test_cake_uses_both_npt_fields(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='cake', + radial_npt=80, azimuthal_npt=45) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + params = pyfai['integrations'][0]['integration_params'] + assert params['npt_rad'] == 80 + assert params['npt_azim'] == 45 + + def test_azimuthal_and_cake_have_multi_geometry(self, tmp_path): + for itype in ('azimuthal', 'cake'): + _write_integration(tmp_path / f'{itype}.yaml', integration_type=itype) + for itype in ('azimuthal', 'cake'): + convert_configs(str(tmp_path), [str(tmp_path / f'{itype}.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert 'multi_geometry' in pyfai['integrations'][0], itype + + def test_radial_does_not_have_multi_geometry(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='radial') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert 'multi_geometry' not in pyfai['integrations'][0] + + def test_integration_name_from_title(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', title='waxs_1d') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['name'] == 'waxs_1d' + + def test_multiple_tools_produce_multiple_integrations(self, tmp_path): + _write_integration(tmp_path / 'a.yaml', title='a') + _write_integration(tmp_path / 'b.yaml', title='b') + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert len(pyfai['integrations']) == 2 + + +class TestConvertConfigsDetectors: + @pytest.mark.parametrize('prefix,expected_shape', [ + ('PIL5', [619, 487]), + ('PIL9', [407, 487]), + ('PIL11', [407, 487]), + ]) + def test_known_detector_shape(self, tmp_path, prefix, expected_shape): + dets = [{'prefix': prefix, 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + det = next(d for d in det_cfg['detectors'] if d['id'] == prefix) + assert det['shape'] == expected_shape + + def test_unknown_detector_gets_placeholder_shape(self, tmp_path): + dets = [{'prefix': 'NEWDET', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + det = next(d for d in det_cfg['detectors'] if d['id'] == 'NEWDET') + assert det['shape'] == [1, 1] + + def test_detector_deduplicated_across_tools(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'a.yaml', title='a', detectors=dets) + _write_integration(tmp_path / 'b.yaml', title='b', detectors=dets) + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + assert [d['id'] for d in det_cfg['detectors']].count('PIL5') == 1 + + def test_azimuthal_integrator_deduplicated_across_tools(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'a.yaml', title='a', detectors=dets) + _write_integration(tmp_path / 'b.yaml', title='b', detectors=dets) + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + ids = [ai['id'] for ai in pyfai['azimuthal_integrators']] + assert ids.count('PIL5') == 1 + + def test_azimuthal_integrator_poni_and_mask_stored(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/exp/det.poni', + 'mask_file': '/exp/mask.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + ai = pyfai['azimuthal_integrators'][0] + assert ai['poni_file'] == '/exp/det.poni' + assert ai['mask_file'] == '/exp/mask.tif' + + +class TestConvertConfigsCorrections: + def test_corrections_tool_written_to_corrections_config(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert len(corr['corrections']) == 1 + + def test_corrections_excludes_tool_type_key(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert 'tool_type' not in corr['corrections'][0] + + def test_corrections_excludes_validate_data_present(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert 'validate_data_present' not in corr['corrections'][0] + + def test_corrections_preserves_other_fields(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml', custom_field='hello') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert corr['corrections'][0].get('custom_field') == 'hello' + + def test_integration_tool_does_not_appear_in_corrections(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert corr['corrections'] == [] + + def test_corrections_tool_does_not_appear_in_detectors(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det, _, _ = _read_outputs(tmp_path) + assert det['detectors'] == [] + + +# ── make_pipeline ───────────────────────────────────────────────────────────── + +class TestMakePipeline: + def _run(self, tmp_path, mock_map, **kwargs): + with patch('CHAP.common.reader.YAMLReader') as MockReader: + MockReader.run.return_value = mock_map + result = make_pipeline(str(tmp_path), **kwargs) + with open(tmp_path / kwargs.get('pipeline_filename', 'pipeline.yaml')) as f: + written = yaml.safe_load(f) + return result, written + + def test_pipeline_file_written(self, tmp_path): + _, _ = self._run(tmp_path, _make_mock_map()) + assert (tmp_path / 'pipeline.yaml').exists() + + def test_custom_pipeline_filename(self, tmp_path): + self._run(tmp_path, _make_mock_map(), pipeline_filename='custom.yaml') + assert (tmp_path / 'custom.yaml').exists() + + def test_returns_dict_with_required_keys(self, tmp_path): + result, _ = self._run(tmp_path, _make_mock_map()) + assert isinstance(result, dict) + assert 'config' in result + assert 'setup' in result + assert 'convert' in result + + def test_config_root_is_resolved_outputdir(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + assert written['config']['root'] == str(tmp_path.resolve()) + + def test_setup_has_five_yaml_readers(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + readers = [s for s in written['setup'] if 'common.reader.YAMLReader' in s] + assert len(readers) == 5 + + def test_setup_yaml_reader_schemas(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + schemas = [s['common.reader.YAMLReader']['schema'] + for s in written['setup'] if 'common.reader.YAMLReader' in s] + assert 'common.models.map.DetectorConfig' in schemas + assert 'common.models.map.MapConfig' in schemas + assert 'common.models.integration.PyfaiIntegrationConfig' in schemas + assert 'saxswaxs.models.CorrectionsConfig' in schemas + assert 'saxswaxs.models.FitsConfig' in schemas + + def test_setup_has_setup_processor(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + procs = [s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s] + assert len(procs) == 1 + + def test_setup_processor_raw_data_false(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['raw_data'] is False + + def test_setup_has_zarr_writer(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + writers = [s for s in written['setup'] if 'common.writer.ZarrWriter' in s] + assert len(writers) == 1 + + def test_zarr_filename_derived_from_map_title(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(title='my_exp')) + writer = next(s for s in written['setup'] if 'common.writer.ZarrWriter' in s) + assert writer['common.writer.ZarrWriter']['filename'] == 'my_exp.zarr' + + @pytest.mark.parametrize('scans,expected_update_count', [ + ([('/spec', 1, 10, [10])], 1), # 1D scan → 1 update + ([('/spec', 1, 20, [5, 4])], 4), # 2D scan, 4 rows → 4 updates + ([('/spec', 1, 10, [10]), + ('/spec', 2, 5, [5])], 2), # two 1D scans → 2 updates + ]) + def test_update_pipeline_count(self, tmp_path, scans, expected_update_count): + _, written = self._run(tmp_path, _make_mock_map(scans=scans)) + update_keys = [k for k in written if k.startswith('update_')] + assert len(update_keys) == expected_update_count + + def test_update_pipeline_idx_slice_1d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc = next(s for s in written['update_0'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + assert proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] == { + 'start': 0, 'stop': 10, 'step': 1, + } + + def test_update_pipeline_idx_slice_2d_rows(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 20, [5, 4])])) + for row in range(4): + proc = next(s for s in written[f'update_{row}'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + sl = proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + assert sl == {'start': row * 5, 'stop': row * 5 + 5, 'step': 1} + + def test_update_npts_accumulates_across_scans(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(scans=[ + ('/spec', 1, 10, [10]), + ('/spec', 2, 5, [5]), + ])) + proc = next(s for s in written['update_1'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + sl = proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + assert sl['start'] == 10 + assert sl['stop'] == 15 + + def test_dataset_chunks_equals_row_npts_1d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['dataset_chunks'] == [10] + + def test_dataset_chunks_equals_row_npts_2d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 30, [10, 3])])) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['dataset_chunks'] == [10] + + def test_convert_pipeline_zarr_and_nxs(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(title='exp')) + assert len(written['convert']) == 1 + cfg = written['convert'][0]['common.processor.ZarrToNexusProcessor'] + assert cfg['zarr_filename'] == 'exp.zarr' + assert cfg['nexus_filename'] == 'exp.nxs' + + def test_update_pipeline_has_zarr_values_writer(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + writers = [s for s in written['update_0'] + if 'common.ZarrValuesWriter' in s] + assert len(writers) == 1 + + def test_update_pipeline_zarr_values_writer_idx_slice_matches_processor(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc_sl = next(s for s in written['update_0'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s + )['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + writer_sl = next(s for s in written['update_0'] + if 'common.ZarrValuesWriter' in s + )['common.ZarrValuesWriter']['idx_slice'] + assert proc_sl == writer_sl + + def test_reader_filenames_resolved_against_outputdir(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + reader = written['setup'][0]['common.reader.YAMLReader'] + assert reader['filename'].startswith(str(tmp_path.resolve())) + + +# ── saxswaxs_to_chap ────────────────────────────────────────────────────────── + +class TestSaxswaxsToChap: + def test_calls_convert_configs_with_outputdir_and_tools(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline'): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + mock_cc.assert_called_once() + args, kwargs = mock_cc.call_args + assert args[0] == str(tmp_path) + assert args[1] == [tool] + + def test_calls_make_pipeline_with_outputdir_and_map(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs'), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + mock_mp.assert_called_once() + args, kwargs = mock_mp.call_args + assert args[0] == str(tmp_path) + assert kwargs['map_filename'] == 'map.yaml' + + def test_default_filenames_forwarded_to_convert_configs(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline'): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + _, kwargs = mock_cc.call_args + assert kwargs['detector_filename'] == 'detector_config.yaml' + assert kwargs['pyfai_filename'] == 'pyfai_integration_processor_config.yaml' + assert kwargs['correction_filename'] == 'corrections_config.yaml' + + def test_default_filenames_forwarded_to_make_pipeline(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs'), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + _, kwargs = mock_mp.call_args + assert kwargs['fits_filename'] == 'fits_config.yaml' + assert kwargs['pipeline_filename'] == 'pipeline.yaml' + + def test_custom_filenames_forwarded(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap( + 'map.yaml', [tool], str(tmp_path), + detector_filename='det.yaml', + pyfai_filename='pyfai.yaml', + correction_filename='corr.yaml', + fits_filename='fits.yaml', + pipeline_filename='pipe.yaml', + ) + _, cc_kwargs = mock_cc.call_args + assert cc_kwargs['detector_filename'] == 'det.yaml' + assert cc_kwargs['pyfai_filename'] == 'pyfai.yaml' + assert cc_kwargs['correction_filename'] == 'corr.yaml' + _, mp_kwargs = mock_mp.call_args + assert mp_kwargs['detector_filename'] == 'det.yaml' + assert mp_kwargs['fits_filename'] == 'fits.yaml' + assert mp_kwargs['pipeline_filename'] == 'pipe.yaml' + + def test_convert_configs_called_before_make_pipeline(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + call_order = [] + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs', + side_effect=lambda *a, **kw: call_order.append('cc')), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline', + side_effect=lambda *a, **kw: call_order.append('mp')): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + assert call_order == ['cc', 'mp'] From c366c1f43ab0f798fb3183397b1af9ba1ebda033 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Wed, 12 Aug 2026 11:41:57 -0400 Subject: [PATCH 08/12] bump: pydantify SpecScanToMapConfigProcessor --- CHAP/common/map_utils.py | 90 +++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/CHAP/common/map_utils.py b/CHAP/common/map_utils.py index c3db28a..e05dc7a 100755 --- a/CHAP/common/map_utils.py +++ b/CHAP/common/map_utils.py @@ -335,44 +335,48 @@ class SpecScanToMapConfigProcessor(Processor): """Processor to get the :class:`~CHAP.common.models.map.MapConfig` dictionary configuration representation of a single CHESS SPEC scan. + + :ivar spec_file: Path to the SPEC file. + :vartype spec_file: pydantic.FilePath + :ivar scan_number: Scan number within the SPEC file. + :vartype scan_number: int + :ivar station: Name of the station at which the data was collected. + :vartype station: Literal["id1a3", "id3a", "id3b", "id4b"] + :ivar experiment: Experiment type. + :vartype experiment: Literal[ + 'EDD', 'GIWAXS', 'HDRM', 'SAXSWAXS', 'TOMO', 'XRF'] + :ivar dwell_time_actual_counter_name: Name of the counter used to + record the actual dwell time at time of data collection. + :vartype dwell_time_actual_counter_name: str + :ivar presample_intensity_counter_name: Name of the counter used to + record the incident beam intensity at time of data collection. + :vartype presample_intensity_counter_name: str + :ivar postsample_intensity_counter_name: Name of the counter used to + record the post-sample beam intensity at time of data collection. + Defaults to ``None``. + :vartype postsample_intensity_counter_name: str, optional + :ivar validate_data_present: Whether to include a + ``validate_data_present`` key in the output map configuration. + Defaults to ``True``. + :vartype validate_data_present: bool """ - def process(self, data, - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name=None, - validate_data_present=True): + spec_file: FilePath + scan_number: int + station: str + experiment: str + dwell_time_actual_counter_name: str + presample_intensity_counter_name: str + postsample_intensity_counter_name: Optional[str] = None + validate_data_present: bool = True + + def process(self, data): """Return a dictionary representing a valid :class:`~CHAP.common.models.map.MapConfig` object that contains only the single given scan. - :param spec_file: Spec file name - :type spec_file: str - :param scan_number: Scan number - :type scan_number: int - :param station: Name of the station at which the data was - collected. - :type station: Literal["id1a3", "id3a", "id3b", "id4b"] - :param experiment: Experiment type - :type experiment_type: Literal[ - 'EDD', 'GIWAXS', 'HDRM', 'SAXSWAXS', 'TOMO', 'XRF'] - :param dwell_time_actual_counter_name: Name of the counter used - to record the actual dwell time at time of data collection. - :type dwell_time_actual_counter_name: str - :param presample_intensity_counter_name: Name of the counter - used to record the incident beam intensity at time of data - collection. - :type presample_intensity_counter_name: str - :param postsample_intensity_counter_name: Name of the counter - used to record the post sample beam intensity at time of - data collection. - :type postsample_intensity_counter_name: str, optional - :param validate_data_present: Optional `validate_data_present` - key-value pair to the output map configuration, defaults - to `True`. - :type validate_data_present: - :returns: Single-scan map configuration + :param data: Unused; present for pipeline interface compatibility. + :returns: Single-scan map configuration. :rtype: dict """ # System modules @@ -381,8 +385,8 @@ def process(self, data, # Local modules from chess_scanparsers import choose_scanparser - SP = choose_scanparser(station, experiment) - sp = SP(spec_file, scan_number) + SP = choose_scanparser(self.station, self.experiment) + sp = SP(str(self.spec_file), self.scan_number) def get_independent_dimensions(_scanparser): """Return a value for the `independent_dimensions` field of @@ -470,35 +474,35 @@ def get_independent_dimensions(_scanparser): for mne in _scanparser.spec_scan_motor_mnes], []) - normalized_spec_file = os.path.realpath(spec_file).replace( + normalized_spec_file = os.path.realpath(self.spec_file).replace( '/daq/', '/raw/') independent_dimensions, scalar_data = get_independent_dimensions(sp) mapconfig_dict = { - 'validate_data_present': validate_data_present, + 'validate_data_present': self.validate_data_present, 'title': sp.scan_title, - 'station': station, - 'experiment_type': experiment.upper(), + 'station': self.station, + 'experiment_type': self.experiment.upper(), 'sample': { 'name': sp.scan_name }, 'spec_scans': [ { 'spec_file': normalized_spec_file, - 'scan_numbers': [scan_number] + 'scan_numbers': [self.scan_number] } ], 'independent_dimensions': independent_dimensions, 'dwell_time_actual': { 'data_type': 'scan_column', - 'name': dwell_time_actual_counter_name}, + 'name': self.dwell_time_actual_counter_name}, 'presample_intensity': { 'data_type': 'scan_column', - 'name': presample_intensity_counter_name}, + 'name': self.presample_intensity_counter_name}, 'scalar_data': scalar_data, } - if postsample_intensity_counter_name: + if self.postsample_intensity_counter_name: mapconfig_dict['postsample_intensity'] = { 'data_type': 'scan_column', - 'name': postsample_intensity_counter_name, + 'name': self.postsample_intensity_counter_name, } return mapconfig_dict From 8d6b10b628e9ee6732b305f9ccc66be689b46343 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Tue, 1 Sep 2026 12:29:22 -0400 Subject: [PATCH 09/12] feat: rotating datestamped file-based logging for saxswaxs-server --- CHAP/saxswaxs/server/__init__.py | 29 ------ CHAP/saxswaxs/server/chap.py | 16 +++- CHAP/saxswaxs/server/logging_config.py | 122 +++++++++++++++++++++++++ CHAP/saxswaxs/server/server.py | 3 +- CHAP/saxswaxs/server/task_queue.py | 2 +- 5 files changed, 135 insertions(+), 37 deletions(-) create mode 100644 CHAP/saxswaxs/server/logging_config.py diff --git a/CHAP/saxswaxs/server/__init__.py b/CHAP/saxswaxs/server/__init__.py index 7540a6c..f8e0e1a 100644 --- a/CHAP/saxswaxs/server/__init__.py +++ b/CHAP/saxswaxs/server/__init__.py @@ -1,31 +1,2 @@ """Daemon-like application for efficient automated SAXS/WAXS data processing.""" - -def get_logger(name=__name__, log_level="DEBUG"): - """Create and return a :class:`logging.Logger` with a stream handler. - - Configures the logger with a formatted :class:`logging.StreamHandler` - that writes to stderr. Re-assigning ``logger.handlers`` ensures no - duplicate handlers accumulate on repeated calls with the same *name*. - - :param name: Logger name, typically the calling module's ``__name__``. - :type name: str - :param log_level: Case-insensitive logging level string - (e.g. ``"DEBUG"``, ``"INFO"``, ``"WARNING"``). - :type log_level: str - :returns: Configured logger instance. - :rtype: logging.Logger - """ - import logging - - logger = logging.getLogger(name) - log_level = getattr(logging, log_level.upper()) - logger.setLevel(log_level) - log_handler = logging.StreamHandler() - log_handler.setFormatter(logging.Formatter( - '{asctime}: {name:20} (L{lineno}): {levelname}: {message}', - datefmt='%Y-%m-%d %H:%M:%S', style='{')) - logger.addHandler(log_handler) - logger.handlers = [log_handler] - logger.propagate = False - return logger diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py index 1ce9abf..2836793 100644 --- a/CHAP/saxswaxs/server/chap.py +++ b/CHAP/saxswaxs/server/chap.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict -from CHAP.saxswaxs.server import get_logger +from CHAP.saxswaxs.server.logging_config import get_logger from CHAP.saxswaxs.server.saxswaxs_to_chap import ( saxswaxs_to_chap, make_pipeline as _make_pipeline, @@ -133,13 +133,15 @@ def setup(cfg): raw_data=False, ), name='saxswaxs.processor.SetupProcessor.run', + logger=get_logger("SetupProcessor.run"), ) ] logger.info('Writing') ZarrWriter.run( data=zarr_tree, filename=str(cfg.data_zarr), - force_overwrite=True + force_overwrite=True, + logger=get_logger("ZarrWriter.run"), ) @@ -174,6 +176,7 @@ def update(cfg): raw_data=True, ), name='UpdateValuesProcessor.run', + logger=get_logger("UpdateValuesProcessor.run"), ) ] logger.info('Writing') @@ -187,6 +190,7 @@ def update(cfg): step=cfg.idx_slice_step, ), force_overwrite=True, + logger=get_logger("ZarrValuesWriter.run") ) @@ -229,7 +233,7 @@ def setup_configs(cfg): and output file paths. :type cfg: SetupCfg """ - map_config = PipelineData( + map_config = [PipelineData( data=SpecScanToMapConfigProcessor.run( spec_file=cfg.spec_file, scan_number=cfg.scan_number, @@ -239,11 +243,13 @@ def setup_configs(cfg): presample_intensity_counter_name=cfg.presample_intensity_counter_name, postsample_intensity_counter_name=cfg.postsample_intensity_counter_name, validate_data_present=False, + logger=get_logger("SpecScanToMapConfigProcessor.run") ), - ) + )] YAMLWriter.run( data=map_config, - filename=cfg.map_yaml, + filename=str(cfg.map_yaml), + logger=get_logger("YAMLWriter.run") ) logger.info( diff --git a/CHAP/saxswaxs/server/logging_config.py b/CHAP/saxswaxs/server/logging_config.py new file mode 100644 index 0000000..4c9a167 --- /dev/null +++ b/CHAP/saxswaxs/server/logging_config.py @@ -0,0 +1,122 @@ +"""Logging configuration""" + +import logging +import os +import sys +from logging.handlers import TimedRotatingFileHandler + + +_LOG_HANDLER = None +_STDOUT_WRAPPER = None +_STDERR_WRAPPER = None + +def get_logger(name=__name__, log_level="DEBUG"): + global _STDOUT_WRAPPER + global _STDERR_WRAPPER + + logger = logging.getLogger(name) + logger.propagate = False + logger.setLevel(getattr(logging, log_level.upper())) + + handler = _get_log_handler() + + # Avoid duplicate handlers. + logger.handlers = [handler] + + # Redirect stdout/stderr once. + if _STDOUT_WRAPPER is None: + _STDOUT_WRAPPER = StreamToLogFile(handler) + _STDERR_WRAPPER = StreamToLogFile(handler) + + sys.stdout = _STDOUT_WRAPPER + sys.stderr = _STDERR_WRAPPER + + return logger + + +class StreamToLogFile: + """Write stdout/stderr directly to the rotating log file.""" + + def __init__(self, handler): + self.handler = handler + + def write(self, message): + if not message: + return + + # Click and some other libraries may write bytes rather than str. + if isinstance(message, bytes): + message = message.decode("utf-8", errors="replace") + + self.handler.acquire() + try: + # Rotate if necessary. + record = logging.LogRecord( + name="stream", + level=logging.INFO, + pathname="", + lineno=0, + msg="", + args=(), + exc_info=None, + ) + + if self.handler.shouldRollover(record): + self.handler.doRollover() + + self.handler.stream.write(message) + self.handler.stream.flush() + + finally: + self.handler.release() + + def flush(self): + self.handler.acquire() + try: + if self.handler.stream is not None: + self.handler.stream.flush() + finally: + self.handler.release() + + def isatty(self): + return False + + @property + def encoding(self): + return self.handler.encoding + + def __getattr__(self, name): + # Let libraries such as Click access other normal stream + # attributes/methods if necessary. + return getattr(sys.__stdout__, name) + + +def _get_log_handler(): + global _LOG_HANDLER + + if _LOG_HANDLER is None: + log_file = os.path.join( + os.path.dirname(__file__), + "saxswaxs-server", + ) + + _LOG_HANDLER = TimedRotatingFileHandler( + log_file, + when="midnight", + interval=1, + backupCount=30, + encoding="utf-8", + ) + + _LOG_HANDLER.suffix = "%Y-%m-%d.log" + + _LOG_HANDLER.setFormatter( + logging.Formatter( + "{asctime}: {name:20} (L{lineno}): " + "{levelname}: {message}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{", + ) + ) + + return _LOG_HANDLER diff --git a/CHAP/saxswaxs/server/server.py b/CHAP/saxswaxs/server/server.py index 25ea775..d099e00 100644 --- a/CHAP/saxswaxs/server/server.py +++ b/CHAP/saxswaxs/server/server.py @@ -5,7 +5,7 @@ import time from traceback import print_exc -from CHAP.saxswaxs.server import get_logger +from CHAP.saxswaxs.server.logging_config import get_logger from CHAP.saxswaxs.server.task_queue import put from CHAP.saxswaxs.server.chap import ( setup, update, convert, make_pipeline, convert_configs, @@ -14,7 +14,6 @@ app = Flask(__name__) app.logger = get_logger('server') -app.logger.propagate = False # Logging middleware @app.before_request diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py index 52dbc44..a24a9fd 100644 --- a/CHAP/saxswaxs/server/task_queue.py +++ b/CHAP/saxswaxs/server/task_queue.py @@ -5,7 +5,7 @@ from traceback import print_exc from time import sleep, time -from CHAP.saxswaxs.server import get_logger +from CHAP.saxswaxs.server.logging_config import get_logger logger = get_logger('task_queue') From 1caccc53503b712deb6e68fe42f7751eb2928c28 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Tue, 1 Sep 2026 13:01:02 -0400 Subject: [PATCH 10/12] feat: additional unique log files for each chap task --- CHAP/saxswaxs/server/chap.py | 2 +- CHAP/saxswaxs/server/logging_config.py | 61 +++++++++++++++++++++++++- CHAP/saxswaxs/server/task_queue.py | 31 ++++++++----- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py index 2836793..51c36f0 100644 --- a/CHAP/saxswaxs/server/chap.py +++ b/CHAP/saxswaxs/server/chap.py @@ -207,7 +207,7 @@ def convert(cfg): logger.info("CHAP convert starting") logname = cfg.outputdir / "chap_convert.log" - with open(logname, "w") as logfile: + with open(logname, "a") as logfile: process = subprocess.Popen( [ "CHAP", diff --git a/CHAP/saxswaxs/server/logging_config.py b/CHAP/saxswaxs/server/logging_config.py index 4c9a167..ab04582 100644 --- a/CHAP/saxswaxs/server/logging_config.py +++ b/CHAP/saxswaxs/server/logging_config.py @@ -1,14 +1,17 @@ """Logging configuration""" +import contextlib import logging import os import sys from logging.handlers import TimedRotatingFileHandler +from pathlib import Path _LOG_HANDLER = None _STDOUT_WRAPPER = None _STDERR_WRAPPER = None +_TASK_HANDLER = None def get_logger(name=__name__, log_level="DEBUG"): global _STDOUT_WRAPPER @@ -20,8 +23,10 @@ def get_logger(name=__name__, log_level="DEBUG"): handler = _get_log_handler() - # Avoid duplicate handlers. - logger.handlers = [handler] + handlers = [handler] + if _TASK_HANDLER is not None: + handlers.append(_TASK_HANDLER) + logger.handlers = handlers # Redirect stdout/stderr once. if _STDOUT_WRAPPER is None: @@ -34,6 +39,49 @@ def get_logger(name=__name__, log_level="DEBUG"): return logger +@contextlib.contextmanager +def task_log_context(log_path): + """Context manager that tees log output to a per-task append-only file. + + While active, all loggers created via :func:`get_logger` write to + *log_path* in addition to the shared rotating log file, using the same + formatter. Raw stdout/stderr writes are also teed to the same file + without any additional formatting. The file is always opened in append + mode so successive calls accumulate rather than overwrite. + """ + global _TASK_HANDLER + + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + + task_handler = logging.FileHandler(str(log_path), mode='a', encoding='utf-8') + task_handler.setFormatter(_get_log_handler().formatter) + + prev_task_handler = _TASK_HANDLER + _TASK_HANDLER = task_handler + + # Add to all loggers that were created by get_logger (propagate=False). + pre_existing = [ + lgr for lgr in logging.Logger.manager.loggerDict.values() + if isinstance(lgr, logging.Logger) and not lgr.propagate + ] + for lgr in pre_existing: + lgr.addHandler(task_handler) + + try: + yield + finally: + _TASK_HANDLER = prev_task_handler + + # Remove from every logger that received this handler (pre-existing + # and any created inside the context via get_logger). + for lgr in logging.Logger.manager.loggerDict.values(): + if isinstance(lgr, logging.Logger) and task_handler in lgr.handlers: + lgr.handlers = [h for h in lgr.handlers if h is not task_handler] + + task_handler.close() + + class StreamToLogFile: """Write stdout/stderr directly to the rotating log file.""" @@ -70,6 +118,15 @@ def write(self, message): finally: self.handler.release() + # Tee raw output to the active task log file. + if _TASK_HANDLER is not None: + _TASK_HANDLER.acquire() + try: + _TASK_HANDLER.stream.write(message) + _TASK_HANDLER.stream.flush() + finally: + _TASK_HANDLER.release() + def flush(self): self.handler.acquire() try: diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py index a24a9fd..6bcf23f 100644 --- a/CHAP/saxswaxs/server/task_queue.py +++ b/CHAP/saxswaxs/server/task_queue.py @@ -5,7 +5,10 @@ from traceback import print_exc from time import sleep, time -from CHAP.saxswaxs.server.logging_config import get_logger +from CHAP.saxswaxs.server.logging_config import ( + get_logger, + task_log_context, +) logger = get_logger('task_queue') @@ -18,20 +21,21 @@ def _worker(): and the task is marked done. """ while True: - task, args, kwargs = _task_queue.get() + task, args, kwargs, log_path = _task_queue.get() logger.info( f'Starting task: {str(task)}, args: {args}, kwargs: {kwargs}') t0 = time() success = False while not success: - # Handle race conditions from missing data - try: - task(*args, **kwargs) - success = True - except Exception as exc: - logger.error(f'Task failed: {exc}') - print_exc() - sleep(5) + with task_log_context(log_path): + # Handle race conditions from missing data + try: + task(*args, **kwargs) + success = True + except Exception as exc: + logger.error(f'Task failed: {exc}') + print_exc() + sleep(5) _task_queue.task_done() tf = time() logger.info(f'Task done. ({tf-t0:.5f} seconds)') @@ -45,6 +49,11 @@ def put(task, args, kwargs): :param kwargs: Keyword arguments to pass to ``task``. :type kwargs: dict """ - _task_queue.put((task, args, kwargs)) + cfg = args[0] + if hasattr(cfg, 'outputdir'): + log_path = cfg.outputdir / f"chap_{task.__name__}.log" + else: + log_path = cfg.data_zarr.parent / f"chap_{task.__name__}.log" + _task_queue.put((task, args, kwargs, log_path)) threading.Thread(target=_worker, daemon=True).start() From c00adb632ffcebd0f7fac6864148396e525414b0 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Tue, 1 Sep 2026 13:21:43 -0400 Subject: [PATCH 11/12] fix: saxswaxs automation: create nexus file one directory above zarr file --- CHAP/saxswaxs/server/saxswaxs_to_chap.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHAP/saxswaxs/server/saxswaxs_to_chap.py b/CHAP/saxswaxs/server/saxswaxs_to_chap.py index 4b3f236..6f8e20f 100755 --- a/CHAP/saxswaxs/server/saxswaxs_to_chap.py +++ b/CHAP/saxswaxs/server/saxswaxs_to_chap.py @@ -79,10 +79,12 @@ def make_pipeline(outputdir, pyfai_filename='pyfai_integration_processor_config.yaml', correction_filename='corrections_config.yaml', fits_filename='fits_config.yaml', - pipeline_filename='pipeline.yaml'): + pipeline_filename='pipeline.yaml', + ): """Compose a pipeline file for a complete saxswaxs workflow based on the config files provided, and asssuming they all already exist. Sort of a lightweight version of wf_to_chap.""" + from CHAP.common.models.map import MapConfig from CHAP.common.reader import YAMLReader outputdir = os.path.abspath(outputdir) @@ -98,10 +100,11 @@ def make_pipeline(outputdir, if not os.path.isabs(fits_filename): fits_filename = os.path.join(outputdir, fits_filename) - map_config = YAMLReader.run( - filename=map_filename, schema='common.models.map.MapConfig') - zarr_filename = f'{map_config.title}.zarr' - nxs_filename = f'{map_config.title}.nxs' + map_config = MapConfig(**YAMLReader.run( + filename=map_filename, schema='common.models.map.MapConfig')) + + zarr_filename = os.path.join(outputdir, f'{map_config.title}.zarr') + nxs_filename = os.path.abspath(os.path.join(outputdir, '..', f'{map_config.title}.nxs')) readers = [ { From 7101e487133ab2b8f1d61dbb4abe2582da49fcf2 Mon Sep 17 00:00:00 2001 From: Keara Soloway Date: Tue, 1 Sep 2026 13:22:17 -0400 Subject: [PATCH 12/12] fix: saxswaxs-server logging bug --- CHAP/saxswaxs/server/chap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py index 51c36f0..2f2ecfa 100644 --- a/CHAP/saxswaxs/server/chap.py +++ b/CHAP/saxswaxs/server/chap.py @@ -131,9 +131,9 @@ def setup(cfg): data=data, dataset_chunks=cfg.dataset_chunks, raw_data=False, + logger=get_logger("SetupProcessor.run"), ), name='saxswaxs.processor.SetupProcessor.run', - logger=get_logger("SetupProcessor.run"), ) ] logger.info('Writing') @@ -243,12 +243,13 @@ def setup_configs(cfg): presample_intensity_counter_name=cfg.presample_intensity_counter_name, postsample_intensity_counter_name=cfg.postsample_intensity_counter_name, validate_data_present=False, - logger=get_logger("SpecScanToMapConfigProcessor.run") + logger=get_logger("SpecScanToMapConfigProcessor.run"), ), )] YAMLWriter.run( data=map_config, filename=str(cfg.map_yaml), + force_overwrite=True, logger=get_logger("YAMLWriter.run") )