diff --git a/CHAP/common/map_utils.py b/CHAP/common/map_utils.py index c3db28ae..e05dc7a2 100755 --- a/CHAP/common/map_utils.py +++ b/CHAP/common/map_utils.py @@ -335,44 +335,48 @@ class SpecScanToMapConfigProcessor(Processor): """Processor to get the :class:`~CHAP.common.models.map.MapConfig` dictionary configuration representation of a single CHESS SPEC scan. + + :ivar spec_file: Path to the SPEC file. + :vartype spec_file: pydantic.FilePath + :ivar scan_number: Scan number within the SPEC file. + :vartype scan_number: int + :ivar station: Name of the station at which the data was collected. + :vartype station: Literal["id1a3", "id3a", "id3b", "id4b"] + :ivar experiment: Experiment type. + :vartype experiment: Literal[ + 'EDD', 'GIWAXS', 'HDRM', 'SAXSWAXS', 'TOMO', 'XRF'] + :ivar dwell_time_actual_counter_name: Name of the counter used to + record the actual dwell time at time of data collection. + :vartype dwell_time_actual_counter_name: str + :ivar presample_intensity_counter_name: Name of the counter used to + record the incident beam intensity at time of data collection. + :vartype presample_intensity_counter_name: str + :ivar postsample_intensity_counter_name: Name of the counter used to + record the post-sample beam intensity at time of data collection. + Defaults to ``None``. + :vartype postsample_intensity_counter_name: str, optional + :ivar validate_data_present: Whether to include a + ``validate_data_present`` key in the output map configuration. + Defaults to ``True``. + :vartype validate_data_present: bool """ - def process(self, data, - spec_file, scan_number, station, experiment, - dwell_time_actual_counter_name, - presample_intensity_counter_name, - postsample_intensity_counter_name=None, - validate_data_present=True): + spec_file: FilePath + scan_number: int + station: str + experiment: str + dwell_time_actual_counter_name: str + presample_intensity_counter_name: str + postsample_intensity_counter_name: Optional[str] = None + validate_data_present: bool = True + + def process(self, data): """Return a dictionary representing a valid :class:`~CHAP.common.models.map.MapConfig` object that contains only the single given scan. - :param spec_file: Spec file name - :type spec_file: str - :param scan_number: Scan number - :type scan_number: int - :param station: Name of the station at which the data was - collected. - :type station: Literal["id1a3", "id3a", "id3b", "id4b"] - :param experiment: Experiment type - :type experiment_type: Literal[ - 'EDD', 'GIWAXS', 'HDRM', 'SAXSWAXS', 'TOMO', 'XRF'] - :param dwell_time_actual_counter_name: Name of the counter used - to record the actual dwell time at time of data collection. - :type dwell_time_actual_counter_name: str - :param presample_intensity_counter_name: Name of the counter - used to record the incident beam intensity at time of data - collection. - :type presample_intensity_counter_name: str - :param postsample_intensity_counter_name: Name of the counter - used to record the post sample beam intensity at time of - data collection. - :type postsample_intensity_counter_name: str, optional - :param validate_data_present: Optional `validate_data_present` - key-value pair to the output map configuration, defaults - to `True`. - :type validate_data_present: - :returns: Single-scan map configuration + :param data: Unused; present for pipeline interface compatibility. + :returns: Single-scan map configuration. :rtype: dict """ # System modules @@ -381,8 +385,8 @@ def process(self, data, # Local modules from chess_scanparsers import choose_scanparser - SP = choose_scanparser(station, experiment) - sp = SP(spec_file, scan_number) + SP = choose_scanparser(self.station, self.experiment) + sp = SP(str(self.spec_file), self.scan_number) def get_independent_dimensions(_scanparser): """Return a value for the `independent_dimensions` field of @@ -470,35 +474,35 @@ def get_independent_dimensions(_scanparser): for mne in _scanparser.spec_scan_motor_mnes], []) - normalized_spec_file = os.path.realpath(spec_file).replace( + normalized_spec_file = os.path.realpath(self.spec_file).replace( '/daq/', '/raw/') independent_dimensions, scalar_data = get_independent_dimensions(sp) mapconfig_dict = { - 'validate_data_present': validate_data_present, + 'validate_data_present': self.validate_data_present, 'title': sp.scan_title, - 'station': station, - 'experiment_type': experiment.upper(), + 'station': self.station, + 'experiment_type': self.experiment.upper(), 'sample': { 'name': sp.scan_name }, 'spec_scans': [ { 'spec_file': normalized_spec_file, - 'scan_numbers': [scan_number] + 'scan_numbers': [self.scan_number] } ], 'independent_dimensions': independent_dimensions, 'dwell_time_actual': { 'data_type': 'scan_column', - 'name': dwell_time_actual_counter_name}, + 'name': self.dwell_time_actual_counter_name}, 'presample_intensity': { 'data_type': 'scan_column', - 'name': presample_intensity_counter_name}, + 'name': self.presample_intensity_counter_name}, 'scalar_data': scalar_data, } - if postsample_intensity_counter_name: + if self.postsample_intensity_counter_name: mapconfig_dict['postsample_intensity'] = { 'data_type': 'scan_column', - 'name': postsample_intensity_counter_name, + 'name': self.postsample_intensity_counter_name, } return mapconfig_dict diff --git a/CHAP/saxswaxs/server/__init__.py b/CHAP/saxswaxs/server/__init__.py new file mode 100644 index 00000000..f8e0e1af --- /dev/null +++ b/CHAP/saxswaxs/server/__init__.py @@ -0,0 +1,2 @@ +"""Daemon-like application for efficient automated SAXS/WAXS data +processing.""" diff --git a/CHAP/saxswaxs/server/chap.py b/CHAP/saxswaxs/server/chap.py new file mode 100644 index 00000000..2f2ecfa4 --- /dev/null +++ b/CHAP/saxswaxs/server/chap.py @@ -0,0 +1,471 @@ +"""CHAP processing code using "cached" CHAP ``PipelineItem``s for +better performance.""" + +from CHAP.common.map_utils import SpecScanToMapConfigProcessor +from CHAP.common.reader import YAMLReader +from CHAP.common.writer import YAMLWriter, ZarrWriter, ZarrValuesWriter +from CHAP.pipeline import PipelineData +from CHAP.saxswaxs.processor import SetupProcessor, UpdateValuesProcessor +from functools import cache +from pathlib import Path +import subprocess +from typing import Optional + +from pydantic import BaseModel, ConfigDict + +from CHAP.saxswaxs.server.logging_config import get_logger +from CHAP.saxswaxs.server.saxswaxs_to_chap import ( + saxswaxs_to_chap, + make_pipeline as _make_pipeline, + convert_configs as _convert_configs, +) + +# functions whose caches will need to be cleared regularly to work +# with live data processing +from CHAP.common.models.map import get_scanparser +from chess_scanparsers.scanparsers import ( + filespec, + list_fmb_saxswaxs_detector_files, +) + +logger = get_logger('chap') + +def cache_clear(): + """Clear scan parser and file listing caches before processing new data.""" + get_scanparser.cache_clear() + filespec.cache_clear() + list_fmb_saxswaxs_detector_files.cache_clear() + + +def init(): + """No-op placeholder for module initialization.""" + pass + + +@cache +def _read_yaml(filename, schema): + """Read a YAML config file and return it wrapped in a :class:`PipelineData` object. + + Results are cached so the same file is only read once per process lifetime. + Call :func:`cache_clear` before processing new data to invalidate stale entries. + + :param filename: Path to the YAML file to read. + :type filename: str or Path + :param schema: CHAP schema string used to validate and parse the YAML contents. + :type schema: str + :returns: Parsed config wrapped in a PipelineData container. + :rtype: PipelineData + """ + return PipelineData( + name='YAMLReader', + data=YAMLReader.run( + filename=str(filename), + schema=schema, + ), + schema=schema, + ) + + +@cache +def read_configs(detectors_yaml, map_yaml, pyfai_yaml, corrections_yaml, fits_yaml): + """Read the four config YAML files required for SAXS/WAXS processing. + + Results are cached; call :func:`cache_clear` before processing a new scan. + + :param detectors_yaml: Path to the detector config YAML file. + :type detectors_yaml: Path + :param map_yaml: Path to the map config YAML file. + :type map_yaml: Path + :param pyfai_yaml: Path to the pyFAI integration processor config YAML file. + :type pyfai_yaml: Path + :param corrections_yaml: Path to the corrections config YAML file. + :type corrections_yaml: Path + :returns: List of four PipelineData objects for detector, map, pyFAI, and + corrections configs respectively. + :rtype: list[PipelineData] + """ + return [ + _read_yaml( + detectors_yaml, + 'common.models.map.DetectorConfig' + ), + _read_yaml( + map_yaml, + 'common.models.map.MapConfig' + ), + _read_yaml( + pyfai_yaml, + 'common.models.integration.PyfaiIntegrationConfig' + ), + _read_yaml( + corrections_yaml, + 'saxswaxs.models.CorrectionsConfig' + ), + _read_yaml( + fits_yaml, + 'saxswaxs.models.FitsConfig' + ), + ] + + +def setup(cfg): + """Run the CHAP setup pipeline to create the Zarr dataset structure. + + Generates map and pipeline config files from the spec scan, reads all + config files, runs :class:`SetupProcessor` to create the Zarr dataset + structure, and writes the result to disk. + + :param cfg: Configuration for the setup task. + :type cfg: SetupCfg + """ + cache_clear() + setup_configs(cfg) + logger.info('Reading') + data = read_configs( + cfg.detectors_yaml, cfg.map_yaml, cfg.pyfai_yaml, cfg.corrections_yaml, cfg.fits_yaml, + ) + logger.info('Processing') + zarr_tree = [ + PipelineData( + data=SetupProcessor.run( + data=data, + dataset_chunks=cfg.dataset_chunks, + raw_data=False, + logger=get_logger("SetupProcessor.run"), + ), + name='saxswaxs.processor.SetupProcessor.run', + ) + ] + logger.info('Writing') + ZarrWriter.run( + data=zarr_tree, + filename=str(cfg.data_zarr), + force_overwrite=True, + logger=get_logger("ZarrWriter.run"), + ) + + +def update(cfg): + """Run the CHAP update pipeline to process a scan's data into the Zarr dataset. + + Reads all config files, runs :class:`UpdateValuesProcessor` for the specified + scan and index slice, and writes the resulting values into the existing Zarr + dataset. + + :param cfg: Configuration for the update task. + :type cfg: UpdateCfg + """ + cache_clear() + logger.info('Reading') + data = read_configs( + cfg.detectors_yaml, cfg.map_yaml, cfg.pyfai_yaml, cfg.corrections_yaml, cfg.fits_yaml, + ) + logger.info('Processing') + values = [ + PipelineData( + data=UpdateValuesProcessor.run( + data=data, + filename=str(cfg.data_zarr), + spec_file=cfg.spec_file, + scan_number=cfg.scan_number, + idx_slice=dict( + start=cfg.idx_slice_start, + stop=cfg.idx_slice_stop, + step=cfg.idx_slice_step, + ), + raw_data=True, + ), + name='UpdateValuesProcessor.run', + logger=get_logger("UpdateValuesProcessor.run"), + ) + ] + logger.info('Writing') + ZarrValuesWriter.run( + data=values, + filename=str(cfg.data_zarr), + resize_axis=0, + idx_slice=dict( + start=cfg.idx_slice_start, + stop=cfg.idx_slice_stop, + step=cfg.idx_slice_step, + ), + force_overwrite=True, + logger=get_logger("ZarrValuesWriter.run") + ) + + +def convert(cfg): + """Run the CHAP convert pipeline to convert the Zarr dataset to NeXus format. + + Launches CHAP as a subprocess with the ``convert`` pipeline defined in + ``cfg.outputdir/pipeline.yaml``. Subprocess output is written to + ``cfg.outputdir/chap_convert.log``. + + :param cfg: Configuration for the convert task. + :type cfg: ConvertCfg + """ + logger.info("CHAP convert starting") + + logname = cfg.outputdir / "chap_convert.log" + with open(logname, "a") as logfile: + process = subprocess.Popen( + [ + "CHAP", + cfg.outputdir / "pipeline.yaml", + "-p", + "convert", + ], + stdout=logfile, + stderr=subprocess.STDOUT, + ) + process.wait() + logger.info(f"CHAP convert logging to {logname}") + + +def setup_configs(cfg): + """Write map config and CHAP pipeline config YAML files for a spec scan. + + Calls :func:`scan_to_map` to generate the map config from the spec scan, + then :func:`saxswaxs_to_chap` to generate the corresponding CHAP pipeline + config files in the output directory. + + :param cfg: Configuration containing spec file, scan number, counter names, + and output file paths. + :type cfg: SetupCfg + """ + map_config = [PipelineData( + data=SpecScanToMapConfigProcessor.run( + spec_file=cfg.spec_file, + scan_number=cfg.scan_number, + station="id3b", + experiment="SAXSWAXS", + dwell_time_actual_counter_name=cfg.dwell_time_actual_counter_name, + presample_intensity_counter_name=cfg.presample_intensity_counter_name, + postsample_intensity_counter_name=cfg.postsample_intensity_counter_name, + validate_data_present=False, + logger=get_logger("SpecScanToMapConfigProcessor.run"), + ), + )] + YAMLWriter.run( + data=map_config, + filename=str(cfg.map_yaml), + force_overwrite=True, + logger=get_logger("YAMLWriter.run") + ) + + logger.info( + f"saxswaxs_to_chap({cfg.map_yaml}, {cfg.tool_yamls}, {cfg.outputdir})" + ) + saxswaxs_to_chap( + str(cfg.map_yaml), [str(t_y) for t_y in cfg.tool_yamls], str(cfg.outputdir), + detector_filename=str(cfg.detectors_yaml), + pyfai_filename=str(cfg.pyfai_yaml), + correction_filename=str(cfg.corrections_yaml), + fits_filename=str(cfg.fits_yaml), + ) + + +class SaxswaxsCfg(BaseModel): + """Base configuration shared by all SAXS/WAXS processing tasks. + + Contains paths to the spec file, all four YAML config files (detector, + map, pyFAI integration, corrections), and the output Zarr dataset. + + :ivar spec_file: Path to the SPEC file containing the scan. + :vartype spec_file: Path + :ivar scan_number: Number of the scan within the SPEC file. + :vartype scan_number: int + :ivar detectors_yaml: Path to the detector config YAML file. + :vartype detectors_yaml: Path + :ivar map_yaml: Path to the map config YAML file. + :vartype map_yaml: Path + :ivar pyfai_yaml: Path to the pyFAI integration processor config YAML file. + :vartype pyfai_yaml: Path + :ivar corrections_yaml: Path to the corrections config YAML file. + :vartype corrections_yaml: Path + :ivar data_zarr: Path to the output Zarr dataset. + :vartype data_zarr: Path + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + spec_file: Path + scan_number: int + + detectors_yaml: Path + map_yaml: Path + pyfai_yaml: Path + corrections_yaml: Path + fits_yaml: Path + + data_zarr: Path + + +class SetupCfg(SaxswaxsCfg): + """Configuration for the setup task, which creates the Zarr dataset structure. + + Extends :class:`SaxswaxsCfg` with the output directory, tool config files, + SPEC counter names for intensity and dwell time, and dataset chunk sizes. + + :ivar outputdir: Directory for output CHAP config files and the Zarr dataset. + :vartype outputdir: Path + :ivar tool_yamls: List of tool config YAML file paths. + :vartype tool_yamls: list[Path] + :ivar dwell_time_actual_counter_name: SPEC counter column name for actual dwell times. + :vartype dwell_time_actual_counter_name: str + :ivar presample_intensity_counter_name: SPEC counter column name for presample intensity. + :vartype presample_intensity_counter_name: str + :ivar postsample_intensity_counter_name: SPEC counter column name for postsample + intensity, or ``None`` if not recorded. + :vartype postsample_intensity_counter_name: str or None + :ivar dataset_chunks: Chunk sizes for the Zarr dataset dimensions. + :vartype dataset_chunks: list[int] + """ + + outputdir: Path + + tool_yamls: list[Path] + + dwell_time_actual_counter_name: str + presample_intensity_counter_name: str + postsample_intensity_counter_name: Optional[str] = None + + dataset_chunks: list[int] + + +class UpdateCfg(SaxswaxsCfg): + """Configuration for the update task, which fills data into the Zarr dataset. + + Extends :class:`SaxswaxsCfg` with an index slice identifying which rows + of the dataset to process in this update. + + :ivar idx_slice_start: Start index of the row slice to process, defaults to ``0``. + :vartype idx_slice_start: int + :ivar idx_slice_stop: Stop index of the row slice to process, defaults to ``-1``. + :vartype idx_slice_stop: int + :ivar idx_slice_step: Step of the row slice to process, defaults to ``1``. + :vartype idx_slice_step: int + """ + + idx_slice_start: int = 0 + idx_slice_stop: int = -1 + idx_slice_step: int = 1 + + +class ConvertCfg(BaseModel): + """Configuration for the convert task, which converts the Zarr dataset to NeXus. + + Only requires the output directory containing the ``pipeline.yaml`` written + by the setup task. + + :ivar outputdir: Directory containing the CHAP ``pipeline.yaml`` and Zarr dataset. + :vartype outputdir: Path + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + outputdir: Path + + +class ConfigFilesCfg(BaseModel): + """Base configuration shared by tasks that read and write CHAP config files + in a common output directory. + + :ivar outputdir: Directory to which output config files will be written, and + against which relative filenames are resolved. + :vartype outputdir: Path + :ivar detector_filename: Path to the detector config YAML file. If relative, + resolved against ``outputdir``. Defaults to ``'detector_config.yaml'``. + :vartype detector_filename: str + :ivar pyfai_filename: Path to the pyFAI integration processor config YAML + file. If relative, resolved against ``outputdir``. Defaults to + ``'pyfai_integration_processor_config.yaml'``. + :vartype pyfai_filename: str + :ivar correction_filename: Path to the corrections config YAML file. If + relative, resolved against ``outputdir``. Defaults to + ``'corrections_config.yaml'``. + :vartype correction_filename: str + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + outputdir: Path + detector_filename: str = 'detector_config.yaml' + pyfai_filename: str = 'pyfai_integration_processor_config.yaml' + correction_filename: str = 'corrections_config.yaml' + + +class MakePipelineCfg(ConfigFilesCfg): + """Configuration for the make_pipeline task, which writes a ``pipeline.yaml`` + from pre-existing config files without requiring the old workflow library. + + Extends :class:`ConfigFilesCfg` with the map, fits, and pipeline filenames. + + :ivar map_filename: Path to the map config YAML file. If relative, resolved + against ``outputdir``. Defaults to ``'map_config.yaml'``. + :vartype map_filename: str + :ivar fits_filename: Path to the fits config YAML file. If relative, resolved + against ``outputdir``. Defaults to ``'fits_config.yaml'``. + :vartype fits_filename: str + :ivar pipeline_filename: Output filename for the pipeline YAML. Defaults to + ``'pipeline.yaml'``. + :vartype pipeline_filename: str + """ + + map_filename: str = 'map_config.yaml' + fits_filename: str = 'fits_config.yaml' + pipeline_filename: str = 'pipeline.yaml' + + +class ConvertConfigsCfg(ConfigFilesCfg): + """Configuration for the convert_configs task, which writes detector, + pyFAI integration, and corrections config YAML files from old-style + saxswaxs workflow tool config files. + + Extends :class:`ConfigFilesCfg` with the list of tool YAML files to convert. + + :ivar tool_yamls: List of tool config YAML file paths to convert. + :vartype tool_yamls: list[Path] + """ + + tool_yamls: list[Path] + + +def make_pipeline(cfg): + """Run the make_pipeline task to write a ``pipeline.yaml`` from pre-existing + config files. + + Calls :func:`CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline` with the paths + and filenames from ``cfg``. + + :param cfg: Configuration for the make_pipeline task. + :type cfg: MakePipelineCfg + """ + _make_pipeline( + str(cfg.outputdir), + map_filename=cfg.map_filename, + detector_filename=cfg.detector_filename, + pyfai_filename=cfg.pyfai_filename, + correction_filename=cfg.correction_filename, + fits_filename=cfg.fits_filename, + pipeline_filename=cfg.pipeline_filename, + ) + + +def convert_configs(cfg): + """Run the convert_configs task to write detector, pyFAI integration, and + corrections config YAML files from old-style saxswaxs workflow tool configs. + + Calls :func:`CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs` with the paths + and filenames from ``cfg``. + + :param cfg: Configuration for the convert_configs task. + :type cfg: ConvertConfigsCfg + """ + _convert_configs( + str(cfg.outputdir), + [str(t) for t in cfg.tool_yamls], + detector_filename=cfg.detector_filename, + pyfai_filename=cfg.pyfai_filename, + correction_filename=cfg.correction_filename, + ) diff --git a/CHAP/saxswaxs/server/logging_config.py b/CHAP/saxswaxs/server/logging_config.py new file mode 100644 index 00000000..ab045824 --- /dev/null +++ b/CHAP/saxswaxs/server/logging_config.py @@ -0,0 +1,179 @@ +"""Logging configuration""" + +import contextlib +import logging +import os +import sys +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path + + +_LOG_HANDLER = None +_STDOUT_WRAPPER = None +_STDERR_WRAPPER = None +_TASK_HANDLER = None + +def get_logger(name=__name__, log_level="DEBUG"): + global _STDOUT_WRAPPER + global _STDERR_WRAPPER + + logger = logging.getLogger(name) + logger.propagate = False + logger.setLevel(getattr(logging, log_level.upper())) + + handler = _get_log_handler() + + handlers = [handler] + if _TASK_HANDLER is not None: + handlers.append(_TASK_HANDLER) + logger.handlers = handlers + + # Redirect stdout/stderr once. + if _STDOUT_WRAPPER is None: + _STDOUT_WRAPPER = StreamToLogFile(handler) + _STDERR_WRAPPER = StreamToLogFile(handler) + + sys.stdout = _STDOUT_WRAPPER + sys.stderr = _STDERR_WRAPPER + + return logger + + +@contextlib.contextmanager +def task_log_context(log_path): + """Context manager that tees log output to a per-task append-only file. + + While active, all loggers created via :func:`get_logger` write to + *log_path* in addition to the shared rotating log file, using the same + formatter. Raw stdout/stderr writes are also teed to the same file + without any additional formatting. The file is always opened in append + mode so successive calls accumulate rather than overwrite. + """ + global _TASK_HANDLER + + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + + task_handler = logging.FileHandler(str(log_path), mode='a', encoding='utf-8') + task_handler.setFormatter(_get_log_handler().formatter) + + prev_task_handler = _TASK_HANDLER + _TASK_HANDLER = task_handler + + # Add to all loggers that were created by get_logger (propagate=False). + pre_existing = [ + lgr for lgr in logging.Logger.manager.loggerDict.values() + if isinstance(lgr, logging.Logger) and not lgr.propagate + ] + for lgr in pre_existing: + lgr.addHandler(task_handler) + + try: + yield + finally: + _TASK_HANDLER = prev_task_handler + + # Remove from every logger that received this handler (pre-existing + # and any created inside the context via get_logger). + for lgr in logging.Logger.manager.loggerDict.values(): + if isinstance(lgr, logging.Logger) and task_handler in lgr.handlers: + lgr.handlers = [h for h in lgr.handlers if h is not task_handler] + + task_handler.close() + + +class StreamToLogFile: + """Write stdout/stderr directly to the rotating log file.""" + + def __init__(self, handler): + self.handler = handler + + def write(self, message): + if not message: + return + + # Click and some other libraries may write bytes rather than str. + if isinstance(message, bytes): + message = message.decode("utf-8", errors="replace") + + self.handler.acquire() + try: + # Rotate if necessary. + record = logging.LogRecord( + name="stream", + level=logging.INFO, + pathname="", + lineno=0, + msg="", + args=(), + exc_info=None, + ) + + if self.handler.shouldRollover(record): + self.handler.doRollover() + + self.handler.stream.write(message) + self.handler.stream.flush() + + finally: + self.handler.release() + + # Tee raw output to the active task log file. + if _TASK_HANDLER is not None: + _TASK_HANDLER.acquire() + try: + _TASK_HANDLER.stream.write(message) + _TASK_HANDLER.stream.flush() + finally: + _TASK_HANDLER.release() + + def flush(self): + self.handler.acquire() + try: + if self.handler.stream is not None: + self.handler.stream.flush() + finally: + self.handler.release() + + def isatty(self): + return False + + @property + def encoding(self): + return self.handler.encoding + + def __getattr__(self, name): + # Let libraries such as Click access other normal stream + # attributes/methods if necessary. + return getattr(sys.__stdout__, name) + + +def _get_log_handler(): + global _LOG_HANDLER + + if _LOG_HANDLER is None: + log_file = os.path.join( + os.path.dirname(__file__), + "saxswaxs-server", + ) + + _LOG_HANDLER = TimedRotatingFileHandler( + log_file, + when="midnight", + interval=1, + backupCount=30, + encoding="utf-8", + ) + + _LOG_HANDLER.suffix = "%Y-%m-%d.log" + + _LOG_HANDLER.setFormatter( + logging.Formatter( + "{asctime}: {name:20} (L{lineno}): " + "{levelname}: {message}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{", + ) + ) + + return _LOG_HANDLER diff --git a/CHAP/saxswaxs/server/saxswaxs_to_chap.py b/CHAP/saxswaxs/server/saxswaxs_to_chap.py new file mode 100755 index 00000000..6f8e20fd --- /dev/null +++ b/CHAP/saxswaxs/server/saxswaxs_to_chap.py @@ -0,0 +1,386 @@ + +"""Script to act as a drop-in replacement for the old saxswaxsworkflow +CLI tool. +Instead of running the old workflow code, a new CHAP pipeline will be +constructed & run from the given tool & map configs. +""" + +import argparse +import logging +import os +import sys + +import yaml + + +class VerboseSafeDumper(yaml.SafeDumper): + """YAML SafeDumper subclass that writes all nodes explicitly without aliases.""" + + def ignore_aliases(self, data): + """Return True to prevent YAML from using anchor/alias references.""" + return True + + +def saxswaxs_to_chap( + map_config_file, tool_config_files, outputdir, + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml', + fits_filename='fits_config.yaml', + pipeline_filename='pipeline.yaml'): + """Build CHAP pipeline config files from old-style saxswaxs workflow configs. + + Converts old-style tool config files to CHAP-format detector, pyFAI + integration, and corrections config files via :func:`convert_configs`, + then composes the CHAP pipeline config YAML via :func:`make_pipeline`. + + :param map_config_file: Path to the map config YAML file. + :type map_config_file: str + :param tool_config_files: List of tool config YAML file paths. + :type tool_config_files: list[str] + :param outputdir: Directory in which to write the output CHAP config files. + :type outputdir: str + :param detector_filename: Output filename for the detector config YAML, + defaults to ``'detector_config.yaml'``. + :type detector_filename: str, optional + :param pyfai_filename: Output filename for the pyFAI integration processor + config YAML, defaults to ``'pyfai_integration_processor_config.yaml'``. + :type pyfai_filename: str, optional + :param correction_filename: Output filename for the corrections config YAML, + defaults to ``'corrections_config.yaml'``. + :type correction_filename: str, optional + :param fits_filename: Output filename for the fits config YAML, + defaults to ``'fits_config.yaml'``. + :type fits_filename: str, optional + :param pipeline_filename: Output filename for the CHAP pipeline config YAML, + defaults to ``'pipeline.yaml'``. + :type pipeline_filename: str, optional + """ + convert_configs( + outputdir, tool_config_files, + detector_filename=detector_filename, + pyfai_filename=pyfai_filename, + correction_filename=correction_filename, + ) + make_pipeline( + outputdir, + map_filename=map_config_file, + detector_filename=detector_filename, + pyfai_filename=pyfai_filename, + correction_filename=correction_filename, + fits_filename=fits_filename, + pipeline_filename=pipeline_filename, + ) + + +def make_pipeline(outputdir, + map_filename='map_config.yaml', + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml', + fits_filename='fits_config.yaml', + pipeline_filename='pipeline.yaml', + ): + """Compose a pipeline file for a complete saxswaxs workflow based + on the config files provided, and asssuming they all already + exist. Sort of a lightweight version of wf_to_chap.""" + from CHAP.common.models.map import MapConfig + from CHAP.common.reader import YAMLReader + + outputdir = os.path.abspath(outputdir) + + if not os.path.isabs(map_filename): + map_filename = os.path.join(outputdir, map_filename) + if not os.path.isabs(detector_filename): + detector_filename = os.path.join(outputdir, detector_filename) + if not os.path.isabs(pyfai_filename): + pyfai_filename = os.path.join(outputdir, pyfai_filename) + if not os.path.isabs(correction_filename): + correction_filename = os.path.join(outputdir, correction_filename) + if not os.path.isabs(fits_filename): + fits_filename = os.path.join(outputdir, fits_filename) + + map_config = MapConfig(**YAMLReader.run( + filename=map_filename, schema='common.models.map.MapConfig')) + + zarr_filename = os.path.join(outputdir, f'{map_config.title}.zarr') + nxs_filename = os.path.abspath(os.path.join(outputdir, '..', f'{map_config.title}.nxs')) + + readers = [ + { + 'common.reader.YAMLReader': { + 'filename': detector_filename, + 'schema': 'common.models.map.DetectorConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': map_filename, + 'schema': 'common.models.map.MapConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': pyfai_filename, + 'schema': 'common.models.integration.PyfaiIntegrationConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': correction_filename, + 'schema': 'saxswaxs.models.CorrectionsConfig' + } + }, + { + 'common.reader.YAMLReader': { + 'filename': fits_filename, + 'schema': 'saxswaxs.models.FitsConfig' + } + }, + ] + + update_pipelines = {} + npts = 0 + nrows = 0 + row_npts = 1 + for scans in map_config.spec_scans: + for scan_number in scans.scan_numbers: + sp = scans.get_scanparser(scan_number) + _npts = int(sp.spec_scan_npts) + if len(sp.spec_scan_shape) > 1: + _nrows = sp.spec_scan_shape[1] + row_npts = sp.spec_scan_shape[0] + else: + _nrows = 1 + row_npts = _npts + for i in range(_nrows): + idx_slice = { + 'start': npts + (i * row_npts), + 'stop': npts + (i * row_npts) + row_npts, + 'step': 1, + } + update_pipelines[f'update_{nrows + i}'] = [ + *readers, + { + 'saxswaxs.processor.UpdateValuesProcessor': { + 'raw_data': False, + 'filename': zarr_filename, + 'spec_file': scans.spec_file, + 'scan_number': scan_number, + 'idx_slice': idx_slice, + } + }, + { + 'common.ZarrValuesWriter': { + 'filename': zarr_filename, + 'resize_axis': 0, + 'idx_slice': idx_slice, + 'force_overwrite': True, + } + } + ] + nrows += _nrows + npts += _npts + + chap_config = { + 'config': { + 'root': outputdir, + 'log_level': 'debug', + }, + 'setup': [ + *readers, + { + 'saxswaxs.processor.SetupProcessor': { + 'raw_data': False, + 'dataset_chunks': [row_npts], + } + }, + { + 'common.writer.ZarrWriter': { + 'filename': zarr_filename, + 'force_overwrite': True, + } + } + ], + **update_pipelines, + 'convert': [ + { + 'common.processor.ZarrToNexusProcessor': { + 'zarr_filename': zarr_filename, + 'nexus_filename': nxs_filename, + } + } + ] + } + pipeline_path = os.path.join(outputdir, pipeline_filename) + print(f'Writing to {pipeline_path}') + os.makedirs(os.path.dirname(pipeline_path), exist_ok=True) + with open(pipeline_path, 'w') as outf: + yaml.dump(chap_config, outf, sort_keys=False, Dumper=VerboseSafeDumper) + return chap_config + + +def convert_configs(outputdir, tool_config_files, + detector_filename='detector_config.yaml', + pyfai_filename='pyfai_integration_processor_config.yaml', + correction_filename='corrections_config.yaml'): + """Write the new CHAP.saxswaxs-formatted pyfai and corrections + config files based on the old workflow tool files provided. Should + be independent from any map configuration or + saxswaxsworkflow.workflow.Workflow object -- use the dictionaries + in the tool_config_files only.""" + + outputdir = os.path.abspath(outputdir) + + pyfai_integration_processor_config = { + 'azimuthal_integrators': [], + 'integrations': [], + } + detectors = [] + corrections = [] + for tool_config_file in tool_config_files: + with open(tool_config_file) as f: + t = yaml.safe_load(f) + if t.get('tool_type') == 'integration': + # Build azimuthal_integrators entries + for det in t.get('detectors', []): + prefix = det['prefix'] + already_added = any( + ai['id'] == prefix + and ai['poni_file'] == str(det['poni_file']) + and ai['mask_file'] == str(det['mask_file']) + for ai in pyfai_integration_processor_config['azimuthal_integrators'] + ) + if not already_added: + pyfai_integration_processor_config['azimuthal_integrators'].append( + { + 'id': prefix, + 'poni_file': str(det['poni_file']), + 'mask_file': str(det['mask_file']), + } + ) + # Build detector_config entries + visited = any(prefix == d['id'] for d in detectors) + if not visited: + placeholder_shape = (1, 1) + if prefix == 'PIL5': + shape = [619, 487] + elif prefix in ('PIL9', 'PIL11'): + shape = [407, 487] + else: + print( + f'WARNING: unrecognized detector prefix {prefix}; ' + + f'using placeholder shape {placeholder_shape}' + ) + shape = list(placeholder_shape) + detectors.append({'id': prefix, 'shape': shape}) + + # Build integrations entry + integration_config = {'name': t['title']} + if t.get('integration_type') == 'radial': + integration_config['integration_method'] = 'integrate_radial' + integration_config['integration_params'] = { + 'ais': [det['prefix'] for det in t.get('detectors', [])], + 'npt': t['azimuthal_npt'], + 'npt_rad': t['radial_npt'], + 'radial_range': [t['radial_min'], t['radial_max']], + 'azimuth_range': [t['azimuthal_min'], t['azimuthal_max']], + 'unit': t['azimuthal_units'], + 'radial_unit': t['radial_units'], + 'method': 'bbox_csr_cython', + } + else: + integration_config['multi_geometry'] = { + 'ais': [det['prefix'] for det in t.get('detectors', [])], + 'unit': t['radial_units'], + 'radial_range': [t['radial_min'], t['radial_max']], + 'azimuth_range': [t['azimuthal_min'], t['azimuthal_max']], + } + if t.get('integration_type') == 'azimuthal': + integration_config['integration_method'] = 'integrate1d' + integration_config['integration_params'] = { + 'npt': t['radial_npt'], + 'method': 'bbox_csr_cython', + } + elif t.get('integration_type') == 'cake': + integration_config['integration_method'] = 'integrate2d' + integration_config['integration_params'] = { + 'npt_rad': t['radial_npt'], + 'npt_azim': t['azimuthal_npt'], + 'method': 'bbox_csr_cython', + } + pyfai_integration_processor_config['integrations'].append(integration_config) + else: + # It's a corrections tool — include all fields except tool_type + correction = {k: v for k, v in t.items() + if k not in ('tool_type', 'validate_data_present')} + corrections.append(correction) + + # Write detector config .yaml + detector_config = {'detectors': detectors} + if not os.path.isabs(detector_filename): + detector_filename = os.path.join(outputdir, detector_filename) + print(f'Writing to {detector_filename}') + os.makedirs(os.path.dirname(detector_filename), exist_ok=True) + with open(detector_filename, 'w') as outf: + yaml.dump(detector_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write pyfai config .yaml + if not os.path.isabs(pyfai_filename): + pyfai_filename = os.path.join(outputdir, pyfai_filename) + print(f'Writing to {pyfai_filename}') + os.makedirs(os.path.dirname(pyfai_filename), exist_ok=True) + with open(pyfai_filename, 'w') as outf: + yaml.dump(pyfai_integration_processor_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + # Write corrections config .yaml + correction_config = {'corrections': corrections} + if not os.path.isabs(correction_filename): + correction_filename = os.path.join(outputdir, correction_filename) + print(f'Writing to {correction_filename}') + os.makedirs(os.path.dirname(correction_filename), exist_ok=True) + with open(correction_filename, 'w') as outf: + yaml.dump(correction_config, outf, sort_keys=False, + Dumper=VerboseSafeDumper) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', '--map_config_file', required=True, metavar='map.yaml', + help='''Map configuration .yaml file to use for constructing + the workflow.''' + ) + parser.add_argument( + '-t', '--tool_config_files', required=True, action='extend', + nargs='+', metavar='tool.yaml', help='''List of .yaml files + containing tool configurations to apply to the map + configuration provided.''' + ) + parser.add_argument( + '-f', '--force_overwrite', action='store_true', help='''Use + this flag to overwrite the output file if it already + exists.''' + ) + parser.add_argument( + '-i', '--inputdir', default='.', help='''Directory containing + all input files''' + ) + parser.add_argument( + '-o', '--outputdir', default='.', help='''Directory in which + to place output file.''' + ) + parser.add_argument( + '-l', '--log', choices=logging._nameToLevel.keys(), + default='INFO', help='''Specify a preferred logging level.''' + ) + args = parser.parse_args(sys.argv[1:]) + + map_config_file = os.path.join(args.inputdir, args.map_config_file) + tool_config_files = [ + os.path.join(args.inputdir, tool_config_file) + for tool_config_file in args.tool_config_files + ] + saxswaxs_to_chap(map_config_file, tool_config_files, args.outputdir) diff --git a/CHAP/saxswaxs/server/server.py b/CHAP/saxswaxs/server/server.py new file mode 100644 index 00000000..d099e000 --- /dev/null +++ b/CHAP/saxswaxs/server/server.py @@ -0,0 +1,195 @@ +"""Server for handling HTTP requests.""" + +from flask import Flask, jsonify, request +import logging +import time +from traceback import print_exc + +from CHAP.saxswaxs.server.logging_config import get_logger +from CHAP.saxswaxs.server.task_queue import put +from CHAP.saxswaxs.server.chap import ( + setup, update, convert, make_pipeline, convert_configs, + SetupCfg, UpdateCfg, ConvertCfg, MakePipelineCfg, ConvertConfigsCfg, +) + +app = Flask(__name__) +app.logger = get_logger('server') + +# Logging middleware +@app.before_request +def start_timer(): + """Record the request start time for duration logging.""" + request.start_time = time.time() + +@app.after_request +def log_request(response): + """Log the HTTP method, path, status code, and duration for the request.""" + now = time.time() + duration = round(now - request.start_time, 4) + app.logger.info( + f"{request.method} {request.path} - {response.status_code} ({duration}s)" + ) + return response + +# API endpoints +@app.route('/setup', methods=['POST']) +def setup_handler(): + """Handle POST /setup — parse JSON body and queue a setup task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = SetupCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + setup_args = (cfg,) + setup_kwargs = {} + put(setup, setup_args, setup_kwargs) + return jsonify({'status': 'queued'}), 202 + +@app.route('/update', methods=['POST']) +def update_handler(): + """Handle POST /update — parse JSON body and queue an update task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = UpdateCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + update_args = (cfg,) + update_kwargs = {} + put(update, update_args, update_kwargs) + return jsonify({'status': 'queued'}), 202 + +@app.route('/convert', methods=['POST']) +def convert_handler(): + """Handle POST /convert — parse JSON body and queue a convert task.""" + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = ConvertCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'sattus': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + convert_args = (cfg,) + convert_kwargs = {} + put(convert, convert_args, convert_kwargs) + return jsonify({'status': 'queued'}), 202 + + +@app.route('/make_pipeline', methods=['POST']) +def make_pipeline_handler(): + """Handle POST /make_pipeline — parse JSON body and queue a make_pipeline task. + + Constructs a :class:`~CHAP.saxswaxs.server.chap.MakePipelineCfg` from the request + body and queues :func:`~CHAP.saxswaxs.server.chap.make_pipeline` to write a + ``pipeline.yaml`` from the pre-existing config files in ``outputdir``. + """ + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = MakePipelineCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'status': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + put(make_pipeline, (cfg,), {}) + return jsonify({'status': 'queued'}), 202 + + +@app.route('/convert_configs', methods=['POST']) +def convert_configs_handler(): + """Handle POST /convert_configs — parse JSON body and queue a convert_configs task. + + Constructs a :class:`~CHAP.saxswaxs.server.chap.ConvertConfigsCfg` from the request + body and queues :func:`~CHAP.saxswaxs.server.chap.convert_configs` to write detector, + pyFAI integration, and corrections config YAML files from the provided + tool config files. + """ + body = request.get_json(force=True, silent=True) + if body is None: + return jsonify( + { + 'status': 'error', + 'reason': 'no data in body of request' + } + ), 400 + try: + cfg = ConvertConfigsCfg(**body) + except Exception as exc: + print_exc() + return ( + jsonify( + { + 'status': 'error', + 'reason': repr(exc), + } + ), + 400 + ) + put(convert_configs, (cfg,), {}) + return jsonify({'status': 'queued'}), 202 + + +def run(): + """Start the Flask development server.""" + app.run(debug=False) + +if __name__ == '__main__': + run() diff --git a/CHAP/saxswaxs/server/task_queue.py b/CHAP/saxswaxs/server/task_queue.py new file mode 100644 index 00000000..6bcf23fa --- /dev/null +++ b/CHAP/saxswaxs/server/task_queue.py @@ -0,0 +1,59 @@ +"""Queueing system for data processing tasks""" + +import queue +import threading +from traceback import print_exc +from time import sleep, time + +from CHAP.saxswaxs.server.logging_config import ( + get_logger, + task_log_context, +) + +logger = get_logger('task_queue') + +_task_queue = queue.Queue() + +def _worker(): + """Continuously dequeue and execute tasks from the task queue. + + Runs in a background daemon thread. On task failure the error is logged + and the task is marked done. + """ + while True: + task, args, kwargs, log_path = _task_queue.get() + logger.info( + f'Starting task: {str(task)}, args: {args}, kwargs: {kwargs}') + t0 = time() + success = False + while not success: + with task_log_context(log_path): + # Handle race conditions from missing data + try: + task(*args, **kwargs) + success = True + except Exception as exc: + logger.error(f'Task failed: {exc}') + print_exc() + sleep(5) + _task_queue.task_done() + tf = time() + logger.info(f'Task done. ({tf-t0:.5f} seconds)') + +def put(task, args, kwargs): + """Enqueue a task for execution by the background worker thread. + + :param task: Callable to execute. + :param args: Positional arguments to pass to ``task``. + :type args: tuple + :param kwargs: Keyword arguments to pass to ``task``. + :type kwargs: dict + """ + cfg = args[0] + if hasattr(cfg, 'outputdir'): + log_path = cfg.outputdir / f"chap_{task.__name__}.log" + else: + log_path = cfg.data_zarr.parent / f"chap_{task.__name__}.log" + _task_queue.put((task, args, kwargs, log_path)) + +threading.Thread(target=_worker, daemon=True).start() diff --git a/CHAP/test/data/saxswaxs/1/convert.json b/CHAP/test/data/saxswaxs/1/convert.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/convert.json @@ -0,0 +1 @@ +{} diff --git a/CHAP/test/data/saxswaxs/1/setup.json b/CHAP/test/data/saxswaxs/1/setup.json new file mode 100644 index 00000000..f9c44a56 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/setup.json @@ -0,0 +1,17 @@ +{ + "spec_file": "/data/spec/scan.spec", + "scan_number": 1, + "detectors_yaml": "/data/config/detectors.yaml", + "map_yaml": "/data/config/map.yaml", + "pyfai_yaml": "/data/config/pyfai.yaml", + "corrections_yaml": "/data/config/corrections.yaml", + "fits_yaml": "/data/config/fits.yaml", + "tool_yamls": [ + "/data/tools/tool1.yaml", + "/data/tools/tool2.yaml" + ], + "dwell_time_actual_counter_name": "dwell_counter", + "presample_intensity_counter_name": "presample_counter", + "postsample_intensity_counter_name": "postsample_counter", + "dataset_chunks": [10] +} diff --git a/CHAP/test/data/saxswaxs/1/update.json b/CHAP/test/data/saxswaxs/1/update.json new file mode 100644 index 00000000..74860668 --- /dev/null +++ b/CHAP/test/data/saxswaxs/1/update.json @@ -0,0 +1,12 @@ +{ + "spec_file": "/data/spec/scan.spec", + "scan_number": 1, + "detectors_yaml": "/data/config/detectors.yaml", + "map_yaml": "/data/config/map.yaml", + "pyfai_yaml": "/data/config/pyfai.yaml", + "corrections_yaml": "/data/config/corrections.yaml", + "fits_yaml": "/data/config/fits.yaml", + "idx_slice_start": 10, + "idx_slice_stop": 20, + "idx_slice_step": 1 +} diff --git a/CHAP/test/data/saxswaxs/2/convert.json b/CHAP/test/data/saxswaxs/2/convert.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/convert.json @@ -0,0 +1 @@ +{} diff --git a/CHAP/test/data/saxswaxs/2/setup.json b/CHAP/test/data/saxswaxs/2/setup.json new file mode 100644 index 00000000..396270e0 --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/setup.json @@ -0,0 +1,17 @@ +{ + "spec_file": "/experiment/spec/run2.spec", + "scan_number": 5, + "detectors_yaml": "/experiment/config/det.yaml", + "map_yaml": "/experiment/config/mapping.yaml", + "pyfai_yaml": "/experiment/config/azimuthal.yaml", + "corrections_yaml": "/experiment/config/corr.yaml", + "fits_yaml": "/experiment/config/fit_params.yaml", + "tool_yamls": [ + "/experiment/tools/a.yaml", + "/experiment/tools/b.yaml", + "/experiment/tools/c.yaml" + ], + "dwell_time_actual_counter_name": "timer", + "presample_intensity_counter_name": "mon", + "dataset_chunks": [50, 50] +} diff --git a/CHAP/test/data/saxswaxs/2/update.json b/CHAP/test/data/saxswaxs/2/update.json new file mode 100644 index 00000000..27f46670 --- /dev/null +++ b/CHAP/test/data/saxswaxs/2/update.json @@ -0,0 +1,9 @@ +{ + "spec_file": "/experiment/spec/run2.spec", + "scan_number": 5, + "detectors_yaml": "/experiment/config/det.yaml", + "map_yaml": "/experiment/config/mapping.yaml", + "pyfai_yaml": "/experiment/config/azimuthal.yaml", + "corrections_yaml": "/experiment/config/corr.yaml", + "fits_yaml": "/experiment/config/fit_params.yaml" +} diff --git a/CHAP/test/saxswaxs/server/__init__.py b/CHAP/test/saxswaxs/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/CHAP/test/saxswaxs/server/conftest.py b/CHAP/test/saxswaxs/server/conftest.py new file mode 100644 index 00000000..fa1f0bc6 --- /dev/null +++ b/CHAP/test/saxswaxs/server/conftest.py @@ -0,0 +1,48 @@ +"""Shared fixtures for CHAP.saxswaxs.server tests.""" + +import json +from pathlib import Path + +import pytest + +_DATA_ROOT = Path(__file__).parent.parent.parent / "data" / "saxswaxs" + +# Table of fixture directories — add a new entry here to add a new test case set. +FIXTURE_DIRS = [ + _DATA_ROOT / "1", + _DATA_ROOT / "2", +] + + +@pytest.fixture(params=FIXTURE_DIRS, ids=[d.name for d in FIXTURE_DIRS]) +def fixture_dir(request): + return request.param + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +@pytest.fixture +def setup_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import SetupCfg + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + return SetupCfg(**data) + + +@pytest.fixture +def update_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import UpdateCfg + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + return UpdateCfg(**data) + + +@pytest.fixture +def convert_cfg(fixture_dir, tmp_path): + from CHAP.saxswaxs.server.chap import ConvertCfg + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + return ConvertCfg(**data) diff --git a/CHAP/test/saxswaxs/server/test_chap.py b/CHAP/test/saxswaxs/server/test_chap.py new file mode 100644 index 00000000..5018781e --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_chap.py @@ -0,0 +1,206 @@ +"""Unit tests for CHAP.saxswaxs.server.chap with all CHAP processors mocked.""" + +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +import pytest + +# Fixtures setup_cfg, update_cfg, convert_cfg come from conftest.py + +CHAP_MOCKS = { + "CHAP.saxswaxs.server.chap.YAMLReader": MagicMock, + "CHAP.saxswaxs.server.chap.ZarrWriter": MagicMock, + "CHAP.saxswaxs.server.chap.ZarrValuesWriter": MagicMock, + "CHAP.saxswaxs.server.chap.SetupProcessor": MagicMock, + "CHAP.saxswaxs.server.chap.UpdateValuesProcessor": MagicMock, + "CHAP.saxswaxs.server.chap.PipelineData": MagicMock, + "CHAP.saxswaxs.server.chap.setup_configs": MagicMock, + "CHAP.saxswaxs.server.chap.read_configs": MagicMock, + "CHAP.saxswaxs.server.chap.cache_clear": MagicMock, +} + + +def _patch_all(): + """Return a single patch.multiple context manager covering all CHAP seams.""" + return patch.multiple("CHAP.saxswaxs.server.chap", **{k.split(".")[-1]: MagicMock() for k in CHAP_MOCKS}) + + +class TestCacheClear: + def test_cache_clear_called_in_setup(self, setup_cfg): + with patch("CHAP.saxswaxs.server.chap.cache_clear") as mock_cc, \ + patch("CHAP.saxswaxs.server.chap.setup_configs"), \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=[MagicMock()] * 5), \ + patch("CHAP.saxswaxs.server.chap.SetupProcessor"), \ + patch("CHAP.saxswaxs.server.chap.ZarrWriter"), \ + patch("CHAP.saxswaxs.server.chap.PipelineData"): + from CHAP.saxswaxs.server.chap import setup + setup(setup_cfg) + mock_cc.assert_called_once() + + def test_cache_clear_called_in_update(self, update_cfg): + with patch("CHAP.saxswaxs.server.chap.cache_clear") as mock_cc, \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=[MagicMock()] * 5), \ + patch("CHAP.saxswaxs.server.chap.UpdateValuesProcessor"), \ + patch("CHAP.saxswaxs.server.chap.ZarrValuesWriter"), \ + patch("CHAP.saxswaxs.server.chap.PipelineData"): + from CHAP.saxswaxs.server.chap import update + update(update_cfg) + mock_cc.assert_called_once() + + +class TestSetup: + def _run(self, cfg): + mock_data = [MagicMock()] * 5 + mock_zarr_tree = MagicMock() + mock_pipeline_data = MagicMock(return_value=mock_zarr_tree) + mock_setup_proc = MagicMock() + mock_setup_proc.run = MagicMock(return_value=MagicMock()) + mock_zarr_writer = MagicMock() + + with patch("CHAP.saxswaxs.server.chap.cache_clear"), \ + patch("CHAP.saxswaxs.server.chap.setup_configs") as mock_sc, \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=mock_data) as mock_rc, \ + patch("CHAP.saxswaxs.server.chap.SetupProcessor", mock_setup_proc), \ + patch("CHAP.saxswaxs.server.chap.ZarrWriter", mock_zarr_writer), \ + patch("CHAP.saxswaxs.server.chap.PipelineData", mock_pipeline_data): + from CHAP.saxswaxs.server.chap import setup + setup(cfg) + return mock_sc, mock_rc, mock_setup_proc, mock_zarr_writer, mock_pipeline_data + + def test_setup_configs_called(self, setup_cfg): + mock_sc, *_ = self._run(setup_cfg) + mock_sc.assert_called_once_with(setup_cfg) + + def test_read_configs_called_with_correct_paths(self, setup_cfg): + _, mock_rc, *_ = self._run(setup_cfg) + mock_rc.assert_called_once_with( + setup_cfg.detectors_yaml, + setup_cfg.map_yaml, + setup_cfg.pyfai_yaml, + setup_cfg.corrections_yaml, + setup_cfg.fits_yaml, + ) + + def test_setup_processor_called_with_dataset_chunks(self, setup_cfg): + _, _, mock_setup_proc, *_ = self._run(setup_cfg) + mock_setup_proc.run.assert_called_once() + _, call_kwargs = mock_setup_proc.run.call_args + assert call_kwargs["dataset_chunks"] == setup_cfg.dataset_chunks + assert call_kwargs["raw_data"] is False + + def test_zarr_writer_called_with_correct_args(self, setup_cfg): + _, _, _, mock_zarr_writer, _ = self._run(setup_cfg) + mock_zarr_writer.run.assert_called_once() + _, call_kwargs = mock_zarr_writer.run.call_args + assert call_kwargs["filename"] == str(setup_cfg.data_zarr) + assert call_kwargs["force_overwrite"] is True + + +class TestUpdate: + def _run(self, cfg): + mock_data = [MagicMock()] * 5 + mock_update_proc = MagicMock() + mock_update_proc.run = MagicMock(return_value=MagicMock()) + mock_zarr_values_writer = MagicMock() + mock_pipeline_data = MagicMock() + + with patch("CHAP.saxswaxs.server.chap.cache_clear"), \ + patch("CHAP.saxswaxs.server.chap.read_configs", return_value=mock_data), \ + patch("CHAP.saxswaxs.server.chap.UpdateValuesProcessor", mock_update_proc), \ + patch("CHAP.saxswaxs.server.chap.ZarrValuesWriter", mock_zarr_values_writer), \ + patch("CHAP.saxswaxs.server.chap.PipelineData", mock_pipeline_data): + from CHAP.saxswaxs.server.chap import update + update(cfg) + return mock_update_proc, mock_zarr_values_writer + + def test_update_processor_called_with_correct_slice(self, update_cfg): + mock_proc, _ = self._run(update_cfg) + mock_proc.run.assert_called_once() + _, call_kwargs = mock_proc.run.call_args + assert call_kwargs["idx_slice"] == { + "start": update_cfg.idx_slice_start, + "stop": update_cfg.idx_slice_stop, + "step": update_cfg.idx_slice_step, + } + + def test_update_processor_called_with_spec_and_zarr(self, update_cfg): + mock_proc, _ = self._run(update_cfg) + _, call_kwargs = mock_proc.run.call_args + assert call_kwargs["spec_file"] == update_cfg.spec_file + assert call_kwargs["scan_number"] == update_cfg.scan_number + assert call_kwargs["filename"] == str(update_cfg.data_zarr) + assert call_kwargs["raw_data"] is True + + def test_zarr_values_writer_called_with_correct_args(self, update_cfg): + _, mock_writer = self._run(update_cfg) + mock_writer.run.assert_called_once() + _, call_kwargs = mock_writer.run.call_args + assert call_kwargs["filename"] == str(update_cfg.data_zarr) + assert call_kwargs["resize_axis"] == 0 + assert call_kwargs["idx_slice"] == { + "start": update_cfg.idx_slice_start, + "stop": update_cfg.idx_slice_stop, + "step": update_cfg.idx_slice_step, + } + assert call_kwargs["force_overwrite"] is True + + +class TestConvert: + def _run(self, cfg): + mock_process = MagicMock() + mock_popen = MagicMock(return_value=mock_process) + + with patch("CHAP.saxswaxs.server.chap.subprocess.Popen", mock_popen): + from CHAP.saxswaxs.server.chap import convert + convert(cfg) + return mock_popen, mock_process + + def test_popen_called_with_chap_convert(self, convert_cfg): + mock_popen, _ = self._run(convert_cfg) + mock_popen.assert_called_once() + args, kwargs = mock_popen.call_args + cmd = args[0] + assert cmd[0] == "CHAP" + assert cmd[1] == convert_cfg.outputdir / "pipeline.yaml" + assert cmd[2] == "-p" + assert cmd[3] == "convert" + + def test_popen_stdout_is_logfile(self, convert_cfg, tmp_path): + import subprocess as _subprocess + mock_process = MagicMock() + + captured = {} + + def fake_popen(cmd, stdout, stderr): + captured["stdout"] = stdout + captured["stderr"] = stderr + return mock_process + + with patch("CHAP.saxswaxs.server.chap.subprocess.Popen", fake_popen): + from CHAP.saxswaxs.server.chap import convert + convert(convert_cfg) + + assert hasattr(captured["stdout"], "write"), "stdout should be a file object" + assert captured["stderr"] == _subprocess.STDOUT + + def test_logfile_path(self, convert_cfg): + mock_process = MagicMock() + opened_paths = [] + + original_open = open + + def fake_open(path, mode="r", *args, **kwargs): + if mode == "w": + opened_paths.append(Path(path)) + return original_open(path, mode, *args, **kwargs) + + with patch("builtins.open", fake_open), \ + patch("CHAP.saxswaxs.server.chap.subprocess.Popen", return_value=mock_process): + from CHAP.saxswaxs.server.chap import convert + convert(convert_cfg) + + assert any(p == convert_cfg.outputdir / "chap_convert.log" for p in opened_paths) + + def test_process_wait_called(self, convert_cfg): + _, mock_process = self._run(convert_cfg) + mock_process.wait.assert_called_once() diff --git a/CHAP/test/saxswaxs/server/test_models.py b/CHAP/test/saxswaxs/server/test_models.py new file mode 100644 index 00000000..b3abe1a1 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_models.py @@ -0,0 +1,105 @@ +"""Unit tests for Pydantic config models in CHAP.saxswaxs.server.chap.""" + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from CHAP.saxswaxs.server.chap import ConvertCfg, SetupCfg, UpdateCfg + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +class TestSetupCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = SetupCfg(**data) + assert cfg.scan_number == data["scan_number"] + assert cfg.dwell_time_actual_counter_name == data["dwell_time_actual_counter_name"] + assert cfg.presample_intensity_counter_name == data["presample_intensity_counter_name"] + assert isinstance(cfg.spec_file, Path) + assert isinstance(cfg.detectors_yaml, Path) + assert isinstance(cfg.map_yaml, Path) + assert isinstance(cfg.pyfai_yaml, Path) + assert isinstance(cfg.corrections_yaml, Path) + assert isinstance(cfg.fits_yaml, Path) + assert isinstance(cfg.data_zarr, Path) + assert isinstance(cfg.outputdir, Path) + assert len(cfg.tool_yamls) == len(data["tool_yamls"]) + assert all(isinstance(p, Path) for p in cfg.tool_yamls) + assert cfg.dataset_chunks == data["dataset_chunks"] + + def test_missing_required_field_raises(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + del data["spec_file"] + with pytest.raises(ValidationError): + SetupCfg(**data) + + def test_postsample_intensity_optional(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + data.pop("postsample_intensity_counter_name", None) + cfg = SetupCfg(**data) + assert cfg.postsample_intensity_counter_name is None + + def test_fits_yaml_present(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = SetupCfg(**data) + assert cfg.fits_yaml is not None + assert isinstance(cfg.fits_yaml, Path) + + +class TestUpdateCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + cfg = UpdateCfg(**data) + assert cfg.scan_number == data["scan_number"] + assert isinstance(cfg.spec_file, Path) + assert isinstance(cfg.data_zarr, Path) + if "idx_slice_start" in data: + assert cfg.idx_slice_start == data["idx_slice_start"] + if "idx_slice_stop" in data: + assert cfg.idx_slice_stop == data["idx_slice_stop"] + if "idx_slice_step" in data: + assert cfg.idx_slice_step == data["idx_slice_step"] + + def test_slice_defaults(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + data.pop("idx_slice_start", None) + data.pop("idx_slice_stop", None) + data.pop("idx_slice_step", None) + cfg = UpdateCfg(**data) + assert cfg.idx_slice_start == 0 + assert cfg.idx_slice_stop == -1 + assert cfg.idx_slice_step == 1 + + def test_missing_required_field_raises(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + del data["scan_number"] + with pytest.raises(ValidationError): + UpdateCfg(**data) + + +class TestConvertCfg: + def test_parses_fixture(self, fixture_dir, tmp_path): + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + cfg = ConvertCfg(**data) + assert isinstance(cfg.outputdir, Path) + + def test_missing_outputdir_raises(self): + with pytest.raises(ValidationError): + ConvertCfg() diff --git a/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py b/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py new file mode 100644 index 00000000..a23a0a15 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_saxswaxs_to_chap.py @@ -0,0 +1,525 @@ +"""Tests for CHAP.saxswaxs.server.saxswaxs_to_chap.""" + +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +import pytest +import yaml + +from CHAP.saxswaxs.server.saxswaxs_to_chap import ( + VerboseSafeDumper, + convert_configs, + make_pipeline, + saxswaxs_to_chap, +) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +# Minimal integration tool payload; override individual keys per test. +_INTEGRATION_DEFAULTS = { + 'tool_type': 'integration', + 'title': 'my_integration', + 'integration_type': 'azimuthal', + 'detectors': [ + {'prefix': 'PIL5', 'poni_file': '/cal/det.poni', 'mask_file': '/cal/mask.tif'} + ], + 'radial_npt': 100, + 'azimuthal_npt': 72, + 'radial_min': 0.1, + 'radial_max': 5.0, + 'azimuthal_min': -180.0, + 'azimuthal_max': 180.0, + 'radial_units': 'q_A^-1', + 'azimuthal_units': 'chi_deg', +} + +_CORRECTIONS_DEFAULTS = { + 'tool_type': 'corrections', + 'title': 'my_correction', + 'correction_type': 'transmission', + 'validate_data_present': False, +} + + +def _write_tool(path, base, **overrides): + """Write a tool YAML to path, merging overrides into base.""" + data = {**base, **overrides} + path.write_text(yaml.dump(data)) + return data + + +def _write_integration(path, **overrides): + return _write_tool(path, _INTEGRATION_DEFAULTS, **overrides) + + +def _write_corrections(path, **overrides): + return _write_tool(path, _CORRECTIONS_DEFAULTS, **overrides) + + +def _read_outputs(tmp_path, det_name='detector_config.yaml', + pyfai_name='pyfai_integration_processor_config.yaml', + corr_name='corrections_config.yaml'): + with open(tmp_path / det_name) as f: + det = yaml.safe_load(f) + with open(tmp_path / pyfai_name) as f: + pyfai = yaml.safe_load(f) + with open(tmp_path / corr_name) as f: + corr = yaml.safe_load(f) + return det, pyfai, corr + + +def _make_mock_map(title='scan', scans=None): + """Return a mock MapConfig for make_pipeline tests. + + scans: list of (spec_file, scan_number, npts, shape) tuples. + """ + if scans is None: + scans = [('/spec/scan.spec', 1, 10, [10])] + mock_spec_scans = [] + for spec_file, scan_number, npts, shape in scans: + sp = MagicMock() + sp.spec_scan_npts = npts + sp.spec_scan_shape = shape + sg = MagicMock() + sg.spec_file = spec_file + sg.scan_numbers = [scan_number] + sg.get_scanparser.return_value = sp + mock_spec_scans.append(sg) + mc = MagicMock() + mc.title = title + mc.spec_scans = mock_spec_scans + return mc + + +# ── VerboseSafeDumper ───────────────────────────────────────────────────────── + +class TestVerboseSafeDumper: + def test_shared_reference_produces_no_alias(self): + shared = {'key': 'value'} + out = yaml.dump({'a': shared, 'b': shared}, Dumper=VerboseSafeDumper) + assert '*' not in out + assert '&' not in out + + def test_output_roundtrips(self): + data = {'x': [1, 2, 3], 'y': {'nested': True}} + out = yaml.dump(data, Dumper=VerboseSafeDumper) + assert yaml.safe_load(out) == data + + def test_nested_shared_reference(self): + node = [1, 2, 3] + out = yaml.dump({'p': node, 'q': node, 'r': node}, Dumper=VerboseSafeDumper) + assert out.count('- 1') == 3 + + +# ── convert_configs ─────────────────────────────────────────────────────────── + +class TestConvertConfigsOutputFiles: + def test_all_three_files_created(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + assert (tmp_path / 'detector_config.yaml').exists() + assert (tmp_path / 'pyfai_integration_processor_config.yaml').exists() + assert (tmp_path / 'corrections_config.yaml').exists() + + def test_absolute_output_filenames_respected(self, tmp_path): + sub = tmp_path / 'sub' + sub.mkdir() + _write_integration(tmp_path / 'tool.yaml') + det = str(sub / 'det.yaml') + pyfai = str(sub / 'pyfai.yaml') + corr = str(sub / 'corr.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')], + detector_filename=det, + pyfai_filename=pyfai, + correction_filename=corr) + assert Path(det).exists() + assert Path(pyfai).exists() + assert Path(corr).exists() + + def test_relative_output_filenames_resolved_against_outputdir(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')], + detector_filename='custom_det.yaml') + assert (tmp_path / 'custom_det.yaml').exists() + + +class TestConvertConfigsIntegrationMethod: + @pytest.mark.parametrize('integration_type,expected_method', [ + ('azimuthal', 'integrate1d'), + ('radial', 'integrate_radial'), + ('cake', 'integrate2d'), + ]) + def test_integration_method(self, tmp_path, integration_type, expected_method): + _write_integration(tmp_path / 'tool.yaml', integration_type=integration_type) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['integration_method'] == expected_method + + def test_radial_uses_azimuthal_npt_as_npt(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='radial', + radial_npt=50, azimuthal_npt=36) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + params = pyfai['integrations'][0]['integration_params'] + assert params['npt'] == 36 + assert params['npt_rad'] == 50 + + def test_azimuthal_uses_radial_npt_as_npt(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='azimuthal', + radial_npt=200) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['integration_params']['npt'] == 200 + + def test_cake_uses_both_npt_fields(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='cake', + radial_npt=80, azimuthal_npt=45) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + params = pyfai['integrations'][0]['integration_params'] + assert params['npt_rad'] == 80 + assert params['npt_azim'] == 45 + + def test_azimuthal_and_cake_have_multi_geometry(self, tmp_path): + for itype in ('azimuthal', 'cake'): + _write_integration(tmp_path / f'{itype}.yaml', integration_type=itype) + for itype in ('azimuthal', 'cake'): + convert_configs(str(tmp_path), [str(tmp_path / f'{itype}.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert 'multi_geometry' in pyfai['integrations'][0], itype + + def test_radial_does_not_have_multi_geometry(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', integration_type='radial') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert 'multi_geometry' not in pyfai['integrations'][0] + + def test_integration_name_from_title(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml', title='waxs_1d') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert pyfai['integrations'][0]['name'] == 'waxs_1d' + + def test_multiple_tools_produce_multiple_integrations(self, tmp_path): + _write_integration(tmp_path / 'a.yaml', title='a') + _write_integration(tmp_path / 'b.yaml', title='b') + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + assert len(pyfai['integrations']) == 2 + + +class TestConvertConfigsDetectors: + @pytest.mark.parametrize('prefix,expected_shape', [ + ('PIL5', [619, 487]), + ('PIL9', [407, 487]), + ('PIL11', [407, 487]), + ]) + def test_known_detector_shape(self, tmp_path, prefix, expected_shape): + dets = [{'prefix': prefix, 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + det = next(d for d in det_cfg['detectors'] if d['id'] == prefix) + assert det['shape'] == expected_shape + + def test_unknown_detector_gets_placeholder_shape(self, tmp_path): + dets = [{'prefix': 'NEWDET', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + det = next(d for d in det_cfg['detectors'] if d['id'] == 'NEWDET') + assert det['shape'] == [1, 1] + + def test_detector_deduplicated_across_tools(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'a.yaml', title='a', detectors=dets) + _write_integration(tmp_path / 'b.yaml', title='b', detectors=dets) + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + det_cfg, _, _ = _read_outputs(tmp_path) + assert [d['id'] for d in det_cfg['detectors']].count('PIL5') == 1 + + def test_azimuthal_integrator_deduplicated_across_tools(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/p.poni', 'mask_file': '/m.tif'}] + _write_integration(tmp_path / 'a.yaml', title='a', detectors=dets) + _write_integration(tmp_path / 'b.yaml', title='b', detectors=dets) + convert_configs(str(tmp_path), + [str(tmp_path / 'a.yaml'), str(tmp_path / 'b.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + ids = [ai['id'] for ai in pyfai['azimuthal_integrators']] + assert ids.count('PIL5') == 1 + + def test_azimuthal_integrator_poni_and_mask_stored(self, tmp_path): + dets = [{'prefix': 'PIL5', 'poni_file': '/exp/det.poni', + 'mask_file': '/exp/mask.tif'}] + _write_integration(tmp_path / 'tool.yaml', detectors=dets) + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, pyfai, _ = _read_outputs(tmp_path) + ai = pyfai['azimuthal_integrators'][0] + assert ai['poni_file'] == '/exp/det.poni' + assert ai['mask_file'] == '/exp/mask.tif' + + +class TestConvertConfigsCorrections: + def test_corrections_tool_written_to_corrections_config(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert len(corr['corrections']) == 1 + + def test_corrections_excludes_tool_type_key(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert 'tool_type' not in corr['corrections'][0] + + def test_corrections_excludes_validate_data_present(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert 'validate_data_present' not in corr['corrections'][0] + + def test_corrections_preserves_other_fields(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml', custom_field='hello') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert corr['corrections'][0].get('custom_field') == 'hello' + + def test_integration_tool_does_not_appear_in_corrections(self, tmp_path): + _write_integration(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + _, _, corr = _read_outputs(tmp_path) + assert corr['corrections'] == [] + + def test_corrections_tool_does_not_appear_in_detectors(self, tmp_path): + _write_corrections(tmp_path / 'tool.yaml') + convert_configs(str(tmp_path), [str(tmp_path / 'tool.yaml')]) + det, _, _ = _read_outputs(tmp_path) + assert det['detectors'] == [] + + +# ── make_pipeline ───────────────────────────────────────────────────────────── + +class TestMakePipeline: + def _run(self, tmp_path, mock_map, **kwargs): + with patch('CHAP.common.reader.YAMLReader') as MockReader: + MockReader.run.return_value = mock_map + result = make_pipeline(str(tmp_path), **kwargs) + with open(tmp_path / kwargs.get('pipeline_filename', 'pipeline.yaml')) as f: + written = yaml.safe_load(f) + return result, written + + def test_pipeline_file_written(self, tmp_path): + _, _ = self._run(tmp_path, _make_mock_map()) + assert (tmp_path / 'pipeline.yaml').exists() + + def test_custom_pipeline_filename(self, tmp_path): + self._run(tmp_path, _make_mock_map(), pipeline_filename='custom.yaml') + assert (tmp_path / 'custom.yaml').exists() + + def test_returns_dict_with_required_keys(self, tmp_path): + result, _ = self._run(tmp_path, _make_mock_map()) + assert isinstance(result, dict) + assert 'config' in result + assert 'setup' in result + assert 'convert' in result + + def test_config_root_is_resolved_outputdir(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + assert written['config']['root'] == str(tmp_path.resolve()) + + def test_setup_has_five_yaml_readers(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + readers = [s for s in written['setup'] if 'common.reader.YAMLReader' in s] + assert len(readers) == 5 + + def test_setup_yaml_reader_schemas(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + schemas = [s['common.reader.YAMLReader']['schema'] + for s in written['setup'] if 'common.reader.YAMLReader' in s] + assert 'common.models.map.DetectorConfig' in schemas + assert 'common.models.map.MapConfig' in schemas + assert 'common.models.integration.PyfaiIntegrationConfig' in schemas + assert 'saxswaxs.models.CorrectionsConfig' in schemas + assert 'saxswaxs.models.FitsConfig' in schemas + + def test_setup_has_setup_processor(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + procs = [s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s] + assert len(procs) == 1 + + def test_setup_processor_raw_data_false(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['raw_data'] is False + + def test_setup_has_zarr_writer(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + writers = [s for s in written['setup'] if 'common.writer.ZarrWriter' in s] + assert len(writers) == 1 + + def test_zarr_filename_derived_from_map_title(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(title='my_exp')) + writer = next(s for s in written['setup'] if 'common.writer.ZarrWriter' in s) + assert writer['common.writer.ZarrWriter']['filename'] == 'my_exp.zarr' + + @pytest.mark.parametrize('scans,expected_update_count', [ + ([('/spec', 1, 10, [10])], 1), # 1D scan → 1 update + ([('/spec', 1, 20, [5, 4])], 4), # 2D scan, 4 rows → 4 updates + ([('/spec', 1, 10, [10]), + ('/spec', 2, 5, [5])], 2), # two 1D scans → 2 updates + ]) + def test_update_pipeline_count(self, tmp_path, scans, expected_update_count): + _, written = self._run(tmp_path, _make_mock_map(scans=scans)) + update_keys = [k for k in written if k.startswith('update_')] + assert len(update_keys) == expected_update_count + + def test_update_pipeline_idx_slice_1d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc = next(s for s in written['update_0'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + assert proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] == { + 'start': 0, 'stop': 10, 'step': 1, + } + + def test_update_pipeline_idx_slice_2d_rows(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 20, [5, 4])])) + for row in range(4): + proc = next(s for s in written[f'update_{row}'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + sl = proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + assert sl == {'start': row * 5, 'stop': row * 5 + 5, 'step': 1} + + def test_update_npts_accumulates_across_scans(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(scans=[ + ('/spec', 1, 10, [10]), + ('/spec', 2, 5, [5]), + ])) + proc = next(s for s in written['update_1'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s) + sl = proc['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + assert sl['start'] == 10 + assert sl['stop'] == 15 + + def test_dataset_chunks_equals_row_npts_1d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['dataset_chunks'] == [10] + + def test_dataset_chunks_equals_row_npts_2d(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 30, [10, 3])])) + proc = next(s for s in written['setup'] + if 'saxswaxs.processor.SetupProcessor' in s) + assert proc['saxswaxs.processor.SetupProcessor']['dataset_chunks'] == [10] + + def test_convert_pipeline_zarr_and_nxs(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map(title='exp')) + assert len(written['convert']) == 1 + cfg = written['convert'][0]['common.processor.ZarrToNexusProcessor'] + assert cfg['zarr_filename'] == 'exp.zarr' + assert cfg['nexus_filename'] == 'exp.nxs' + + def test_update_pipeline_has_zarr_values_writer(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + writers = [s for s in written['update_0'] + if 'common.ZarrValuesWriter' in s] + assert len(writers) == 1 + + def test_update_pipeline_zarr_values_writer_idx_slice_matches_processor(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map( + scans=[('/spec', 1, 10, [10])])) + proc_sl = next(s for s in written['update_0'] + if 'saxswaxs.processor.UpdateValuesProcessor' in s + )['saxswaxs.processor.UpdateValuesProcessor']['idx_slice'] + writer_sl = next(s for s in written['update_0'] + if 'common.ZarrValuesWriter' in s + )['common.ZarrValuesWriter']['idx_slice'] + assert proc_sl == writer_sl + + def test_reader_filenames_resolved_against_outputdir(self, tmp_path): + _, written = self._run(tmp_path, _make_mock_map()) + reader = written['setup'][0]['common.reader.YAMLReader'] + assert reader['filename'].startswith(str(tmp_path.resolve())) + + +# ── saxswaxs_to_chap ────────────────────────────────────────────────────────── + +class TestSaxswaxsToChap: + def test_calls_convert_configs_with_outputdir_and_tools(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline'): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + mock_cc.assert_called_once() + args, kwargs = mock_cc.call_args + assert args[0] == str(tmp_path) + assert args[1] == [tool] + + def test_calls_make_pipeline_with_outputdir_and_map(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs'), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + mock_mp.assert_called_once() + args, kwargs = mock_mp.call_args + assert args[0] == str(tmp_path) + assert kwargs['map_filename'] == 'map.yaml' + + def test_default_filenames_forwarded_to_convert_configs(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline'): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + _, kwargs = mock_cc.call_args + assert kwargs['detector_filename'] == 'detector_config.yaml' + assert kwargs['pyfai_filename'] == 'pyfai_integration_processor_config.yaml' + assert kwargs['correction_filename'] == 'corrections_config.yaml' + + def test_default_filenames_forwarded_to_make_pipeline(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs'), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + _, kwargs = mock_mp.call_args + assert kwargs['fits_filename'] == 'fits_config.yaml' + assert kwargs['pipeline_filename'] == 'pipeline.yaml' + + def test_custom_filenames_forwarded(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs') as mock_cc, \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline') as mock_mp: + saxswaxs_to_chap( + 'map.yaml', [tool], str(tmp_path), + detector_filename='det.yaml', + pyfai_filename='pyfai.yaml', + correction_filename='corr.yaml', + fits_filename='fits.yaml', + pipeline_filename='pipe.yaml', + ) + _, cc_kwargs = mock_cc.call_args + assert cc_kwargs['detector_filename'] == 'det.yaml' + assert cc_kwargs['pyfai_filename'] == 'pyfai.yaml' + assert cc_kwargs['correction_filename'] == 'corr.yaml' + _, mp_kwargs = mock_mp.call_args + assert mp_kwargs['detector_filename'] == 'det.yaml' + assert mp_kwargs['fits_filename'] == 'fits.yaml' + assert mp_kwargs['pipeline_filename'] == 'pipe.yaml' + + def test_convert_configs_called_before_make_pipeline(self, tmp_path): + tool = str(tmp_path / 'tool.yaml') + call_order = [] + with patch('CHAP.saxswaxs.server.saxswaxs_to_chap.convert_configs', + side_effect=lambda *a, **kw: call_order.append('cc')), \ + patch('CHAP.saxswaxs.server.saxswaxs_to_chap.make_pipeline', + side_effect=lambda *a, **kw: call_order.append('mp')): + saxswaxs_to_chap('map.yaml', [tool], str(tmp_path)) + assert call_order == ['cc', 'mp'] diff --git a/CHAP/test/saxswaxs/server/test_server.py b/CHAP/test/saxswaxs/server/test_server.py new file mode 100644 index 00000000..97c6bf37 --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_server.py @@ -0,0 +1,133 @@ +"""Tests for the Flask HTTP interface in CHAP.saxswaxs.server.server.""" + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + + +def _load(fixture_dir, name): + return json.loads((fixture_dir / name).read_text()) + + +@pytest.fixture +def client(): + with patch("CHAP.saxswaxs.server.server.put"): + from CHAP.saxswaxs.server.server import app + app.config["TESTING"] = True + with app.test_client() as c: + yield c + + +def _setup_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "setup.json") + data["outputdir"] = str(tmp_path) + data["data_zarr"] = str(tmp_path / "data.zarr") + return data + + +def _update_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "update.json") + data["data_zarr"] = str(tmp_path / "data.zarr") + return data + + +def _convert_body(fixture_dir, tmp_path): + data = _load(fixture_dir, "convert.json") + data["outputdir"] = str(tmp_path) + return data + + +class TestSetupEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_setup_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + mock_put.assert_called_once() + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "setup" + assert type(args[0]).__name__ == "SetupCfg" + assert kwargs == {} + + def test_setup_cfg_fields_correct(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + body = _setup_body(fixture_dir, tmp_path) + client.post("/setup", json=body) + _, args, _ = mock_put.call_args[0] + cfg = args[0] + assert cfg.scan_number == body["scan_number"] + assert cfg.dwell_time_actual_counter_name == body["dwell_time_actual_counter_name"] + assert cfg.dataset_chunks == body["dataset_chunks"] + + def test_missing_required_field_returns_error(self, client, fixture_dir, tmp_path): + body = _setup_body(fixture_dir, tmp_path) + del body["spec_file"] + resp = client.post("/setup", json=body) + assert resp.status_code >= 400 + + def test_malformed_json_returns_error(self, client): + resp = client.post("/setup", data="not json", content_type="application/json") + assert resp.status_code >= 400 + + +class TestUpdateEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_update_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/update", json=_update_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + mock_put.assert_called_once() + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "update" + assert type(args[0]).__name__ == "UpdateCfg" + + def test_update_cfg_slice_correct(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + body = _update_body(fixture_dir, tmp_path) + client.post("/update", json=body) + _, args, _ = mock_put.call_args[0] + cfg = args[0] + assert cfg.idx_slice_start == body.get("idx_slice_start", 0) + assert cfg.idx_slice_stop == body.get("idx_slice_stop", -1) + assert cfg.idx_slice_step == body.get("idx_slice_step", 1) + + +class TestConvertEndpoint: + def test_returns_202(self, client, fixture_dir, tmp_path): + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + + def test_returns_queued_status(self, client, fixture_dir, tmp_path): + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.get_json() == {"status": "queued"} + + def test_put_called_with_convert_function(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put") as mock_put: + resp = client.post("/convert", json=_convert_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 + task, args, kwargs = mock_put.call_args[0] + assert task.__name__ == "convert" + assert type(args[0]).__name__ == "ConvertCfg" + + +class TestRequestLogging: + def test_request_completes_without_logging_error(self, client, fixture_dir, tmp_path): + with patch("CHAP.saxswaxs.server.server.put"): + resp = client.post("/setup", json=_setup_body(fixture_dir, tmp_path)) + assert resp.status_code == 202 diff --git a/CHAP/test/saxswaxs/server/test_task_queue.py b/CHAP/test/saxswaxs/server/test_task_queue.py new file mode 100644 index 00000000..d4f53d2e --- /dev/null +++ b/CHAP/test/saxswaxs/server/test_task_queue.py @@ -0,0 +1,73 @@ +"""Unit tests for CHAP.saxswaxs.server.task_queue.""" + +import queue +import threading +from unittest.mock import MagicMock, call, patch + +import pytest + +import CHAP.saxswaxs.server.task_queue as tq + + +def drain(q, timeout=2): + """Block until q is empty or timeout expires.""" + q.join() + + +class TestPut: + def test_put_enqueues_tuple(self): + task = MagicMock() + args = (1, 2) + kwargs = {"a": 3} + tq.put(task, args, kwargs) + item = tq._task_queue.get_nowait() + tq._task_queue.task_done() + assert item == (task, args, kwargs) + + +class TestWorker: + def _run_worker_once(self, task, args=(), kwargs={}): + """Put one task, wait for it to complete.""" + tq.put(task, args, kwargs) + tq._task_queue.join() + + def test_worker_calls_task(self): + called = threading.Event() + task = MagicMock(side_effect=lambda: called.set()) + self._run_worker_once(task) + assert called.wait(timeout=5), "worker never called the task" + task.assert_called_once() + + def test_worker_passes_args_and_kwargs(self): + received = {} + + def capture(*args, **kwargs): + received["args"] = args + received["kwargs"] = kwargs + + tq.put(capture, (10, 20), {"x": 99}) + tq._task_queue.join() + assert received["args"] == (10, 20) + assert received["kwargs"] == {"x": 99} + + def test_worker_retries_on_exception(self): + """Worker should retry a failing task until it succeeds.""" + attempt_count = [0] + done = threading.Event() + + def flaky(): + attempt_count[0] += 1 + if attempt_count[0] < 2: + raise RuntimeError("transient failure") + done.set() + + tq.put(flaky, (), {}) + assert done.wait(timeout=15), "worker never retried the task" + assert attempt_count[0] == 2 + + def test_worker_marks_task_done_after_success(self): + sentinel = threading.Event() + tq.put(lambda: sentinel.set(), (), {}) + # join() blocks until task_done() is called; it would hang on failure + tq._task_queue.join() + assert sentinel.is_set() diff --git a/setup.py b/setup.py index 01496704..381e8cc1 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,10 @@ def datafiles(idir, pattern=None): 'examples': data_files, }, entry_points={ - 'console_scripts': ['CHAP = CHAP.runner:main'], + 'console_scripts': [ + 'CHAP = CHAP.runner:main', + 'saxswaxs-server = CHAP.saxswaxs.server.server:run', + ], }, classifiers=[ "Programming Language :: Python :: 3",