From b0436b0adf804a60d3d29a987c852100d42f11f6 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Thu, 13 Aug 2026 08:03:15 +0000 Subject: [PATCH 1/2] ts: anchor tcsh initial prompt match to a fresh output line completion_tcsh_start expects the prompt right after "set prompt=..." using a bare "$completion_prompt\$" pattern. That command own echoed- back input already contains the exact prompt text ("set prompt= 'MODULESTEST> '"), so under load Expect can match that mid-flight echo before tcsh has even processed Enter, once a partial read happens to land right after it. That races the rest of session setup and can leave the completion script not fully sourced, surfacing later as a hang on the first real completion listing. Require the same leading carriage-return-plus-newline prefix completion_tcsh_raw already relies on for its own matching, so this only matches the real prompt tcsh prints once the command is done. Assisted-by: Claude:claude-sonnet-5 Signed-off-by: Xavier Delaruelle --- testsuite/completion.00-init/050-tcsh-procs.exp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/testsuite/completion.00-init/050-tcsh-procs.exp b/testsuite/completion.00-init/050-tcsh-procs.exp index a72a3742d..9a91aea92 100644 --- a/testsuite/completion.00-init/050-tcsh-procs.exp +++ b/testsuite/completion.00-init/050-tcsh-procs.exp @@ -117,9 +117,17 @@ proc completion_tcsh_start {} { # shell prompt pattern, since completion_tcsh_raw's own matching only # works once this prompt is in effect); every setup command after this # point goes through completion_tcsh_raw + # + # the command line setting it is itself 'set prompt='$completion_prompt'' + # -- its own echoed-back input already contains that exact text, so a + # bare "$completion_prompt\$" pattern can match that echo mid-flight, + # before tcsh has even processed Enter, the moment a partial read + # happens to end right after it; require the same '\r+\n' fresh-line + # prefix completion_tcsh_raw's own matching below relies on, so this + # only matches the real prompt tcsh prints once the command is done send -- "set prompt='$completion_prompt'\r" expect { - -re "$completion_prompt\$" {} + -re "\r+\n$completion_prompt\$" {} timeout { unresolved {completion tcsh prompt setup timed out}; return } } From f42363503d9dfc8d0751fe89e7abeadd70eb7446 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Fri, 14 Aug 2026 07:19:30 +0000 Subject: [PATCH 2/2] ts: add cookbook testsuite to validate documented recipes Add a 'cookbook' DejaGnu tool, alongside modules/install/lint/completion, that builds a sandboxed fixture from the exact files a cookbook recipe ships under doc/example//, then runs the commands demonstrated in the recipe's "Usage example" documentation in the order and against the output parsed straight out of the recipe's .rst file, rather than a hand-copied duplicate that could silently drift from it. cookbook_parse_transcript (005-init_ts.exp) walks a recipe's '.. parsed-literal::' blocks and returns {command output} pairs, with role markup stripped from the output. cookbook_output_re turns one such illustrated output into a regexp a sandboxed run's real output is checked against, widening the two kinds of detail a doc transcript cannot reproduce as-is: a run of '-' padding of unpredictable width, and whatever placeholder path a recipe test maps to its own sandbox path. Implements the first recipe test, sync-remote-appdir.exp, which also fixed the recipe doc's own inaccuracy caught along the way: 'module load' was shown printing a "Loading foo/2.1" banner it does not actually print at normal verbosity, now reflected by adding '-v' to the doc's load commands instead. Wired into 'make testcookbook' and 'script/mt cook'. Documented in doc/source/devel/testsuite.rst. Assisted-by: Claude:claude-sonnet-5 Signed-off-by: Xavier Delaruelle --- .gitignore | 3 + .hunspell.en.dic | 10 + Makefile | 9 +- NEWS.rst | 7 + doc/source/cookbook/sync-remote-appdir.rst | 4 +- doc/source/devel/testsuite.rst | 111 +++++++-- script/mt | 7 + testsuite/cookbook.00-init/005-init_ts.exp | 230 ++++++++++++++++++ testsuite/cookbook.00-init/006-procs.exp | 61 +++++ testsuite/cookbook.00-init/010-environ.exp | 81 ++++++ .../cookbook.00-init/011-save_test_env.exp | 23 ++ .../020-sync-remote-appdir.exp | 207 ++++++++++++++++ testsuite/cookbook.00-init/999-cleanup.exp | 29 +++ 13 files changed, 762 insertions(+), 20 deletions(-) create mode 100644 testsuite/cookbook.00-init/005-init_ts.exp create mode 100644 testsuite/cookbook.00-init/006-procs.exp create mode 100644 testsuite/cookbook.00-init/010-environ.exp create mode 100644 testsuite/cookbook.00-init/011-save_test_env.exp create mode 100644 testsuite/cookbook.00-init/020-sync-remote-appdir.exp create mode 100644 testsuite/cookbook.00-init/999-cleanup.exp diff --git a/.gitignore b/.gitignore index d69258ac5..11d0642c2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ /lint.sum /completion.log /completion.sum +/cookbook.log +/cookbook.sum /icdiff /.noicdiff /miniconda3 @@ -58,6 +60,7 @@ /testsuite/.config /testsuite/completion-use-fixture /testsuite/completion-home +/testsuite/cookbook-sandbox /doc/build /doc/source/version.py /doc/demo/*/*.svg diff --git a/.hunspell.en.dic b/.hunspell.en.dic index 936934eb1..6989c8ffa 100644 --- a/.hunspell.en.dic +++ b/.hunspell.en.dic @@ -348,6 +348,9 @@ config configs contrib conun +cookbookdocdir +cookbookexampledir +cookbooksandbox cov cp cr @@ -421,6 +424,7 @@ etcdic etcdir eu eval +evals exe execLogger execcmdlist @@ -488,6 +492,7 @@ grp gz gzip hardcode +hardcodes hdf homebrewed hostname @@ -829,6 +834,7 @@ submodule subprojects subshell substring +substrings sudo svn sw @@ -861,6 +867,7 @@ tcsh tcshrc ter testcompletion +testcookbook testdir testfile testfiles @@ -1427,9 +1434,11 @@ path3 RUNTESTFILES RUNTESTFLAGS sandboxed +sandbox's SC1090 serienum setpath +step's shorthands spider1 spider2 @@ -1546,6 +1555,7 @@ API api modnamevr modulerc's +transcript's logRequestedLoad auditRequestedLoad blockModule diff --git a/Makefile b/Makefile index 8a198ab8d..63ce4eb23 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ install-testmodspath-empty install-testmodspath-wild \ install-testmoguicmd uninstall-testconfig uninstall dist dist-tar \ dist-gzip dist-bzip2 dist-win srpm rpm clean distclean test-deps test \ - testinstall testlint testcompletion + testinstall testlint testcompletion testcookbook # commands to install files INSTALL = install @@ -1080,6 +1080,13 @@ testcompletion: initdir $(MODULECMD) TESTSUITEDIR=`cd testsuite;pwd -P`; export TESTSUITEDIR; \ runtest --srcdir $$TESTSUITEDIR --objdir $$OBJDIR $(RUNTESTFLAGS) --tool completion $(RUNTESTFILES) +testcookbook: $(MODULECMD) + TCLSH=$(TCLSH); export TCLSH; \ + MODULECMD=$(MODULECMD); export MODULECMD; \ + OBJDIR=`pwd -P`; export OBJDIR; \ + TESTSUITEDIR=`cd testsuite;pwd -P`; export TESTSUITEDIR; \ + runtest --srcdir $$TESTSUITEDIR --objdir $$OBJDIR $(RUNTESTFLAGS) --tool cookbook $(RUNTESTFILES) + # install enhanced diff tool (to review test results) icdiff: diff --git a/NEWS.rst b/NEWS.rst index 389951537..ecc36692e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -207,6 +207,13 @@ Modules 5.7.0 (not yet released) :subcmd:`initlist`, :subcmd:`initprepend` and :subcmd:`initrm` sub-commands as Tab completion candidates (bash, fish, tcsh, zsh), since ``module help`` does not document them. +* Testsuite: add *cookbook* testsuite to validate the recipes documented + under :ref:`cookbook`. This testsuite can be invoked with ``make + testcookbook`` or ``script/mt cook``. It builds a sandboxed fixture + from the exact files a recipe ships under :file:`doc/example/`, then runs + the commands demonstrated in the recipe's ``Usage example`` documentation + in order and checks that similar output is obtained. Currently covers the + :ref:`sync-remote-appdir` recipe. .. _5.6 release notes: diff --git a/doc/source/cookbook/sync-remote-appdir.rst b/doc/source/cookbook/sync-remote-appdir.rst index ba0c80b36..d72cbf14e 100644 --- a/doc/source/cookbook/sync-remote-appdir.rst +++ b/doc/source/cookbook/sync-remote-appdir.rst @@ -143,7 +143,7 @@ completes: .. parsed-literal:: - :ps:`$` module load foo/2.1 + :ps:`$` module load -v foo/2.1 Syncing 'foo-2.1-build3' application directory from remote share... Loading :sgrhi:`foo/2.1` @@ -167,7 +167,7 @@ either: :ps:`$` module avail foo --------------- :sgrdi:`/path/to/modulefiles` --------------- foo/2.1 - :ps:`$` module load foo/2.1 + :ps:`$` module load -v foo/2.1 Loading :sgrhi:`foo/2.1` .. vim:set tabstop=2 shiftwidth=2 expandtab autoindent: diff --git a/doc/source/devel/testsuite.rst b/doc/source/devel/testsuite.rst index 621f12583..30181e63e 100644 --- a/doc/source/devel/testsuite.rst +++ b/doc/source/devel/testsuite.rst @@ -13,12 +13,12 @@ line tools. DejaGnu groups test files (``.exp``, for *expect script*) under a *tool*: a run of ``runtest --tool `` sources every matching ``.exp`` file it finds for that tool and reports a ``PASS``/``FAIL``/``XFAIL``/ ``UNRESOLVED`` line for each individual check performed. Modules defines -four tools, matched to four kinds of tests (see `Kinds of tests`_ below). +five tools, matched to five kinds of tests (see `Kinds of tests`_ below). Kinds of tests -------------- -The testsuite exercises four different things, run as four separate +The testsuite exercises five different things, run as five separate DejaGnu tools: ``modules`` @@ -51,10 +51,20 @@ DejaGnu tools: currently covers bash, zsh, fish and tcsh, see `completion.00-init layout`_. +``cookbook`` + Checks the recipes documented under :ref:`cookbook`: for each recipe + covered, builds a sandboxed fixture from the exact files it ships under + :file:`doc/example/` and drives it through the commands demonstrated in + the recipe's ``Usage example`` documentation, parsed straight out of + the recipe's ``.rst`` file rather than hand-copied (see + `cookbook.00-init layout`_). Driven by the :file:`cookbook.00-init` + directory. + Each tool corresponds to one Makefile target (``test``, ``testinstall``, -``testlint``, ``testcompletion``, see `Running the testsuite`_) and to one -log file (:file:`modules.log`, :file:`install.log`, :file:`lint.log`, -:file:`completion.log`) produced in the top build directory. +``testlint``, ``testcompletion``, ``testcookbook``, see `Running the +testsuite`_) and to one log file (:file:`modules.log`, :file:`install.log`, +:file:`lint.log`, :file:`completion.log`, :file:`cookbook.log`) produced in +the top build directory. Two additional run modes apply to the ``modules`` tool rather than adding a new one: @@ -77,10 +87,11 @@ Test file directories Test files are grouped in numbered directories named ``.-``, e.g. :file:`modules.50-cmds`, -:file:`install.00-init`, :file:`lint.00-init`, :file:`completion.00-init`. -The ```` prefix ties the directory to one of the four DejaGnu tools -above (DejaGnu only looks at directories whose prefix matches the ``--tool`` -given to ``runtest``); the two-digit ```` number fixes run order +:file:`install.00-init`, :file:`lint.00-init`, :file:`completion.00-init`, +:file:`cookbook.00-init`. The ```` prefix ties the directory to one of +the five DejaGnu tools above (DejaGnu only looks at directories whose +prefix matches the ``--tool`` given to ``runtest``); the two-digit +```` number fixes run order and is what you pass to :file:`script/mt` to select a whole directory (e.g. ``script/mt 50``); the ```` suffix is just a human-readable label. @@ -147,8 +158,9 @@ Current ``modules.*`` series, in run order: ``99-finish`` Testsuite teardown (removes cache files created for the run) -``install.00-init``, ``lint.00-init`` and ``completion.00-init`` are each a -single series (those tools are much smaller and don't need topic splitting). +``install.00-init``, ``lint.00-init``, ``completion.00-init`` and +``cookbook.00-init`` are each a single series (those tools are much smaller +and don't need topic splitting). Every series directory ends with a ``999-cleanup.exp`` file (see `Test file anatomy`_) and, for the ``modules`` tool, most series begin with a @@ -199,6 +211,66 @@ shell-specific in a way none of the other three tools are: Adding a new shell means adding its own ``completion__*`` procs file and test file; nothing in ``006-procs.exp`` needs to change. +``cookbook.00-init`` layout +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``cookbook`` tool follows the common ``005``/``006``/``010``/``011`` +setup numbering (see `Test file anatomy`_ and `Base test procedures`_ +below), driving ``modulecmd.tcl`` directly through ``testouterr_cmd``/ +``testouterr_cmd_re`` exactly like the ``modules`` tool, plus one shared +helper that is specific to this tool: + +- ``005-init_ts.exp`` defines ``cookbookdocdir``/``cookbookexampledir`` + (pointing at :file:`doc/source/cookbook` and :file:`doc/example`), the + ``cookbooksandbox`` fixture root (recreated fresh for the whole run), + ``cookbook_read_example_file``/``cookbook_write_file`` (read a file + shipped under :file:`doc/example//`, applying a ``string map`` + substitution list, and write it into the sandbox), and + ``cookbook_parse_transcript`` -- parses the ``.. parsed-literal::`` blocks + of a named section (e.g. ``Usage example``) of a recipe's ``.rst`` file + and returns the ordered list of ``{command output}`` pairs shown there, + role markup (``:sgrhi:`text``` and friends) stripped from the output, so + a recipe test file drives the exact command sequence its own + documentation demonstrates, in that order, and checks its own real output + is similar to the one illustrated -- instead of both being a hand-copied + duplicate that could silently drift from the documentation. + ``cookbook_output_re`` turns one such illustrated output into the regexp + a sandboxed run's real output is checked against: escaped as a literal, + except a run of ``-`` padding (widened to ``[-]+``, since its exact width + depends on terminal-width detection a non-tty test run does not have) and + whatever literal substrings the recipe test passes it to widen too (e.g. + a placeholder path mapped to the sandbox's real one). +- ``006-procs.exp``/``010-environ.exp``/``011-save_test_env.exp`` set up + ``_test_sub``, a clean baseline environment (sandboxed ``$HOME``, no + color, deterministic ``avail``/``list`` output, ``MODULES_SITECONFIG``/ + ``MODULES_TAG_ABBREV``/``MODULES_NON_EXPORTABLE_TAGS`` left unset so each + recipe states what it actually relies on), and the ``save_test_env`` + checkpoint, exactly as for the other tools. +- ``0NN-.exp`` (e.g. ``020-sync-remote-appdir.exp``), named + after the recipe's own ``.rst`` file, holds one recipe's test: it checks + any external binary the recipe needs (``unsupported`` + ``return`` if + missing, e.g. ``rsync`` for ``sync-remote-appdir``), builds a + ``$cookbooksandbox/`` fixture from the recipe's real + :file:`doc/example//` files via ``cookbook_read_example_file`` (only + substituting whatever real-system absolute path the recipe hardcodes, + e.g. ``/remote_apps``, for a sandbox path -- everything else is used byte + for byte), points the test environment at that sandbox, then loops over + ``cookbook_parse_transcript``'s steps in order, running each with + ``testouterr_cmd_re`` against ``cookbook_output_re`` of its illustrated + output (stdout, the env-var-assignment syntax a shell evals silently and + no recipe doc illustrates, is still checked against a hand-built answer + per step -- ``unresolved`` + ``return`` first if the transcript's step + count no longer matches what the test knows how to build one for, as a + signal it needs updating alongside the doc). Simulates each step's + effect on the environment for the next one exactly as a real shell would + (``setenv_loaded_module``/``setenv_path_var``), since every + ``modulecmd.tcl`` invocation here is an independent process. + +Adding a new recipe means adding its own ``0NN-.exp`` file; +nothing in ``005-init_ts.exp`` needs to change unless the new recipe needs a +kind of substitution or transcript shape the existing helpers do not cover +yet. + Fixture and support directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -609,6 +681,7 @@ With ``make`` make testinstall # 'install' suite against a tree already processed by 'make install' make testlint # 'lint' suite (Nagelfar + ShellCheck) make testcompletion # 'completion' suite (interactive Tab-completion tests) + make testcookbook # 'cookbook' suite (doc/source/cookbook recipes) Each target ends up calling ``runtest --tool $(RUNTESTFLAGS) $(RUNTESTFILES)``: ``RUNTESTFILES``, if set, restricts the run to specific @@ -634,12 +707,14 @@ calling ``make test`` directly when iterating on a specific area. script/mt install # same as: make testinstall script/mt lint # same as: make testlint script/mt comp # same as: make testcompletion + script/mt cook # same as: make testcookbook script/mt 50/470 # only testsuite/modules.50-cmds/470-*.exp script/mt 50 # every file in testsuite/modules.50-cmds script/mt 61 # collection series (always run whole, see below) script/mt lint 00/030 # only testsuite/lint.00-init/030-*.exp script/mt comp 00/021 # only testsuite/completion.00-init/021-*.exp + script/mt cook 00/020 # only testsuite/cookbook.00-init/020-*.exp script/mt 50/{280,290} 61 # several selections at once script/mt --help # full usage @@ -648,7 +723,8 @@ mandatory setup files for that tool (for ``modules``: ``00/005 00/006 00/010 00/050 00/060 00/080 00/085``; for ``install``: ``00/005 00/006 00/010 00/011``; for ``lint``: ``00/005 00/006 00/011``; for ``completion``: ``00/005 00/006 00/007 00/008 00/010 00/011 00/020 00/030 -00/040 00/050``), plus the ``999-cleanup.exp`` of every selected series. +00/040 00/050``; for ``cookbook``: ``00/005 00/006 00/010 00/011``), plus +the ``999-cleanup.exp`` of every selected series. Passing a bare series number always expands to every file in that directory, because several of those series are order-sensitive or enumerate a whole modulepath (see @@ -744,11 +820,12 @@ Debugging a broken test ``send_user`` progress messages emitted by ``config/base-config.exp`` helpers like ``setenv_var``/``change_file_perms``), set ``RUNTESTFLAGS='-v -v'`` and call ``make test``/``testinstall``/ - ``testlint``/``testcompletion`` directly, or invoke ``runtest`` yourself - with the environment variables :file:`script/mt`/the Makefile targets set - up (``TCLSH``, ``MODULECMD``, ``OBJDIR``, ``TESTSUITEDIR``) -- see the - ``test``/``testinstall``/``testlint``/``testcompletion`` targets in - :file:`Makefile` for the exact invocation. + ``testlint``/``testcompletion``/``testcookbook`` directly, or invoke + ``runtest`` yourself with the environment variables :file:`script/mt`/the + Makefile targets set up (``TCLSH``, ``MODULECMD``, ``OBJDIR``, + ``TESTSUITEDIR``) -- see the + ``test``/``testinstall``/``testlint``/``testcompletion``/``testcookbook`` + targets in :file:`Makefile` for the exact invocation. 4. **Check for order dependence.** If a test passes alone but fails in a full run (or vice-versa), suspect a missing/incomplete ``reset_test_env`` footer in an earlier file, or a global-enumeration test (`Adding new test diff --git a/script/mt b/script/mt index c69508919..59aa92e22 100755 --- a/script/mt +++ b/script/mt @@ -33,6 +33,7 @@ Test suites: install Test Modules installation lint Lint script files of this repository comp Test shell completion scripts + cook Test the cookbook recipes documented in doc/source/cookbook Select parts of testsuite: serienum Identification number of testsuite directory containing a @@ -48,6 +49,7 @@ Examples: $0 quick $0 lint $0 comp + $0 cook $0 50/37 61 70/{290,440} " } @@ -109,6 +111,11 @@ if [ $# -gt 0 ]; then testserie='completion' setuptestfiles=(00/005 00/006 00/007 00/008 00/010 00/011 00/020 00/030 00/040 00/050) shift + elif [ "$1" = "cook" ]; then + target='testcookbook' + testserie='cookbook' + setuptestfiles=(00/005 00/006 00/010 00/011) + shift fi fi diff --git a/testsuite/cookbook.00-init/005-init_ts.exp b/testsuite/cookbook.00-init/005-init_ts.exp new file mode 100644 index 000000000..8e7820f1e --- /dev/null +++ b/testsuite/cookbook.00-init/005-init_ts.exp @@ -0,0 +1,230 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Testuite testsequence +# Command: +# Sub-Command: +# +# Comment: %C{ +# Defines all patterns used further in the testsuite +# }C% +# +############################################################################## + +proc find_bin {name} { + set path {} + + if {[info exists ::env(PATH)]} { + foreach dir [split $::env(PATH) ":"] { + if {[file executable "$dir/$name"]} { + set path "$dir/$name" + break + } + } + } + + return $path +} + +if {$verbose} { + send_user "\tInitializing the testsuite ...\n" +} + +set TCLSH $env(TCLSH) +set MODULECMD $env(MODULECMD) + +# every cookbook recipe is documented at doc/source/cookbook/.rst and +# ships its installable files under doc/example// +set cookbookdocdir "$env(TESTSUITEDIR)/../doc/source/cookbook" +set cookbookexampledir "$env(TESTSUITEDIR)/../doc/example" + +# sandbox holding every recipe's fixture tree, recreated fresh for each run; +# each recipe test file builds its own "$cookbooksandbox/" subtree and +# is responsible for removing it once done (see 999-cleanup.exp for the +# final sweep of the whole sandbox) +set cookbooksandbox "$env(TESTSUITEDIR)/cookbook-sandbox" +file delete -force $cookbooksandbox +file mkdir $cookbooksandbox + +# escape a string so it is safe to embed in a Tcl regexp as a literal +proc cookbook_re_escape {str} { + return [regsub -all {[][$.+?*\\()&|^{}]} $str {\\&}] +} + +# read a file shipped with a recipe under doc/example//, applying a +# 'string map' substitution list (e.g. to swap a hardcoded real-system path +# the recipe documents for this test's own sandbox path); used to build a +# recipe test's fixture from the exact files a user would install, instead +# of a hand-copied duplicate that could silently drift from them +proc cookbook_read_example_file {path submap} { + set fid [open $path r] + set content [read $fid] + close $fid + return [string map $submap $content] +} + +# write 'content' to 'path', creating any missing parent directory +proc cookbook_write_file {path content} { + file mkdir [file dirname $path] + set fid [open $path w] + puts -nonewline $fid $content + close $fid +} + +# strip Sphinx inline role markup (':rolename:`text`', used throughout this +# project's docs to color a parsed-literal transcript like a real terminal) +# down to the plain text it wraps +proc cookbook_strip_roles {text} { + return [regsub -all {:[A-Za-z0-9_]+:`([^`]*)`} $text {\1}] +} + +# turn role-stripped illustrative output text into a regexp matching what a +# sandboxed test run actually produces: escape it as a literal, then widen +# the two kinds of detail a doc transcript shows for illustration but a +# sandboxed, non-interactive run cannot reproduce as-is -- a run of '-' +# padding whose exact width depends on terminal-width detection, and +# whatever literal substrings 'subs' (a {pattern replacement ...} list, e.g. +# a placeholder path mapped to this test's real sandbox path) says to widen +proc cookbook_output_re {text {subs {}}} { + set re [cookbook_re_escape $text] + set re [regsub -all -- {-{2,}} $re {[-]+}] + foreach {pattern replacement} $subs { + set re [string map [list [cookbook_re_escape $pattern] $replacement] $re] + } + return $re +} + +# build a {command output} transcript step: 'curoutput' is the list of +# output lines collected so far for 'cmd', with any trailing blank lines +# dropped (blank lines in the middle, e.g. separating an avail listing from +# its trailing 'Key:' block, are kept) and role markup stripped +proc cookbook_flush_step_ {cmd curoutput} { + while {[llength $curoutput] && [lindex $curoutput end] eq {}} { + set curoutput [lrange $curoutput 0 end-1] + } + return [list $cmd [cookbook_strip_roles [join $curoutput \n]]] +} + +# parse the '.. parsed-literal::' blocks of a cookbook recipe's rst file +# found under the section whose title is 'title' (matched as an exact line +# immediately followed by a '-' underline at least as long as the title, +# the section-heading convention used throughout this project's docs), and +# return the ordered list of {command output} pairs typed at/shown after +# the ':ps:`$`' prompt therein, with the output's role markup stripped +# (see cookbook_strip_roles). This lets a recipe test drive the same +# command sequence its documentation demonstrates, in the same order, and +# check its own real output is similar to the one illustrated there +# (see cookbook_output_re), instead of both being a hand-copied duplicate +# that could silently drift from the documentation +proc cookbook_parse_transcript {rstfile title} { + set fid [open $rstfile r] + set data [read $fid] + close $fid + set lines [split $data \n] + set n [llength $lines] + + # locate '\n<underline>' + set start {} + for {set i 0} {$i < $n - 1} {incr i} { + set nextline [lindex $lines [expr {$i + 1}]] + if {[lindex $lines $i] eq $title && [regexp {^-+$} $nextline] + && [string length $nextline] >= [string length $title]} { + set start [expr {$i + 2}] + break + } + } + if {$start eq {}} { + error "cookbook_parse_transcript: section '$title' not found in\ + $rstfile" + } + + # section ends at the next heading (a non-blank line immediately + # followed by an underline at least as long) or end of file + set end $n + for {set i $start} {$i < $n - 1} {incr i} { + set line [lindex $lines $i] + set nextline [lindex $lines [expr {$i + 1}]] + if {$line ne {} && [regexp {^[-=~]+$} $nextline] + && [string length $nextline] >= [string length $line]} { + set end $i + break + } + } + + # walk the section line by line, only collecting content indented under + # a '.. parsed-literal::' directive (prose paragraphs in between are + # ignored); a ':ps:`$`' line starts a new command, every following + # indented line up to the next ':ps:`$`' line or the block's end (a + # dedented line) is that command's illustrated output. Only the block's + # own fixed indentation (its first content line's leading whitespace, + # normally 4 spaces) is stripped off each line, not all of it -- output + # can itself carry meaningful leading whitespace, e.g. a numbered + # 'module list' entry indented one space past its block indent + set steps {} + set inliteral 0 + set blockindent {} + set havecmd 0 + set cmd {} + set curoutput {} + + foreach line [lrange $lines $start [expr {$end - 1}]] { + if {[regexp {^\s*\.\. parsed-literal::\s*$} $line]} { + if {$havecmd} { + lappend steps [cookbook_flush_step_ $cmd $curoutput] + set havecmd 0 + } + set inliteral 1 + set blockindent {} + set curoutput {} + continue + } + if {!$inliteral} { + continue + } + if {[string trim $line] eq {}} { + if {$havecmd} { + lappend curoutput {} + } + continue + } + if {$blockindent eq {}} { + regexp {^(\s*)} $line -> blockindent + } + set indentlen [string length $blockindent] + if {[string range $line 0 [expr {$indentlen - 1}]] ne $blockindent} { + # dedented below the block's own indentation: it has ended + if {$havecmd} { + lappend steps [cookbook_flush_step_ $cmd $curoutput] + set havecmd 0 + } + set inliteral 0 + continue + } + set content [string range $line $indentlen end] + if {[regexp {^:ps:`\$`\s?(.*)$} $content -> newcmd]} { + if {$havecmd} { + lappend steps [cookbook_flush_step_ $cmd $curoutput] + } + set cmd $newcmd + set curoutput {} + set havecmd 1 + } elseif {$havecmd} { + lappend curoutput $content + } + } + if {$havecmd} { + lappend steps [cookbook_flush_step_ $cmd $curoutput] + } + + return $steps +} + +# vim:set tabstop=3 shiftwidth=3 expandtab autoindent: diff --git a/testsuite/cookbook.00-init/006-procs.exp b/testsuite/cookbook.00-init/006-procs.exp new file mode 100644 index 000000000..8d70fa79e --- /dev/null +++ b/testsuite/cookbook.00-init/006-procs.exp @@ -0,0 +1,61 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Test procedures +# Command: +# Sub-Command: +# +# Comment: %C{ +# Defines the testprocedures for probing the outputs +# on stdout and stderr of the test target +# }C% +# +############################################################################## + +# +# Test subprocedures +# + +proc _test_sub {test_shell cmd {input noinput} {launcher {}}} { + global comp_output + global comp_error + global comp_exit + global shell + global verbose + + if {$verbose} { + set cmdreport "modulecmd $test_shell $cmd" + if {$input ne {noinput}} { + append cmdreport " << $input" + } + send_user " ... Testing '$cmdreport'\n" + } + set comp_output "" + set comp_error "" + set comp_exit "" + + if { ![info exists test_shell] || ![info exists cmd] } { + unresolved "$cmd ($test_shell) internal error" + } + + set shell $test_shell + modulecmd_xxx_ "$cmd" $input $launcher + + if { $verbose > 1 } { + set ol [string length $comp_output] + set oe [string length $comp_error] + send_user "OUT\[$ol\]: '$comp_output'#>\n" + send_user "ERR\[$oe\]: '$comp_error'#>\n" + send_user "EXIT: '$comp_exit'#>\n" + } +} + +# vim:set tabstop=3 shiftwidth=3 expandtab autoindent: diff --git a/testsuite/cookbook.00-init/010-environ.exp b/testsuite/cookbook.00-init/010-environ.exp new file mode 100644 index 000000000..f8fc2d759 --- /dev/null +++ b/testsuite/cookbook.00-init/010-environ.exp @@ -0,0 +1,81 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: initialization of the shell environment +# Command: +# Sub-Command: +# +# Comment: %C{ +# First time initialization: setup a clean, reproducible +# environment for the cookbook recipe tests +# }C% +# +############################################################################## + +if {$verbose} { + send_user "\tInitializing a clean environment ...\n" +} + +# clear testsuite-related variables +foreach var [concat [array names env -glob TESTSUITE_*]\ + [list TESTSUITE TS0 TS1 TS2 TS3 TS4 TS4 TS6 TS7 TS8 TS9]] { + # preserve variable externally set to configure testsuite run + if {$var ne {TESTSUITE_ENABLE_SITECONFIG}} { + unsetenv_var $var + } +} + +# start every cookbook test from a clean module environment +unsetenv_var MODULEPATH +unsetenv_var LOADEDMODULES +unsetenv_var _LMFILES_ +unsetenv_var __MODULES_LMALTNAME +unsetenv_var MODULERCFILE +unsetenv_var MODULESHOME +unsetenv_var LD_LIBRARY_PATH +foreach var [array names env] { + switch -glob $var { + __MODULES_SHARE_* - __MODULES_PUSHENV_* - __MODULES_QUAR_* - \ + MODULES_FAMILY_* - MODULES_RUNENV_* - __MODULES_LM* { + unsetenv_var $var + } + } +} + +# each recipe applies its own site configuration/tag setup: start unset so +# every test file has to state what it actually relies on +unsetenv_var MODULES_SITECONFIG +unsetenv_var MODULES_TAG_ABBREV +unsetenv_var MODULES_NON_EXPORTABLE_TAGS +setenv_var MODULES_TAG_COLOR_NAME {} + +# sandbox $HOME so any file a recipe writes there never touches a +# developer's actual home +set cookbookhome "$cookbooksandbox/home" +file mkdir $cookbookhome +setenv_var HOME $cookbookhome + +# ensure colored output is disabled by default, so expected messages do not +# need to account for SGR escape sequences +setenv_var MODULES_COLOR never + +# ensure deterministic avail/list output regardless of the calling shell's +# own configuration +setenv_var MODULES_AVAIL_OUTPUT {modulepath:alias:dirwsym:sym:tag:variantifspec:key} +setenv_var MODULES_LIST_OUTPUT {header:idx:variant:sym:tag:key} + +# ensure tests are run with a regular default verbosity level +setenv_var MODULES_VERBOSITY normal + +# setup basic locale for tests +setenv_var LANG C + +# vim:set tabstop=3 shiftwidth=3 expandtab autoindent: diff --git a/testsuite/cookbook.00-init/011-save_test_env.exp b/testsuite/cookbook.00-init/011-save_test_env.exp new file mode 100644 index 000000000..09c1856e1 --- /dev/null +++ b/testsuite/cookbook.00-init/011-save_test_env.exp @@ -0,0 +1,23 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Testuite testsequence +# Command: +# Sub-Command: +# +# Comment: %C{ +# Save variable set defined to perform test, which will +# be restored at end of each test set +# }C% +# +############################################################################## + +save_test_env diff --git a/testsuite/cookbook.00-init/020-sync-remote-appdir.exp b/testsuite/cookbook.00-init/020-sync-remote-appdir.exp new file mode 100644 index 000000000..c5ec76f4e --- /dev/null +++ b/testsuite/cookbook.00-init/020-sync-remote-appdir.exp @@ -0,0 +1,207 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Testuite testsequence +# Command: avail, load, list, unload +# Modulefiles: sandboxed copy of doc/example/sync-remote-appdir +# Sub-Command: +# +# Comment: %C{ +# Exercise the 'Sync remote application directories on +# first load' cookbook recipe +# (doc/source/cookbook/sync-remote-appdir.rst) against +# the exact files shipped in +# doc/example/sync-remote-appdir/: the remote-share +# application directory gets synced locally on first +# load only, the 'remote' tag shows up on avail until +# then and is dropped once loaded, and the commands +# driven here are the ones the recipe doc's 'Usage +# example' section demonstrates (see +# cookbook_parse_transcript, defined in 005-init_ts.exp) +# }C% +# +############################################################################## + +set recipe sync-remote-appdir +set reciperst "$cookbookdocdir/$recipe.rst" +set recipeexampledir "$cookbookexampledir/$recipe" + +if {![file readable $reciperst] || ![file isdirectory $recipeexampledir]} { + unsupported "cookbook recipe files not found for '$recipe' (this\ + testsuite requires a full source checkout, not a release tarball)" + return +} + +set rsyncbin [find_bin rsync] +if {$rsyncbin eq {}} { + unsupported {no rsync binary found to drive the sync-remote-appdir cookbook recipe} + return +} + +# +# build the sandbox fixture from the exact files shipped with the recipe +# + +set recipedir "$cookbooksandbox/$recipe" +file delete -force $recipedir + +set remoteapps "$recipedir/remote_apps" +set localapps "$recipedir/local_apps" +set recipemodpath "$recipedir/modulefiles" +set recipeetc "$recipedir/etc" +file mkdir $remoteapps $localapps $recipemodpath $recipeetc + +# .module_appdir_map: copied verbatim, no hardcoded path to rewrite, at the +# root of the sandboxed remote share +set mapfile [file join $recipeexampledir .module_appdir_map] +file copy $mapfile "$remoteapps/.module_appdir_map" + +# fake application directory payload on the sandboxed remote share, synced +# locally on first load -- the recipe doc does not ship one since its +# content is site-specific, only its mapped basename +regexp {\S+\s+(\S+)} [cookbook_read_example_file $mapfile {}] -> appdir +file mkdir "$remoteapps/$appdir/bin" "$remoteapps/$appdir/lib" +cookbook_write_file "$remoteapps/$appdir/bin/foo" "#!/bin/sh\necho foo\n" +cookbook_write_file "$remoteapps/$appdir/lib/libfoo.so" "fake shared object\n" + +# every other recipe file hardcodes '/remote_apps'/'/local_apps' as the +# real-system absolute paths it gets installed under: rewrite just those to +# this test's own sandbox paths, everything else is used byte for byte -- +# the same kind of surgical rewrite completion.00-init/005-init_ts.exp +# applies to the tcsh completion script under test. This sandbox is also +# not tied to any particular Modules release, so also drop the recipe's +# '#%Module5.7' minimum-version requirement, which would otherwise reject +# these files on a pre-5.7 development checkout of the very release that +# ships the hook API they exercise +set submap [list /remote_apps $remoteapps /local_apps $localapps\ + #%Module5.7 #%Module] + +cookbook_write_file "$recipemodpath/.modulerc" [cookbook_read_example_file\ + [file join $recipeexampledir modulefiles/.modulerc] $submap] +cookbook_write_file "$recipemodpath/foo/2.1" [cookbook_read_example_file\ + [file join $recipeexampledir modulefiles/foo/2.1] $submap] +cookbook_write_file "$recipeetc/siteconfig.tcl" [cookbook_read_example_file\ + [file join $recipeexampledir siteconfig.tcl] $submap] + +# +# point the test environment at the sandbox +# + +setenv_path_var MODULEPATH $recipemodpath +setenv_var MODULES_SITECONFIG "$recipeetc/siteconfig.tcl" +# give the 'remote' tag its abbreviation and drop it once a module is +# loaded, exactly as documented for the initrc installed alongside +# siteconfig.tcl -- 'module config' from an actual initrc file only runs +# through shell autoinit (see doc/example/sync-remote-appdir/initrc), +# which this testsuite does not drive; setting the equivalent +# configuration through its environment variable override has the same +# effect on modulecmd.tcl itself +setenv_var MODULES_TAG_ABBREV remote=R +setenv_var MODULES_NON_EXPORTABLE_TAGS remote + +# +# drive the exact command sequence the recipe doc's 'Usage example' section +# demonstrates, in the order it demonstrates it, checking the real stderr +# each command produces is similar to the output illustrated there (with +# its placeholder modulepath and unpredictable separator width widened +# into a regexp, see cookbook_output_re) +# + +set transcript [cookbook_parse_transcript $reciperst {Usage example}] + +set modpathre [cookbook_re_escape $recipemodpath] +set appdirbin "$localapps/$appdir/bin" +set appdirlib "$localapps/$appdir/lib" +set altname {foo/2.1&as|foo/default&as|foo/latest} +set origpath $env(PATH) +set outputsubs [list /path/to/modulefiles $modpathre] + +set loadans [list] +lappend loadans [list set LD_LIBRARY_PATH $appdirlib] +lappend loadans [list set __MODULES_LMALTNAME $altname] +lappend loadans [list set PATH "$appdirbin:$origpath"] +lappend loadans [list set _LMFILES_ $recipemodpath/foo/2.1] +lappend loadans [list set LOADEDMODULES foo/2.1] + +set unloadans [list] +lappend unloadans [list unset LD_LIBRARY_PATH] +lappend unloadans [list unset __MODULES_LMALTNAME] +lappend unloadans [list set PATH $origpath] +lappend unloadans [list unset _LMFILES_] +lappend unloadans [list unset LOADEDMODULES] + +# expected stdout (the env-var-assignment syntax a real shell would eval +# silently, so not part of what the doc transcript illustrates) for each +# step, in the same order 'module avail foo'/'load foo/2.1'/'list'/ +# 'unload foo/2.1'/'avail foo'/'load foo/2.1' appears in the doc +set stdoutexpected [list OK $loadans OK $unloadans OK $loadans] + +if {[llength $transcript] != [llength $stdoutexpected]} { + unresolved "sync-remote-appdir.rst 'Usage example' transcript now has\ + [llength $transcript] steps ([llength $stdoutexpected] expected) --\ + this test needs updating to match" + return +} + +set idx 0 +foreach step $transcript { + lassign $step cmd docoutput + if {![regexp {^module\s+(.*)$} $cmd -> modargs]} { + unresolved "unexpected non-'module' command in sync-remote-appdir.rst\ + 'Usage example' transcript: '$cmd'" + return + } + + testouterr_cmd_re sh $modargs [lindex $stdoutexpected $idx]\ + [cookbook_output_re $docoutput $outputsubs] + + # step 1 (0-indexed) is the first 'load foo/2.1': check the sync + # actually happened on disk, then simulate that it took hold in this + # shell, as steps 2 and 3 ('list'/'unload') need to see it + if {$idx == 1} { + if {[file exists "$localapps/.$appdir.synced"]} { + pass "sync-remote-appdir: local sync marker created" + } else { + fail "sync-remote-appdir: local sync marker created" + } + if {[file exists "$appdirbin/foo"]} { + pass "sync-remote-appdir: application directory synced locally" + } else { + fail "sync-remote-appdir: application directory synced locally" + } + + setenv_loaded_module foo/2.1 $recipemodpath/foo/2.1 + setenv_var __MODULES_LMALTNAME $altname + setenv_path_var PATH $appdirbin $origpath + setenv_var LD_LIBRARY_PATH $appdirlib + } + + # step 3 is 'unload foo/2.1': drop the simulated loaded state so steps + # 4 and 5 ('avail'/'load' again) see a clean environment, exactly as a + # real shell would once the unload's env changes above got eval'd + if {$idx == 3} { + unsetenv_loaded_module + unsetenv_var __MODULES_LMALTNAME + unsetenv_var LD_LIBRARY_PATH + setenv_var PATH $origpath + } + + incr idx +} + +# +# Cleanup +# + +file delete -force $recipedir +reset_test_env + +# vim:set tabstop=3 shiftwidth=3 expandtab autoindent: diff --git a/testsuite/cookbook.00-init/999-cleanup.exp b/testsuite/cookbook.00-init/999-cleanup.exp new file mode 100644 index 000000000..96c9a434d --- /dev/null +++ b/testsuite/cookbook.00-init/999-cleanup.exp @@ -0,0 +1,29 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: cookbook.00-init/%M% +# Revision: %I% +# First Edition: 2026/08/13 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Testuite testsequence +# Command: +# Sub-Command: +# +# Comment: %C{ +# Cleans out after end of all cookbook tests +# }C% +# +############################################################################## + +# remove the sandbox holding every recipe's fixture tree (see +# 005-init_ts.exp), including the sandboxed $HOME set up in 010-environ.exp +if {$verbose} { + send_user "\tDeleting $cookbooksandbox\n" +} +file delete -force $cookbooksandbox + +# vim:set tabstop=3 shiftwidth=3 expandtab autoindent: