From 181ac4ed90af613dba5e2513ab9bcbc98de25a27 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Sat, 29 Aug 2026 00:39:34 +0200 Subject: [PATCH 1/6] examples: Separate extraction from manual generation Add AutoDocExtractExamples, which collects a package's manual examples by running its makedoc.g with only the parts of the build that extraction needs. Everything is written below a temporary directory, so the extracted tests need not be stored in the repository, and they can be produced from a read-only package directory such as a system-wide install. The settings describing a manual stay in makedoc.g alone: the mode is requested through a global option, the mechanism already used for nopdf, or through the AUTODOC_EXTRACT_ONLY environment variable for a makedoc.g which ends in QUIT and therefore has to run as a separate process. Generated XML now records where each example was written, as an XML comment which the GAPDoc output backends ignore. Extraction rewrites GAPDoc's origin list from those markers, so a failing example points at the .autodoc file or AutoDoc comment which has to be edited, rather than at the XML generated from it. Fenced markdown examples sitting directly in section text are not covered, as the parser records no source positions there. Fixes #376 AI disclosure: prepared with Claude Code (Claude Opus 5), which investigated the issue, proposed the approach and wrote the implementation under review. Co-Authored-By: Claude Opus 5 --- CHANGES.md | 8 + doc/Tutorials.autodoc | 15 ++ gap/AutoDocMainFunction.gi | 42 +++- gap/DocumentationTree.gi | 7 +- gap/Examples.gd | 41 ++++ gap/Examples.gi | 87 ++++++++ gap/Magic.gd | 17 ++ gap/Magic.gi | 102 ++++++--- gap/Markdown.gi | 17 +- gap/Parser.gi | 4 + gap/ToolFunctions.gd | 4 + gap/ToolFunctions.gi | 208 ++++++++++++++++++ init.g | 2 + read.g | 2 + tst/examples.tst | 175 +++++++++++++++ tst/manual.expected/_Chapter_Reference.xml | 56 +++++ tst/manual.expected/_Chapter_Tutorials.xml | 15 ++ .../autoplain.expected/_Chapter_Test.xml | 3 + .../tst/plain_file.autodoc_test01.tst | 6 +- .../general.expected/_Chapter_SomeChapter.xml | 3 + .../general.expected/tst/general_test01.tst | 6 +- .../_Chapter_Examples_Chapter.xml | 8 + .../tst/paired_examples_test01.tst | 8 +- .../_Chapter_Examples_Chapter.xml | 8 + .../tst/paired_examples_test01.tst | 8 +- 25 files changed, 799 insertions(+), 53 deletions(-) create mode 100644 gap/Examples.gd create mode 100644 gap/Examples.gi create mode 100644 tst/examples.tst diff --git a/CHANGES.md b/CHANGES.md index 169d41fe..6068f71f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,14 @@ This file describes changes in the AutoDoc package. ## unreleased - Fix a spurious "chunk ... was defined but never inserted" warning for chunks that are only inserted from within the body of another chunk + - Add `AutoDocExtractExamples`, which extracts the manual examples of a + package into a temporary directory by running its `makedoc.g` without + building the manual. This allows running the examples from + `tst/testall.g` without storing generated `.tst` files in the + repository, and works from a read-only package directory + - Report the true origin of an extracted example: generated `.tst` files + now point at the `.autodoc` file or AutoDoc comment the example was + written in, instead of the intermediate XML file generated from it ## 2026.06.30 - Fix a regression in `.autodoc` parsing where Markdown-style headings diff --git a/doc/Tutorials.autodoc b/doc/Tutorials.autodoc index 6bb4ec71..9988999c 100644 --- a/doc/Tutorials.autodoc +++ b/doc/Tutorials.autodoc @@ -271,6 +271,21 @@ AutoDoc( rec( extract_examples := rec( subdir := "tst/generated" ) ) ); ``` This writes the extracted examples into tst/generated/ instead. +Either way the generated .tst files have to be regenerated whenever the +manual changes, which tempts one into committing them. To avoid that, extract +them when the tests run, using in your +tst/testall.g: +```@listing +LoadPackage( "mypkg" ); +dirs := DirectoriesPackageLibrary( "mypkg", "tst" ); +Add( dirs, AutoDocExtractExamples( "mypkg" ) ); +TestDirectory( dirs, rec( exitGAP := true ) ); +``` +This reads your makedoc.g, so the settings describing the manual are not +duplicated, but skips building the manual itself. Everything is written to a +temporary directory, so the extracted tests need not be committed and the tests +also run from a read-only package directory. + @Subsection Setting different &GAPDoc; options @SubsectionLabel Tut:IntegrateExisting:GapDocOptions diff --git a/gap/AutoDocMainFunction.gi b/gap/AutoDocMainFunction.gi index d60b71a2..7100adf7 100644 --- a/gap/AutoDocMainFunction.gi +++ b/gap/AutoDocMainFunction.gi @@ -360,9 +360,10 @@ end ); # The following function is based on code by Olexandr Konovalov BindGlobal("AUTODOC_ExtractMyManualExamples", -function( pkgname, pkgdir, docdir, main, files, opt ) - local tst, i, s, basename, name, output, ch, a, location, pos, comment, pkgdirString, - nonempty_units_found, number_of_digits, lpkgname, tstdir; +function( pkgname, pkgdir, docdir, main, files, opt, roots ) + local tst, i, s, basename, name, output, ch, a, location, pos, comment, + nonempty_units_found, number_of_digits, lpkgname, tstdir, composed, + prefixes, prefix; Info(InfoAutoDoc, 1, "Extracting manual examples for ", pkgname, " package ..."); lpkgname := LowercaseString(pkgname); @@ -371,9 +372,20 @@ function( pkgname, pkgdir, docdir, main, files, opt ) if not EndsWith(main, ".xml") then main := Concatenation( main, ".xml" ); fi; - tst:=ExtractExamples( docdir, main, files, opt.units ); + # This is GAPDoc's ExtractExamples, with a pass over the origin list added + # so that examples AutoDoc generated report the file they came from rather + # than the intermediate XML file. + composed := ComposedDocument( "GAPDoc", docdir, main, files, true ); + AUTODOC_RemapSourcePositions( composed[1], composed[2] ); + tst := ExtractExamplesXMLTree( + ParseTreeXMLString( composed[1], composed[2] ), opt.units ); Info(InfoAutoDoc, 1, Length(tst), " ", LowercaseString( opt.units ), "s detected"); - pkgdirString := Filename(pkgdir, ""); + # Directories a source file may live under, most specific first. Locations + # are reported relative to whichever matches, so that generated .tst files + # do not depend on where the package is installed. + prefixes := Concatenation( roots, [ pkgdir, + Directory(AUTODOC_CurrentDirectory()) ] ); + prefixes := List( prefixes, d -> Filename( d, "" ) ); if IsDirectory( opt.subdir ) then tstdir := Filename( opt.subdir, "" ); @@ -434,9 +446,20 @@ function( pkgname, pkgdir, docdir, main, files, opt ) AppendTo(output, "gap> START_TEST(\"", basename, "\");\n\n"); for a in ch do location := a[2][1]; - if StartsWith(location, pkgdirString) then - comment := location{[ Length(pkgdirString)+1 .. Length(location) ]}; + if not StartsWith(location, "/") then + # Already reproducible: AutoDoc recorded this position itself, + # or GAPDoc resolved it relative to the documentation dir. + comment := location; else + comment := fail; + for prefix in prefixes do + if StartsWith(location, prefix) then + comment := location{[ Length(prefix)+1 .. Length(location) ]}; + break; + fi; + od; + fi; + if comment = fail then pos := PositionSublist(location, LowercaseString(pkgname)); if pos <> fail then comment := location{[ pos+Length(pkgname)+1 .. Length(location) ]}; @@ -445,7 +468,10 @@ function( pkgname, pkgdir, docdir, main, files, opt ) if pos <> fail then comment := location{[ pos+2 .. Length(location) ]}; else - Error("oops"); + # Sources outside all of the above, e.g. worksheet + # inputs. The bare filename is still more useful than + # an absolute path, and keeps the output reproducible. + comment := Last( SplitString( location, "/" ) ); fi; fi; fi; diff --git a/gap/DocumentationTree.gi b/gap/DocumentationTree.gi index 8af53b2f..c1b07de4 100644 --- a/gap/DocumentationTree.gi +++ b/gap/DocumentationTree.gi @@ -666,7 +666,12 @@ end ); InstallMethod( WriteDocumentation, [ IsTreeForDocumentationVerbatimNodeRep, IsStream ], function( node, filestream ) - local line, attr_name; + local line, attr_name, marker; + + marker := AUTODOC_SourceMarker( node ); + if marker <> fail then + AppendTo( filestream, marker, "\n" ); + fi; AppendTo( filestream, "<", node!.element_name ); for attr_name in Set( RecNames( node!.attributes ) ) do diff --git a/gap/Examples.gd b/gap/Examples.gd new file mode 100644 index 00000000..2e8535d8 --- /dev/null +++ b/gap/Examples.gd @@ -0,0 +1,41 @@ +# AutoDoc: Generate documentation from GAP source code +# +# Copyright of AutoDoc belongs to its developers. +# Please refer to the COPYRIGHT file for details. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +#! @Chapter Reference +#! @Section Extracting manual examples + +#! @Description +#! Extracts the examples from the manual of the package pkg and +#! returns the directory holding the resulting .tst files. +#! +#! pkg is either the name of a package or a directory object pointing +#! at one. The optional argument makedoc names the script to read, +#! and defaults to makedoc.g. +#! +#! The package's own makedoc.g is used as-is, so the settings which +#! describe the manual — its source files, scaffolding and +#! extract_examples options — are not duplicated. Only the parts of +#! the manual needed to collect the examples are built, no HTML or PDF is +#! produced, and everything is written to a temporary directory, so this +#! works even when the package directory is read-only. +#! +#! This is meant to be used from a package's tst/testall.g, so that +#! extracted tests need not be committed to the repository: +#! +#! +#! Note that makedoc.g is read in the usual way, so any other work it +#! performs still happens; and a script ending in QUIT cannot be used +#! this way. +#! @Returns a directory +#! @Arguments pkg[, makedoc] +DeclareGlobalFunction( "AutoDocExtractExamples" ); + +DeclareGlobalFunction( "AUTODOC_ExtractOnlyDirectory" ); diff --git a/gap/Examples.gi b/gap/Examples.gi new file mode 100644 index 00000000..87a0b317 --- /dev/null +++ b/gap/Examples.gi @@ -0,0 +1,87 @@ +# AutoDoc: Generate documentation from GAP source code +# +# Copyright of AutoDoc belongs to its developers. +# Please refer to the COPYRIGHT file for details. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +# Environment variable equivalent of the AutoDocExtractOnly global option. It +# exists so that a makedoc.g ending in QUIT can still be driven, by running it +# as a separate GAP process. +BindGlobal( "AUTODOC_EXTRACT_ONLY_ENVVAR", "AUTODOC_EXTRACT_ONLY" ); + +BindGlobal( "AUTODOC_DEFAULT_MAKEDOC_FILE", "makedoc.g" ); + +# The scratch directory to work in, or fail if AutoDoc should build the manual +# normally. Both spellings carry the directory; passing just `true` or `1` +# asks for extract-only mode without naming one. +InstallGlobalFunction( "AUTODOC_ExtractOnlyDirectory", +function() + local value; + + value := ValueOption( "AutoDocExtractOnly" ); + if value = fail and + IsBound( GAPInfo.SystemEnvironment.( AUTODOC_EXTRACT_ONLY_ENVVAR ) ) then + value := GAPInfo.SystemEnvironment.( AUTODOC_EXTRACT_ONLY_ENVVAR ); + fi; + + if value = fail or value = false then + return fail; + fi; + + if IsDirectory( value ) then + return value; + fi; + + if value = true or value = "" or value = "1" then + return DirectoryTemporary(); + fi; + + if not IsString( value ) then + Error( "AutoDocExtractOnly must be true, a path, or a directory object" ); + fi; + + AUTODOC_CreateDirIfMissing( value ); + return Directory( value ); +end ); + +## +InstallGlobalFunction( "AutoDocExtractExamples", +function( pkg, makedoc... ) + local pkgdir, script, scratch, olddir; + + if Length( makedoc ) > 0 then + script := makedoc[ 1 ]; + else + script := AUTODOC_DEFAULT_MAKEDOC_FILE; + fi; + + if IsDirectory( pkg ) then + pkgdir := pkg; + elif IsString( pkg ) then + pkgdir := DirectoriesPackageLibrary( pkg, "" ); + if pkgdir = [ ] then + Error( "could not locate package ", pkg ); + fi; + pkgdir := pkgdir[ 1 ]; + else + Error( "pkg must be a package name or a directory object" ); + fi; + + script := Filename( pkgdir, script ); + if script = fail or not IsReadableFile( script ) then + Error( "could not read ", script ); + fi; + + scratch := DirectoryTemporary(); + + # AutoDoc() with no arguments picks up PackageInfo.g from the working + # directory, and makedoc.g scripts name their inputs relative to the + # package, so run the script from there. + olddir := AUTODOC_CurrentDirectory(); + ChangeDirectoryCurrent( Filename( pkgdir, "" ) ); + Read( script : AutoDocExtractOnly := scratch, nopdf ); + ChangeDirectoryCurrent( olddir ); + + return Directory( Filename( scratch, "tst" ) ); +end ); diff --git a/gap/Magic.gd b/gap/Magic.gd index 756e4785..f6122396 100644 --- a/gap/Magic.gd +++ b/gap/Magic.gd @@ -441,6 +441,23 @@ #! Also, if the environment variable `NOPDF` is set, then &AutoDoc; #! behaves as if the global option nopdf had been enabled. #! +#! AutoDocExtractOnly +#! +#! If this global option is set, &AutoDoc; builds only as much of the +#! manual as is needed to collect its examples: no HTML, PDF or manual +#! index is produced, and all output is written below the given scratch +#! directory instead of the package. The extracted tests end up in its +#! tst subdirectory. +#!

+#! The value is a directory, or `true` to have one chosen automatically. +#! If the environment variable `AUTODOC_EXTRACT_ONLY` is set, &AutoDoc; +#! behaves as if this option had been given; that is useful for a +#! makedoc.g which ends in QUIT and therefore has to be run +#! as a separate process. +#!

+#! Rather than setting this by hand, use +#! . +#! #! relativePath #! #! This has the same effect as gapdoc.gap_root_relative_path, but diff --git a/gap/Magic.gi b/gap/Magic.gi index 1d826cfa..4bb5759e 100644 --- a/gap/Magic.gi +++ b/gap/Magic.gi @@ -90,7 +90,7 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) pkgdirstr, docdirstr, title_page, tree, position_document_class, - args, used_legacy_value_options; + args, used_legacy_value_options, extract_only, extract_roots; # # Deprecated feature: Check for user supplied global options. If present, @@ -109,6 +109,12 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) Print("#W passing options via GAP's global options system is deprecated; use an option record instead\n"); fi; + # + # Extract-only mode: build just enough of the manual, in a scratch + # directory, to collect its examples. See AutoDocExtractExamples. + # + extract_only := AUTODOC_ExtractOnlyDirectory(); + # # Setup the output directory # @@ -144,6 +150,20 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) fi; fi; + if extract_only <> fail then + # ComposedDocument resolves relative <#Include SYSTEM ...> against a + # single directory, and handwritten doc files routinely include + # generated ones by relative name. So stage the whole documentation + # directory and generate into the copy; the package directory is then + # never written to, and may even be read-only. + AUTODOC_StageDirectory( doc_dir, Directory( Filename( extract_only, "doc" ) ) ); + doc_dir := Directory( Filename( extract_only, "doc" ) ); + + # doc_dir no longer lies below pkgdir, so paths relative to it would + # be wrong; fall back to the absolute-path branches further down. + Unbind( doc_dir_rel ); + fi; + # Ensure the output directory exists, create it if necessary AUTODOC_CreateDirIfMissing(Filename(doc_dir, "")); @@ -333,7 +353,13 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) tree := DocumentationTree( ); if IsBound( autodoc ) then - AutoDocScanFiles( autodoc.files, pkgname, tree ); + # Hand the parser both the real path and a reproducible name to + # report positions under; the latter ends up in generated files. + AutoDocScanFiles( + List( autodoc.files, + f -> rec( path := f, + display := AUTODOC_RelativeSourcePath( f, pkgdir ) ) ), + pkgname, tree ); fi; if is_worksheet then @@ -564,30 +590,36 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) fi; # Finally, invoke GAPDoc - CallFuncList( MakeGAPDocDoc, args ); - - # NOTE: We cannot just write CopyHTMLStyleFiles(doc_dir) here, as - # CopyHTMLStyleFiles its argument directly to Directory(), leading - # to an error in all GAP versions up to and including 4.8.6. This - # will be fixed with GAP 4.9, where Directory() is made idempotent. - CopyHTMLStyleFiles( Filename( doc_dir, "" ) ); - - # The following (undocumented) API is there for compatibility - # with old-style gapmacro.tex based package manuals. It - # produces a manual.lab file which those packages can use if - # they wish to link to things in the manual we are currently - # generating. This can probably be removed eventually, but for - # now, doing it does not hurt. - - # FIXME: It seems that this command does not work if pdflatex - # is not present. Maybe we should remove it. - - if IsBound( gapdoc.SixFile ) then - file := Filename(pkgdir, gapdoc.SixFile); - if file = fail or not IsReadableFile(file) then - Error("could not open `", file, "' for package `", pkgname, "'.\n"); + if extract_only <> fail then + # Only the composed XML matters here; producing HTML, PDF and the + # manual index would be wasted work. + Info( InfoAutoDoc, 1, "Skipping manual generation, extracting examples only" ); + else + CallFuncList( MakeGAPDocDoc, args ); + + # NOTE: We cannot just write CopyHTMLStyleFiles(doc_dir) here, as + # CopyHTMLStyleFiles its argument directly to Directory(), leading + # to an error in all GAP versions up to and including 4.8.6. This + # will be fixed with GAP 4.9, where Directory() is made idempotent. + CopyHTMLStyleFiles( Filename( doc_dir, "" ) ); + + # The following (undocumented) API is there for compatibility + # with old-style gapmacro.tex based package manuals. It + # produces a manual.lab file which those packages can use if + # they wish to link to things in the manual we are currently + # generating. This can probably be removed eventually, but for + # now, doing it does not hurt. + + # FIXME: It seems that this command does not work if pdflatex + # is not present. Maybe we should remove it. + + if IsBound( gapdoc.SixFile ) then + file := Filename(pkgdir, gapdoc.SixFile); + if file = fail or not IsReadableFile(file) then + Error("could not open `", file, "' for package `", pkgname, "'.\n"); + fi; + GAPDocManualLabFromSixFile( gapdoc.bookname, file ); fi; - GAPDocManualLabFromSixFile( gapdoc.bookname, file ); fi; fi; @@ -604,6 +636,16 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) fi; fi; + if extract_only <> fail and IsBound( gapdoc ) then + # Extracting is the whole point here, so do it even for packages whose + # makedoc.g does not ask for it, and collect the result outside the + # package. Without GAPDoc there is no document to extract from. + if not IsBound( extract_examples ) then + extract_examples := rec( ); + fi; + extract_examples.subdir := Directory( Filename( extract_only, "tst" ) ); + fi; + if IsBound( extract_examples ) then if is_worksheet then # HACK: not even sure this is really what we want for worksheets, but @@ -622,7 +664,15 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) if not IsBound( extract_examples.skip_empty_in_numbering ) then extract_examples.skip_empty_in_numbering := true; fi; - AUTODOC_ExtractMyManualExamples( pkgname, pkgdir, doc_dir, gapdoc.main, gapdoc.files, extract_examples ); + if extract_only <> fail then + # Sources were staged below the scratch directory, mirroring the + # package layout, so report them relative to it. + extract_roots := [ extract_only ]; + else + extract_roots := [ ]; + fi; + AUTODOC_ExtractMyManualExamples( pkgname, pkgdir, doc_dir, gapdoc.main, + gapdoc.files, extract_examples, extract_roots ); fi; return true; diff --git a/gap/Markdown.gi b/gap/Markdown.gi index 8dfe6d1e..296418e4 100644 --- a/gap/Markdown.gi +++ b/gap/Markdown.gi @@ -107,7 +107,7 @@ BindGlobal( "AUTODOC_ConvertFencedMarkdownBlocks", local converted_source_positions, converted_string_list, i, skipped, string_list, trimmed_line, fence_char, fence_length, info_string, fence_element, code_block, - fence_content, source_positions; + fence_content, fence_node, fence_position, source_positions; string_list := arg[ 1 ]; if Length( arg ) > 1 then @@ -163,9 +163,18 @@ BindGlobal( "AUTODOC_ConvertFencedMarkdownBlocks", Add( fence_content, Chomp( string_list[ i ] ) ); i := i + 1; od; - Add( converted_string_list, - DocumentationVerbatim( fence_element, rec( ), fence_content ) ); - Add( converted_source_positions, source_positions[ i - Length( fence_content ) - 1 ] ); + fence_node := DocumentationVerbatim( fence_element, rec( ), fence_content ); + # The opening fence sits one line above the collected content. + fence_position := source_positions[ i - Length( fence_content ) - 1 ]; + if fence_position <> fail then + fence_node!.source_position := fence_position; + fence_node!.source_end_position := rec( + filename := fence_position.filename, + line := fence_position.line + Length( fence_content ) + 1 + ); + fi; + Add( converted_string_list, fence_node ); + Add( converted_source_positions, fence_position ); if code_block = true then i := i + 1; continue; diff --git a/gap/Parser.gi b/gap/Parser.gi index 7c934001..4dc49046 100644 --- a/gap/Parser.gi +++ b/gap/Parser.gi @@ -762,6 +762,7 @@ InstallGlobalFunction( AutoDoc_Parser_ReadFiles, local temp_string_list, temp_curr_line, temp_pos_comment, is_following_line, item_temp, example_node, end_command; example_node := DocumentationExample( element_name ); + example_node!.source_position := CurrentSourcePosition(); temp_string_list := example_node!.content; end_command := Concatenation( "@End", element_name ); is_following_line := false; @@ -794,6 +795,7 @@ InstallGlobalFunction( AutoDoc_Parser_ReadFiles, continue; fi; od; + example_node!.source_end_position := CurrentSourcePosition(); return example_node; end; ReadSessionExample := function( element_name, plain_text_mode ) @@ -801,6 +803,7 @@ InstallGlobalFunction( AutoDoc_Parser_ReadFiles, is_following_line, item_temp, example_node, incorporate_this_line, end_command; example_node := DocumentationExample( element_name ); + example_node!.source_position := CurrentSourcePosition(); temp_string_list := example_node!.content; end_command := Concatenation( "@End", element_name, "Session" ); while true do @@ -823,6 +826,7 @@ InstallGlobalFunction( AutoDoc_Parser_ReadFiles, Add( temp_string_list, temp_curr_line ); fi; od; + example_node!.source_end_position := CurrentSourcePosition(); return example_node; end; command_function_record := rec( diff --git a/gap/ToolFunctions.gd b/gap/ToolFunctions.gd index 224cca59..753b5293 100644 --- a/gap/ToolFunctions.gd +++ b/gap/ToolFunctions.gd @@ -6,6 +6,10 @@ # SPDX-License-Identifier: GPL-2.0-or-later DeclareGlobalFunction( "AUTODOC_CreateDirIfMissing" ); +DeclareGlobalFunction( "AUTODOC_RelativeSourcePath" ); +DeclareGlobalFunction( "AUTODOC_StageDirectory" ); +DeclareGlobalFunction( "AUTODOC_SourceMarker" ); +DeclareGlobalFunction( "AUTODOC_RemapSourcePositions" ); DeclareGlobalFunction( "AUTODOC_CurrentDirectory" ); DeclareGlobalFunction( "AUTODOC_LineStartsCDATA" ); DeclareGlobalFunction( "AUTODOC_LineEndsCDATA" ); diff --git a/gap/ToolFunctions.gi b/gap/ToolFunctions.gi index 01d0da4b..631f882f 100644 --- a/gap/ToolFunctions.gi +++ b/gap/ToolFunctions.gi @@ -553,3 +553,211 @@ function(arg) fi; return result; end); + +# Render a source file path for display and for recording in generated files: +# relative to the package directory when it lies below it, else relative to +# the working directory, else the bare filename. Absolute paths would make +# generated output depend on where the package happens to live. +InstallGlobalFunction( "AUTODOC_RelativeSourcePath", +function( path, pkgdir ) + local prefix, candidate; + + for candidate in [ pkgdir, Directory( AUTODOC_CurrentDirectory() ) ] do + prefix := Filename( candidate, "" ); + if prefix <> fail and Length( prefix ) > 1 and StartsWith( path, prefix ) then + return path{ [ Length( prefix ) + 1 .. Length( path ) ] }; + fi; + od; + + return Last( SplitString( path, "/" ) ); +end ); + +# Elements whose content is a manual example, i.e. those GAPDoc's +# ExtractExamplesXMLTree collects. Only these are worth annotating. +BindGlobal( "AUTODOC_EXAMPLE_ELEMENTS", [ "Example", "Log" ] ); + +BindGlobal( "AUTODOC_SOURCE_MARKER_PREFIX", "" ); +BindGlobal( "AUTODOC_CDATA_CLOSE", "]]> +# GAPDoc parses these into XMLCOMMENT nodes which every output backend +# ignores, so they are invisible in the built manual; AutoDoc reads them back +# in AUTODOC_RemapSourcePositions to report the true origin of an example. +# +# Returns fail if the node carries no position, is not an example, or if the +# path cannot be represented in an XML comment. +InstallGlobalFunction( "AUTODOC_SourceMarker", +function( node ) + local position, end_position, text; + + if not IsBound( node!.element_name ) or + not node!.element_name in AUTODOC_EXAMPLE_ELEMENTS or + not IsBound( node!.source_position ) or + node!.source_position = fail then + return fail; + fi; + + position := node!.source_position; + if IsBound( node!.source_end_position ) and node!.source_end_position <> fail then + end_position := node!.source_end_position; + else + end_position := position; + fi; + + # XML forbids "--" inside comments, and ">" would end ours early. Rather + # than mangle the path, drop the marker and fall back to reporting the + # generated XML file, as AutoDoc did before markers existed. + if PositionSublist( position.filename, "--" ) <> fail or + '>' in position.filename then + Info( InfoAutoDoc, 1, "WARNING: cannot record source position for ", + position.filename, ", path is not valid inside an XML comment" ); + return fail; + fi; + + text := Concatenation( + AUTODOC_SOURCE_MARKER_PREFIX, + position.filename, ":", + String( position.line ), "-", String( end_position.line ), + AUTODOC_SOURCE_MARKER_SUFFIX + ); + return text; +end ); + +# Rewrite GAPDoc's origin list in place so that text AutoDoc generated is +# attributed to the file it was generated *from*. +# +# `str` is a composed document as returned by ComposedDocument, and `src` the +# accompanying list of [position, filename, line] triples which +# OriginalPositionDocument searches. Each marker claims the element that +# follows it, up to and including the line closing its CDATA block. Within +# that region we report the recorded start line, except for the closing line, +# which gets the recorded end line. ExtractExamplesXMLTree only ever looks up +# the start and stop of an example, so both of its lookups land exactly. +# +# <- marker at position p +# 1+1; +# 2 +# ]]> <- reported as gap/Foo.gd:141 +InstallGlobalFunction( "AUTODOC_RemapSourcePositions", +function( str, src ) + local marker_start, marker_end, region_start, region_end, colon, dash, + body, filename, start_line, end_line, first, last, i; + + marker_start := PositionSublist( str, AUTODOC_SOURCE_MARKER_PREFIX ); + + while marker_start <> fail do + marker_end := PositionSublist( str, AUTODOC_SOURCE_MARKER_SUFFIX, marker_start ); + if marker_end = fail then + break; + fi; + marker_end := marker_end + Length( AUTODOC_SOURCE_MARKER_SUFFIX ) - 1; + region_start := marker_start; + + body := str{ [ marker_start + Length( AUTODOC_SOURCE_MARKER_PREFIX ) + .. marker_end - Length( AUTODOC_SOURCE_MARKER_SUFFIX ) ] }; + + # Split "path/to/file.gd:137-141" from the right, so that paths + # containing ':' or '-' survive. + colon := Length( body ); + while colon > 0 and body[ colon ] <> ':' do colon := colon - 1; od; + dash := Length( body ); + while dash > colon and body[ dash ] <> '-' do dash := dash - 1; od; + + marker_start := PositionSublist( str, AUTODOC_SOURCE_MARKER_PREFIX, marker_end ); + + if colon = 0 or dash <= colon then + continue; + fi; + + filename := body{ [ 1 .. colon - 1 ] }; + start_line := Int( body{ [ colon + 1 .. dash - 1 ] } ); + end_line := Int( body{ [ dash + 1 .. Length( body ) ] } ); + if start_line = fail or end_line = fail then + continue; + fi; + + # The marker describes exactly one element, which AutoDoc always + # writes CDATA-wrapped, so its closing line is the first one holding + # "]]> fail and region_end > marker_start ) then + continue; + fi; + region_end := Position( str, '\n', region_end ); + if region_end = fail then + region_end := Length( str ); + fi; + + first := PositionSorted( src, [ region_start ] ); + last := PositionSorted( src, [ region_end ] ) - 1; + for i in [ first .. last ] do + if not IsBound( src[ i ] ) then + continue; + fi; + src[ i ][ 2 ] := filename; + if i = last then + src[ i ][ 3 ] := end_line; + else + src[ i ][ 3 ] := start_line; + fi; + od; + od; +end ); + +# Output of a previous manual build. Staging a documentation directory copies +# its inputs only; these are large, regenerated anyway, and never read back. +# Kept in sync with the `clean` target of the Makefile. +BindGlobal( "AUTODOC_BUILD_ARTIFACT_EXTENSIONS", + [ "aux", "bbl", "blg", "brf", "css", "dvi", "html", "idx", "ilg", "ind", + "js", "lab", "log", "out", "pdf", "pnr", "ps", "six", "tex", "toc", + "txt" ] ); + +# Recursively copy the contents of directory `src` into directory `dst`, +# skipping build artifacts. +# +# GAP has no CopyFile, so this goes through StringFile/FileString. Those read +# and write raw bytes, so binary inputs such as images survive. +InstallGlobalFunction( "AUTODOC_StageDirectory", +function( src, dst ) + local entry, entries, source_path, target_path, contents; + + AUTODOC_CreateDirIfMissing( Filename( dst, "" ) ); + + entries := DirectoryContents( src ); + if entries = fail then + # Nothing to stage; a package may not have a doc directory yet. + return; + fi; + + for entry in entries do + if entry = "." or entry = ".." then + continue; + fi; + + source_path := Filename( src, entry ); + + if IsDirectoryPath( source_path ) then + AUTODOC_StageDirectory( Directory( source_path ), + Directory( Filename( dst, entry ) ) ); + continue; + fi; + + if AUTODOC_GetSuffix( entry ) in AUTODOC_BUILD_ARTIFACT_EXTENSIONS then + continue; + fi; + + contents := StringFile( source_path ); + if contents = fail then + continue; + fi; + + target_path := Filename( dst, entry ); + if FileString( target_path, contents ) = fail then + Error( "failed to stage ", source_path, " to ", target_path ); + fi; + od; +end ); diff --git a/init.g b/init.g index ce6f8135..29d314c3 100644 --- a/init.g +++ b/init.g @@ -15,4 +15,6 @@ ReadPackage( "AutoDoc", "gap/ToolFunctions.gd" ); ReadPackage( "AutoDoc", "gap/Magic.gd" ); +ReadPackage( "AutoDoc", "gap/Examples.gd" ); + ReadPackage( "AutoDoc", "gap/Markdown.gd" ); diff --git a/read.g b/read.g index fcc1d7ff..5acaf36d 100644 --- a/read.g +++ b/read.g @@ -15,4 +15,6 @@ ReadPackage( "AutoDoc", "gap/AutoDocMainFunction.gi" ); ReadPackage( "AutoDoc", "gap/Magic.gi" ); +ReadPackage( "AutoDoc", "gap/Examples.gi" ); + ReadPackage( "AutoDoc", "gap/Markdown.gi" ); diff --git a/tst/examples.tst b/tst/examples.tst new file mode 100644 index 00000000..67118b28 --- /dev/null +++ b/tst/examples.tst @@ -0,0 +1,175 @@ +# +# test example extraction +# +gap> START_TEST( "examples.tst" ); + +# +# Extracted tests must point at the file the example was written in, not at +# the intermediate XML file AutoDoc generated from it. +# +gap> tmpdir := Filename(DirectoryTemporary(), "autodoc-examples-sources");; +gap> if IsDirectoryPath(tmpdir) then RemoveDirectoryRecursively(tmpdir); fi; +gap> AUTODOC_CreateDirIfMissing(tmpdir); +true +gap> sheetdir := DirectoriesPackageLibrary( +> "AutoDoc", "tst/worksheets/paired-examples.sheet" )[1];; +gap> filenames := DirectoryContents(sheetdir);; +gap> filenames := Filtered(filenames, f -> f <> "." and f <> "..");; +gap> filenames := List(filenames, f -> Filename(sheetdir, f));; +gap> old := InfoLevel(InfoAutoDoc);; oldgapdoc := InfoLevel(InfoGAPDoc);; +gap> SetInfoLevel(InfoAutoDoc, 0); SetInfoLevel(InfoGAPDoc, 0); +gap> AutoDocWorksheet(filenames, +> rec( dir := Directory(tmpdir), extract_examples := true ) : nopdf ); +gap> SetInfoLevel(InfoAutoDoc, old); SetInfoLevel(InfoGAPDoc, oldgapdoc); +gap> lines := SplitString(StringFile( +> Filename(Directory(tmpdir), "tst/paired_examples_test01.tst")), "\n");; + +# The four examples start on lines 8, 13, 18 and 23 of worksheet.g. +gap> Perform(Filtered(lines, +> l -> StartsWith(l, "# ") and PositionSublist(l, ":") <> fail), Display); +# tst/worksheets/paired-examples.sheet/worksheet.g:8-11 +# tst/worksheets/paired-examples.sheet/worksheet.g:13-16 +# tst/worksheets/paired-examples.sheet/worksheet.g:18-21 +# tst/worksheets/paired-examples.sheet/worksheet.g:23-26 + +# No location may name a generated file. +gap> ForAny(lines, l -> PositionSublist(l, "_Chapter_") <> fail); +false +gap> RemoveDirectoryRecursively(tmpdir); +true + +# +# Fenced markdown examples are recorded too, wherever the parser tracks +# source positions for the surrounding text. +# +gap> tmpdir := Filename(DirectoryTemporary(), "autodoc-examples-fence");; +gap> if IsDirectoryPath(tmpdir) then RemoveDirectoryRecursively(tmpdir); fi; +gap> AUTODOC_CreateDirIfMissing(Concatenation(tmpdir, "/src")); +true +gap> source := Concatenation(tmpdir, "/src/fence.g");; +gap> FileString(source, Concatenation( +> "#! @Title Fence Test\n", +> "#! @Date 2026-01-01\n", +> "#! @Chapter Ch\n", +> "#! @Section Sec\n", +> "\n", +> "#! @Description\n", +> "#! Some description text.\n", +> "#! ```@example\n", +> "#! gap> 1+1;\n", +> "#! 2\n", +> "#! ```\n", +> "#! @Arguments x\n", +> "DeclareOperation( \"AutoDocFenceDemo\", [ IsInt ] );\n" )) <> fail; +true +gap> old := InfoLevel(InfoAutoDoc);; oldgapdoc := InfoLevel(InfoGAPDoc);; +gap> SetInfoLevel(InfoAutoDoc, 0); SetInfoLevel(InfoGAPDoc, 0); +gap> AutoDocWorksheet([ source ], +> rec( dir := Directory(Concatenation(tmpdir, "/out")), +> extract_examples := true ) : nopdf ); +gap> SetInfoLevel(InfoAutoDoc, old); SetInfoLevel(InfoGAPDoc, oldgapdoc); +gap> lines := SplitString(StringFile(Filename(Directory(tmpdir), +> "out/tst/fence_test01.tst")), "\n");; + +# The fence opens on line 8 and closes on line 11. +gap> Perform(Filtered(lines, +> l -> StartsWith(l, "# ") and PositionSublist(l, ":") <> fail), Display); +# fence.g:8-11 +gap> RemoveDirectoryRecursively(tmpdir); +true + +# +# AutoDocExtractExamples runs a package's own makedoc.g in extract-only mode: +# no manual is built, and nothing is written into the package directory. +# +gap> pkgdir := DirectoriesPackageLibrary( "AutoDoc", "tst/AutoDocTest" )[1];; +gap> docdir := Directory( Filename( pkgdir, "doc" ) );; +gap> before := Set( DirectoryContents( docdir ) );; +gap> old := InfoLevel(InfoAutoDoc);; oldgapdoc := InfoLevel(InfoGAPDoc);; +gap> SetInfoLevel(InfoAutoDoc, 0); SetInfoLevel(InfoGAPDoc, 0); +gap> tstdir := AutoDocExtractExamples( pkgdir, "makedoc-examples-chapter.g" );; +gap> SetInfoLevel(InfoAutoDoc, old); SetInfoLevel(InfoGAPDoc, oldgapdoc); +gap> IsDirectory( tstdir ); +true + +# The package directory must be untouched: no generated XML, no .tst files. +gap> Set( DirectoryContents( docdir ) ) = before; +true +gap> Filtered( DirectoryContents( Directory( Filename( pkgdir, "tst" ) ) ), +> f -> EndsWith( f, ".tst" ) ); +[ ] + +# The extracted tests match the reference output of a full manual build. +gap> expected := Filename( pkgdir, +> "tst/examples-chapter.expected/autodoctest01.tst" );; +gap> AUTODOC_Diff( "-u", expected, Filename( tstdir, "autodoctest01.tst" ) ); +0 + +# +# Extraction must work when the package directory cannot be written to, as +# happens for packages installed system-wide. +# +gap> frozen := Filename(DirectoryTemporary(), "autodoc-examples-frozen");; +gap> if IsDirectoryPath(frozen) then RemoveDirectoryRecursively(frozen); fi; +gap> Exec(Concatenation("cp -R \"", Filename(pkgdir, ""), "\" \"", frozen, "\"")); +gap> Exec(Concatenation("chmod -R a-w \"", frozen, "\"")); + +# Running as root defeats chmod, so only assert when the dir really is locked. +gap> locked := not IsWritableFile(Concatenation(frozen, "/doc"));; +gap> if locked then +> SetInfoLevel(InfoAutoDoc, 0); SetInfoLevel(InfoGAPDoc, 0); +> tstdir := AutoDocExtractExamples( +> Directory(frozen), "makedoc-examples-chapter.g" ); +> SetInfoLevel(InfoAutoDoc, old); SetInfoLevel(InfoGAPDoc, oldgapdoc); +> if AUTODOC_Diff( "-u", expected, Filename(tstdir, "autodoctest01.tst") ) <> 0 then +> Error("extraction from a read-only package dir gave unexpected output"); +> fi; +> fi; +gap> Exec(Concatenation("chmod -R u+w \"", frozen, "\"")); +gap> RemoveDirectoryRecursively(frozen); +true + +# +# AUTODOC_SourceMarker renders an example's provenance, and declines when it +# cannot: "--" may not appear inside an XML comment. +# +gap> node := DocumentationExample( "Example" );; +gap> node!.source_position := rec( filename := "gap/Foo.gd", line := 5 );; +gap> node!.source_end_position := rec( filename := "gap/Foo.gd", line := 9 );; +gap> AUTODOC_SourceMarker( node ); +"" +gap> listing := DocumentationVerbatim( "Listing", rec( ), [ ] );; +gap> listing!.source_position := node!.source_position;; +gap> AUTODOC_SourceMarker( listing ); +fail +gap> node!.source_position := rec( filename := "gap/a--b.gd", line := 5 );; +gap> SetInfoLevel( InfoAutoDoc, 0 ); +gap> AUTODOC_SourceMarker( node ); +fail +gap> SetInfoLevel( InfoAutoDoc, old ); + +# +# AUTODOC_RemapSourcePositions rewrites GAPDoc's origin list, so that the +# start and end of the example both resolve into the original source. +# +gap> str := Concatenation( +> "\n", +> " "gap> 1+1;\n", +> "2\n", +> "]]>\n" );; +gap> starts := Concatenation( [ 1 ], List( Positions( str, '\n' ), p -> p + 1 ) );; +gap> src := List( [ 1 .. Length( starts ) ], +> i -> [ starts[i], "_Chapter_Generated.xml", i ] );; +gap> AUTODOC_RemapSourcePositions( str, src ); +gap> OriginalPositionDocument( src, PositionSublist( str, " OriginalPositionDocument( src, PositionSublist( str, "]]>" ) ); +[ "gap/Foo.gd", 9 ] + +# Text past the example keeps its own origin. +gap> Last( src ); +[ 82, "_Chapter_Generated.xml", 6 ] + +# +gap> STOP_TEST( "examples.tst" ); diff --git a/tst/manual.expected/_Chapter_Reference.xml b/tst/manual.expected/_Chapter_Reference.xml index 71c14bc5..0ee14f2b 100644 --- a/tst/manual.expected/_Chapter_Reference.xml +++ b/tst/manual.expected/_Chapter_Reference.xml @@ -22,6 +22,7 @@ A simple worksheet file can define title-page information and chapter content directly in the source file, including example blocks. If this is stored in worksheet.g, you can generate documentation via + @@ -41,6 +42,44 @@ +

+Extracting manual examples + + + + a directory + + + Extracts the examples from the manual of the package pkg and + returns the directory holding the resulting .tst files. +

+ pkg is either the name of a package or a directory object pointing + at one. The optional argument makedoc names the script to read, + and defaults to makedoc.g. +

+ The package's own makedoc.g is used as-is, so the settings which + describe the manual — its source files, scaffolding and + extract_examples options — are not duplicated. Only the parts of + the manual needed to collect the examples are built, no HTML or PDF is + produced, and everything is written to a temporary directory, so this + works even when the package directory is read-only. +

+ This is meant to be used from a package's tst/testall.g, so that + extracted tests need not be committed to the repository: +

+

+ Note that makedoc.g is read in the usual way, so any other work it + performs still happens; and a script ending in QUIT cannot be used + this way. + + + +

+
The AutoDoc() function @@ -443,6 +482,23 @@ Also, if the environment variable NOPDF is set, then &AutoDoc; behaves as if the global option nopdf had been enabled. + AutoDocExtractOnly + + If this global option is set, &AutoDoc; builds only as much of the + manual as is needed to collect its examples: no HTML, PDF or manual + index is produced, and all output is written below the given scratch + directory instead of the package. The extracted tests end up in its + tst subdirectory. +

+ The value is a directory, or true to have one chosen automatically. + If the environment variable AUTODOC_EXTRACT_ONLY is set, &AutoDoc; + behaves as if this option had been given; that is useful for a + makedoc.g which ends in QUIT and therefore has to be run + as a separate process. +

+ Rather than setting this by hand, use + . + relativePath This has the same effect as gapdoc.gap_root_relative_path, but diff --git a/tst/manual.expected/_Chapter_Tutorials.xml b/tst/manual.expected/_Chapter_Tutorials.xml index ec3bb577..bd782c1b 100644 --- a/tst/manual.expected/_Chapter_Tutorials.xml +++ b/tst/manual.expected/_Chapter_Tutorials.xml @@ -291,6 +291,21 @@ AutoDoc( rec( extract_examples := rec( subdir := "tst/generated" ) ) ); ]]> This writes the extracted examples into tst/generated/ instead.

+Either way the generated .tst files have to be regenerated whenever the +manual changes, which tempts one into committing them. To avoid that, extract +them when the tests run, using in your +tst/testall.g: +

+This reads your makedoc.g, so the settings describing the manual are not +duplicated, but skips building the manual itself. Everything is written to a +temporary directory, so the extracted tests need not be committed and the tests +also run from a read-only package directory. +

diff --git a/tst/worksheets/autoplain.expected/_Chapter_Test.xml b/tst/worksheets/autoplain.expected/_Chapter_Test.xml index becd3a98..6017e396 100644 --- a/tst/worksheets/autoplain.expected/_Chapter_Test.xml +++ b/tst/worksheets/autoplain.expected/_Chapter_Test.xml @@ -12,6 +12,7 @@ the documentation generated from a .autodoc file like this? First Subsection + S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) @@ -25,6 +26,7 @@ Some text between two examples ampersand key less-than key indented command parsing works + A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) @@ -32,6 +34,7 @@ gap> Size(A5); 60 ]]> + plain_mode_value := 6 * > 9; diff --git a/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst b/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst index ce9ed3f3..7818a686 100644 --- a/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst +++ b/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("plain_file.autodoc_test01.tst"); -# _Chapter_Test.xml:15-20 +# tst/worksheets/autoplain.sheet/plain.autodoc:9-14 gap> S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) gap> Size(S5); 120 -# _Chapter_Test.xml:28-33 +# tst/worksheets/autoplain.sheet/plain.autodoc:21-26 gap> A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) gap> Size(A5); 60 -# _Chapter_Test.xml:35-39 +# tst/worksheets/autoplain.sheet/plain.autodoc:27-31 gap> plain_mode_value := 6 * > 9; 54 diff --git a/tst/worksheets/general.expected/_Chapter_SomeChapter.xml b/tst/worksheets/general.expected/_Chapter_SomeChapter.xml index 6366f069..17ecb6f0 100644 --- a/tst/worksheets/general.expected/_Chapter_SomeChapter.xml +++ b/tst/worksheets/general.expected/_Chapter_SomeChapter.xml @@ -5,6 +5,7 @@ SomeChapter This is dummy text + S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) @@ -17,6 +18,7 @@ gap> Size(S5); greater-than key quote key + A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) @@ -27,6 +29,7 @@ gap> [[2]]]]>[[1]]; true ]]> + comment_mode_value := 6 * > 7; diff --git a/tst/worksheets/general.expected/tst/general_test01.tst b/tst/worksheets/general.expected/tst/general_test01.tst index 2d7a8568..0c6a3ab3 100644 --- a/tst/worksheets/general.expected/tst/general_test01.tst +++ b/tst/worksheets/general.expected/tst/general_test01.tst @@ -10,13 +10,13 @@ # gap> START_TEST("general_test01.tst"); -# _Chapter_SomeChapter.xml:8-13 +# tst/worksheets/general.sheet/worksheet.g:13-18 gap> S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) gap> Size(S5); 120 -# _Chapter_SomeChapter.xml:20-28 +# tst/worksheets/general.sheet/worksheet.g:24-32 gap> A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) gap> Size(A5); @@ -25,7 +25,7 @@ gap> # Test whether ]]> can be used safely gap> [[2]]>[[1]]; true -# _Chapter_SomeChapter.xml:30-34 +# tst/worksheets/general.sheet/worksheet.g:33-37 gap> comment_mode_value := 6 * > 7; 42 diff --git a/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml b/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml index 5cf8a1bd..b9271f86 100644 --- a/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml +++ b/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml @@ -12,24 +12,28 @@ This worksheet exercises example and log commands in plain-text mode. Tested examples + plain_example_value := 2 + 3; 5 ]]>

+ plain_alias_example := 3 + 4; 7 ]]>

+ 10 - 3; 7 ]]>

+ 6 * 7; 42 @@ -41,24 +45,28 @@ gap> 6 * 7; Untested logs + plain_log_value := 9; 9 ]]>

+ plain_alias_log := 11; 11 ]]>

+ "plain log session"; "plain log session" ]]>

+ "plain alias log session"; "plain alias log session" diff --git a/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst b/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst index 9153b459..459f40ed 100644 --- a/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst +++ b/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("paired_examples_test01.tst"); -# _Chapter_Examples_Chapter.xml:15-18 +# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:8-11 gap> plain_example_value := 2 + 3; 5 -# _Chapter_Examples_Chapter.xml:21-24 +# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:13-16 gap> plain_alias_example := 3 + 4; 7 -# _Chapter_Examples_Chapter.xml:27-30 +# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:18-21 gap> 10 - 3; 7 -# _Chapter_Examples_Chapter.xml:33-36 +# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:23-26 gap> 6 * 7; 42 diff --git a/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml b/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml index 89dd9a62..debdf6c6 100644 --- a/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml +++ b/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml @@ -11,21 +11,25 @@ Tested examples + comment_example_value := 2 + 3; 5 ]]> + comment_alias_example := 3 + 4; 7 ]]> + 10 - 3; 7 ]]> + 6 * 7; 42 @@ -36,21 +40,25 @@ gap> 6 * 7; Untested logs + comment_log_value := 9; 9 ]]> + comment_alias_log := 11; 11 ]]> + "comment log session"; "comment log session" ]]> + "comment alias log session"; "comment alias log session" diff --git a/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst b/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst index 08a5f4aa..086c2f68 100644 --- a/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst +++ b/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("paired_examples_test01.tst"); -# _Chapter_Examples_Chapter.xml:14-17 +# tst/worksheets/paired-examples.sheet/worksheet.g:8-11 gap> comment_example_value := 2 + 3; 5 -# _Chapter_Examples_Chapter.xml:19-22 +# tst/worksheets/paired-examples.sheet/worksheet.g:13-16 gap> comment_alias_example := 3 + 4; 7 -# _Chapter_Examples_Chapter.xml:24-27 +# tst/worksheets/paired-examples.sheet/worksheet.g:18-21 gap> 10 - 3; 7 -# _Chapter_Examples_Chapter.xml:29-32 +# tst/worksheets/paired-examples.sheet/worksheet.g:23-26 gap> 6 * 7; 42 From 545f0eb40ff8b0416b36e8bd320cc0e5b66c73e6 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Mon, 31 Aug 2026 08:32:04 +0200 Subject: [PATCH 2/6] examples: Anchor worksheet source paths on the inputs Source positions recorded for a worksheet were resolved against the working directory, since a worksheet has no package directory to resolve them against. The recorded paths therefore depended on where AutoDocWorksheet was called from, and the comparison is textual, so reaching the inputs through a symlink defeated it as well and left only the bare file name. Anchor them on the deepest directory the input files share instead. That is derived from the given paths alone, so the generated files no longer vary with the working directory or with symlinks. Paths recorded for a package are unaffected and stay relative to the package directory. Co-Authored-By: Claude Opus 5 --- gap/Magic.gi | 16 +++++++-- gap/ToolFunctions.gd | 1 + gap/ToolFunctions.gi | 34 +++++++++++++++++++ tst/examples.tst | 8 ++--- .../autoplain.expected/_Chapter_Test.xml | 6 ++-- .../tst/plain_file.autodoc_test01.tst | 6 ++-- .../general.expected/_Chapter_SomeChapter.xml | 6 ++-- .../general.expected/tst/general_test01.tst | 6 ++-- .../_Chapter_Examples_Chapter.xml | 16 ++++----- .../tst/paired_examples_test01.tst | 8 ++--- .../_Chapter_Examples_Chapter.xml | 16 ++++----- .../tst/paired_examples_test01.tst | 8 ++--- 12 files changed, 89 insertions(+), 42 deletions(-) diff --git a/gap/Magic.gi b/gap/Magic.gi index 4bb5759e..256abf0f 100644 --- a/gap/Magic.gi +++ b/gap/Magic.gi @@ -88,7 +88,7 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) local scaffold, gapdoc, extract_examples, autodoc, i, doc_dir, doc_dir_rel, tmp, key, val, file, pkgdirstr, docdirstr, - title_page, tree, + title_page, tree, source_anchor, position_document_class, args, used_legacy_value_options, extract_only, extract_roots; @@ -355,10 +355,22 @@ function( is_worksheet, pkgname, pkginfo, pkgdir, opt ) if IsBound( autodoc ) then # Hand the parser both the real path and a reproducible name to # report positions under; the latter ends up in generated files. + source_anchor := pkgdir; + if is_worksheet then + # A worksheet has no package directory: pkgdir is merely the + # working directory, which would make the recorded paths depend on + # where AutoDocWorksheet was called from, and on whether the inputs + # were reached through a symlink. Anchor on the directory the input + # files share instead. + tmp := AUTODOC_CommonParentDirectory( autodoc.files ); + if tmp <> fail then + source_anchor := Directory( tmp ); + fi; + fi; AutoDocScanFiles( List( autodoc.files, f -> rec( path := f, - display := AUTODOC_RelativeSourcePath( f, pkgdir ) ) ), + display := AUTODOC_RelativeSourcePath( f, source_anchor ) ) ), pkgname, tree ); fi; diff --git a/gap/ToolFunctions.gd b/gap/ToolFunctions.gd index 753b5293..a090e033 100644 --- a/gap/ToolFunctions.gd +++ b/gap/ToolFunctions.gd @@ -6,6 +6,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later DeclareGlobalFunction( "AUTODOC_CreateDirIfMissing" ); +DeclareGlobalFunction( "AUTODOC_CommonParentDirectory" ); DeclareGlobalFunction( "AUTODOC_RelativeSourcePath" ); DeclareGlobalFunction( "AUTODOC_StageDirectory" ); DeclareGlobalFunction( "AUTODOC_SourceMarker" ); diff --git a/gap/ToolFunctions.gi b/gap/ToolFunctions.gi index 631f882f..104a4cc4 100644 --- a/gap/ToolFunctions.gi +++ b/gap/ToolFunctions.gi @@ -554,6 +554,40 @@ function(arg) return result; end); +# Return the deepest directory containing all of the given paths, as a string +# ending in "/", or fail if they share none. This is derived purely from the +# given strings, so unlike the working directory it is unaffected by where a +# command was started or by symlinks on the way to the files. +InstallGlobalFunction( "AUTODOC_CommonParentDirectory", +function( paths ) + local components, common, i, n; + + if IsEmpty( paths ) then + return fail; + fi; + + # keep the directory components, dropping the file name + components := List( paths, + p -> SplitString( p, "/" ) ); + components := List( components, c -> c{[ 1 .. Length( c ) - 1 ]} ); + + common := components[1]; + for i in [ 2 .. Length( components ) ] do + n := 0; + while n < Length( common ) and n < Length( components[i] ) + and common[ n + 1 ] = components[i][ n + 1 ] do + n := n + 1; + od; + common := common{[ 1 .. n ]}; + od; + + if IsEmpty( common ) then + return fail; + fi; + + return Concatenation( JoinStringsWithSeparator( common, "/" ), "/" ); +end ); + # Render a source file path for display and for recording in generated files: # relative to the package directory when it lies below it, else relative to # the working directory, else the bare filename. Absolute paths would make diff --git a/tst/examples.tst b/tst/examples.tst index 67118b28..aa3505d7 100644 --- a/tst/examples.tst +++ b/tst/examples.tst @@ -27,10 +27,10 @@ gap> lines := SplitString(StringFile( # The four examples start on lines 8, 13, 18 and 23 of worksheet.g. gap> Perform(Filtered(lines, > l -> StartsWith(l, "# ") and PositionSublist(l, ":") <> fail), Display); -# tst/worksheets/paired-examples.sheet/worksheet.g:8-11 -# tst/worksheets/paired-examples.sheet/worksheet.g:13-16 -# tst/worksheets/paired-examples.sheet/worksheet.g:18-21 -# tst/worksheets/paired-examples.sheet/worksheet.g:23-26 +# worksheet.g:8-11 +# worksheet.g:13-16 +# worksheet.g:18-21 +# worksheet.g:23-26 # No location may name a generated file. gap> ForAny(lines, l -> PositionSublist(l, "_Chapter_") <> fail); diff --git a/tst/worksheets/autoplain.expected/_Chapter_Test.xml b/tst/worksheets/autoplain.expected/_Chapter_Test.xml index 6017e396..9e1df079 100644 --- a/tst/worksheets/autoplain.expected/_Chapter_Test.xml +++ b/tst/worksheets/autoplain.expected/_Chapter_Test.xml @@ -12,7 +12,7 @@ the documentation generated from a .autodoc file like this? First Subsection - + S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) @@ -26,7 +26,7 @@ Some text between two examples ampersand key less-than key indented command parsing works - + A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) @@ -34,7 +34,7 @@ gap> Size(A5); 60 ]]> - + plain_mode_value := 6 * > 9; diff --git a/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst b/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst index 7818a686..9e5f2b98 100644 --- a/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst +++ b/tst/worksheets/autoplain.expected/tst/plain_file.autodoc_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("plain_file.autodoc_test01.tst"); -# tst/worksheets/autoplain.sheet/plain.autodoc:9-14 +# plain.autodoc:9-14 gap> S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) gap> Size(S5); 120 -# tst/worksheets/autoplain.sheet/plain.autodoc:21-26 +# plain.autodoc:21-26 gap> A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) gap> Size(A5); 60 -# tst/worksheets/autoplain.sheet/plain.autodoc:27-31 +# plain.autodoc:27-31 gap> plain_mode_value := 6 * > 9; 54 diff --git a/tst/worksheets/general.expected/_Chapter_SomeChapter.xml b/tst/worksheets/general.expected/_Chapter_SomeChapter.xml index 17ecb6f0..09b72423 100644 --- a/tst/worksheets/general.expected/_Chapter_SomeChapter.xml +++ b/tst/worksheets/general.expected/_Chapter_SomeChapter.xml @@ -5,7 +5,7 @@ SomeChapter This is dummy text - + S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) @@ -18,7 +18,7 @@ gap> Size(S5); greater-than key quote key - + A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) @@ -29,7 +29,7 @@ gap> [[2]]]]>[[1]]; true ]]> - + comment_mode_value := 6 * > 7; diff --git a/tst/worksheets/general.expected/tst/general_test01.tst b/tst/worksheets/general.expected/tst/general_test01.tst index 0c6a3ab3..a9c03eca 100644 --- a/tst/worksheets/general.expected/tst/general_test01.tst +++ b/tst/worksheets/general.expected/tst/general_test01.tst @@ -10,13 +10,13 @@ # gap> START_TEST("general_test01.tst"); -# tst/worksheets/general.sheet/worksheet.g:13-18 +# worksheet.g:13-18 gap> S5 := SymmetricGroup(5); Sym( [ 1 .. 5 ] ) gap> Size(S5); 120 -# tst/worksheets/general.sheet/worksheet.g:24-32 +# worksheet.g:24-32 gap> A5 := AlternatingGroup(5); Alt( [ 1 .. 5 ] ) gap> Size(A5); @@ -25,7 +25,7 @@ gap> # Test whether ]]> can be used safely gap> [[2]]>[[1]]; true -# tst/worksheets/general.sheet/worksheet.g:33-37 +# worksheet.g:33-37 gap> comment_mode_value := 6 * > 7; 42 diff --git a/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml b/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml index b9271f86..d66cd78e 100644 --- a/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml +++ b/tst/worksheets/paired-examples-autoplain.expected/_Chapter_Examples_Chapter.xml @@ -12,28 +12,28 @@ This worksheet exercises example and log commands in plain-text mode. Tested examples - + plain_example_value := 2 + 3; 5 ]]>

- + plain_alias_example := 3 + 4; 7 ]]>

- + 10 - 3; 7 ]]>

- + 6 * 7; 42 @@ -45,28 +45,28 @@ gap> 6 * 7; Untested logs - + plain_log_value := 9; 9 ]]>

- + plain_alias_log := 11; 11 ]]>

- + "plain log session"; "plain log session" ]]>

- + "plain alias log session"; "plain alias log session" diff --git a/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst b/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst index 459f40ed..4e2fa7ba 100644 --- a/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst +++ b/tst/worksheets/paired-examples-autoplain.expected/tst/paired_examples_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("paired_examples_test01.tst"); -# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:8-11 +# plain.autodoc:8-11 gap> plain_example_value := 2 + 3; 5 -# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:13-16 +# plain.autodoc:13-16 gap> plain_alias_example := 3 + 4; 7 -# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:18-21 +# plain.autodoc:18-21 gap> 10 - 3; 7 -# tst/worksheets/paired-examples-autoplain.sheet/plain.autodoc:23-26 +# plain.autodoc:23-26 gap> 6 * 7; 42 diff --git a/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml b/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml index debdf6c6..d7ef722d 100644 --- a/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml +++ b/tst/worksheets/paired-examples.expected/_Chapter_Examples_Chapter.xml @@ -11,25 +11,25 @@ Tested examples - + comment_example_value := 2 + 3; 5 ]]> - + comment_alias_example := 3 + 4; 7 ]]> - + 10 - 3; 7 ]]> - + 6 * 7; 42 @@ -40,25 +40,25 @@ gap> 6 * 7; Untested logs - + comment_log_value := 9; 9 ]]> - + comment_alias_log := 11; 11 ]]> - + "comment log session"; "comment log session" ]]> - + "comment alias log session"; "comment alias log session" diff --git a/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst b/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst index 086c2f68..fd4b86e0 100644 --- a/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst +++ b/tst/worksheets/paired-examples.expected/tst/paired_examples_test01.tst @@ -10,19 +10,19 @@ # gap> START_TEST("paired_examples_test01.tst"); -# tst/worksheets/paired-examples.sheet/worksheet.g:8-11 +# worksheet.g:8-11 gap> comment_example_value := 2 + 3; 5 -# tst/worksheets/paired-examples.sheet/worksheet.g:13-16 +# worksheet.g:13-16 gap> comment_alias_example := 3 + 4; 7 -# tst/worksheets/paired-examples.sheet/worksheet.g:18-21 +# worksheet.g:18-21 gap> 10 - 3; 7 -# tst/worksheets/paired-examples.sheet/worksheet.g:23-26 +# worksheet.g:23-26 gap> 6 * 7; 42 From cba2b2afffdfa7f7d344623ef6748b273f4aa090 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Mon, 31 Aug 2026 08:41:36 +0200 Subject: [PATCH 3/6] examples: Do not require GAP 4.13 to extract examples AutoDocExtractExamples reads a package's makedoc.g with the package directory as the working directory, using ChangeDirectoryCurrent. That function needs GAP 4.13, or the io package on older versions, while AutoDoc itself supports GAP 4.11; naming it unconditionally made loading AutoDoc warn about an unbound global there. Look the function up by name and skip the directory change when it is missing. The script is passed as an absolute path, so AutoDoc still locates the package either way; only a script reading further files by relative path needs the working directory, and that case is now reported and documented. Co-Authored-By: Claude Opus 5 --- gap/Examples.gd | 5 ++++ gap/Examples.gi | 28 ++++++++++++++++++---- tst/manual.expected/_Chapter_Reference.xml | 5 ++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/gap/Examples.gd b/gap/Examples.gd index 2e8535d8..6f4f6f8d 100644 --- a/gap/Examples.gd +++ b/gap/Examples.gd @@ -34,6 +34,11 @@ #! Note that makedoc.g is read in the usual way, so any other work it #! performs still happens; and a script ending in QUIT cannot be used #! this way. +#! +#! The script is read with the package directory as the working directory, +#! which requires &GAP; 4.13 or newer, or the io package +#! on older versions. Without either, the script is still found and read, but +#! one which reads further files by relative path will not find them. #! @Returns a directory #! @Arguments pkg[, makedoc] DeclareGlobalFunction( "AutoDocExtractExamples" ); diff --git a/gap/Examples.gi b/gap/Examples.gi index 87a0b317..45090f4e 100644 --- a/gap/Examples.gi +++ b/gap/Examples.gi @@ -48,7 +48,7 @@ end ); ## InstallGlobalFunction( "AutoDocExtractExamples", function( pkg, makedoc... ) - local pkgdir, script, scratch, olddir; + local pkgdir, script, scratch, olddir, chdir; if Length( makedoc ) > 0 then script := makedoc[ 1 ]; @@ -78,10 +78,28 @@ function( pkg, makedoc... ) # AutoDoc() with no arguments picks up PackageInfo.g from the working # directory, and makedoc.g scripts name their inputs relative to the # package, so run the script from there. - olddir := AUTODOC_CurrentDirectory(); - ChangeDirectoryCurrent( Filename( pkgdir, "" ) ); - Read( script : AutoDocExtractOnly := scratch, nopdf ); - ChangeDirectoryCurrent( olddir ); + # + # ChangeDirectoryCurrent needs GAP >= 4.13, or the io package on older + # versions; look it up by name so that reading this file does not warn + # about an unbound global where it is missing. Passing