examples: Separate extraction from manual generation

CHANGES.md

@@ -1,8 +1,17 @@
 This file describes changes in the AutoDoc package.
 
 ## unreleased
+  - Require GAP 4.13 or newer
   - Fix a spurious "chunk ... was defined but never inserted" warning for
     chunks that are only inserted from within the body of another chunk
+  - Add `AutoDocExtractExamples`, which extracts the manual examples of a
+    package into a temporary directory by running its `makedoc.g` without
+    building the manual. This allows running the examples from
+    `tst/testall.g` without storing generated `.tst` files in the
+    repository, and works from a read-only package directory
+  - Report the true origin of an extracted example: generated `.tst` files
+    now point at the `.autodoc` file or AutoDoc comment the example was
+    written in, instead of the intermediate XML file generated from it
 
 ## 2026.06.30
   - Fix a regression in `.autodoc` parsing where Markdown-style headings

PackageInfo.g

@@ -118,10 +118,10 @@ PackageDoc := rec(
 ),
 
 Dependencies := rec(
-  GAP := ">= 4.11",
+  GAP := ">= 4.13",
   NeededOtherPackages := [ [ "GAPDoc", ">= 1.6.3" ] ],
   SuggestedOtherPackages := [ ],
-  TestPackages := [ [ "io", ">= 4.7.0" ] ],
+  TestPackages := [ ],
   ExternalConditions := [],
 ),
 

doc/Tutorials.autodoc

@@ -271,6 +271,21 @@ AutoDoc( rec( extract_examples := rec( subdir := "tst/generated" ) ) );
 ```
 This writes the extracted examples into <F>tst/generated/</F> instead.
 
+Either way the generated <F>.tst</F> files have to be regenerated whenever the
+manual changes, which tempts one into committing them. To avoid that, extract
+them when the tests run, using <Ref Func="AutoDocExtractExamples"/> in your
+<F>tst/testall.g</F>:
+```@listing
+LoadPackage( "mypkg" );
+dirs := DirectoriesPackageLibrary( "mypkg", "tst" );
+Add( dirs, AutoDocExtractExamples( "mypkg" ) );
+TestDirectory( dirs, rec( exitGAP := true ) );
+```
+This reads your <F>makedoc.g</F>, so the settings describing the manual are not
+duplicated, but skips building the manual itself. Everything is written to a
+temporary directory, so the extracted tests need not be committed and the tests
+also run from a read-only package directory.
+
 @Subsection Setting different &GAPDoc; options
 @SubsectionLabel Tut:IntegrateExisting:GapDocOptions
 

gap/AutoDocMainFunction.gi

@@ -360,9 +360,10 @@ end );
 
 # The following function is based on code by Olexandr Konovalov
 BindGlobal("AUTODOC_ExtractMyManualExamples",
-function( pkgname, pkgdir, docdir, main, files, opt )
-    local tst, i, s, basename, name, output, ch, a, location, pos, comment, pkgdirString,
-      nonempty_units_found, number_of_digits, lpkgname, tstdir;
+function( pkgname, pkgdir, docdir, main, files, opt, roots )
+    local tst, i, s, basename, name, output, ch, a, location, pos, comment,
+      nonempty_units_found, number_of_digits, lpkgname, tstdir, composed,
+      prefixes, prefix;
     Info(InfoAutoDoc, 1, "Extracting manual examples for ", pkgname, " package ...");
 
     lpkgname := LowercaseString(pkgname);
@@ -371,9 +372,20 @@ function( pkgname, pkgdir, docdir, main, files, opt )
     if not EndsWith(main, ".xml") then
         main := Concatenation( main, ".xml" );
     fi;
-    tst:=ExtractExamples( docdir, main, files, opt.units );
+    # This is GAPDoc's ExtractExamples, with a pass over the origin list added
+    # so that examples AutoDoc generated report the file they came from rather
+    # than the intermediate XML file.
+    composed := ComposedDocument( "GAPDoc", docdir, main, files, true );
+    AUTODOC_RemapSourcePositions( composed[1], composed[2] );
+    tst := ExtractExamplesXMLTree(
+        ParseTreeXMLString( composed[1], composed[2] ), opt.units );
     Info(InfoAutoDoc, 1, Length(tst), " ", LowercaseString( opt.units ), "s detected");
-    pkgdirString := Filename(pkgdir, "");
+    # Directories a source file may live under, most specific first. Locations
+    # are reported relative to whichever matches, so that generated .tst files
+    # do not depend on where the package is installed.
+    prefixes := Concatenation( roots, [ pkgdir,
+                                        Directory(AUTODOC_CurrentDirectory()) ] );
+    prefixes := List( prefixes, d -> Filename( d, "" ) );
 
     if IsDirectory( opt.subdir ) then
         tstdir := Filename( opt.subdir, "" );
@@ -434,9 +446,20 @@ function( pkgname, pkgdir, docdir, main, files, opt )
         AppendTo(output, "gap> START_TEST(\"", basename, "\");\n\n");
         for a in ch do
             location := a[2][1];
-            if StartsWith(location, pkgdirString) then
-                comment := location{[ Length(pkgdirString)+1 .. Length(location) ]};
+            if not StartsWith(location, "/") then
+                # Already reproducible: AutoDoc recorded this position itself,
+                # or GAPDoc resolved it relative to the documentation dir.
+                comment := location;
             else
+                comment := fail;
+                for prefix in prefixes do
+                    if StartsWith(location, prefix) then
+                        comment := location{[ Length(prefix)+1 .. Length(location) ]};
+                        break;
+                    fi;
+                od;
+            fi;
+            if comment = fail then
                 pos := PositionSublist(location, LowercaseString(pkgname));
                 if pos <> fail then
                     comment := location{[ pos+Length(pkgname)+1 .. Length(location) ]};
@@ -445,7 +468,10 @@ function( pkgname, pkgdir, docdir, main, files, opt )
                     if pos <> fail then
                         comment := location{[ pos+2 .. Length(location) ]};
                     else
-                        Error("oops");
+                        # Sources outside all of the above, e.g. worksheet
+                        # inputs. The bare filename is still more useful than
+                        # an absolute path, and keeps the output reproducible.
+                        comment := Last( SplitString( location, "/" ) );
                     fi;
                 fi;
             fi;

gap/DocumentationTree.gi

@@ -666,7 +666,12 @@ end );
 
 InstallMethod( WriteDocumentation, [ IsTreeForDocumentationVerbatimNodeRep, IsStream ],
   function( node, filestream )
-    local line, attr_name;
+    local line, attr_name, marker;
+
+    marker := AUTODOC_SourceMarker( node );
+    if marker <> fail then
+        AppendTo( filestream, marker, "\n" );
+    fi;
 
     AppendTo( filestream, "<", node!.element_name );
     for attr_name in Set( RecNames( node!.attributes ) ) do

gap/Examples.gd

@@ -0,0 +1,41 @@
+# AutoDoc: Generate documentation from GAP source code
+#
+# Copyright of AutoDoc belongs to its developers.
+# Please refer to the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+#! @Chapter Reference
+#! @Section Extracting manual examples
+
+#! @Description
+#!  Extracts the examples from the manual of the package <A>pkg</A> and
+#!  returns the directory holding the resulting <F>.tst</F> files.
+#!
+#!  <A>pkg</A> is either the name of a package or a directory object pointing
+#!  at one. The optional argument <A>makedoc</A> names the script to read,
+#!  and defaults to <F>makedoc.g</F>.
+#!
+#!  The package's own <F>makedoc.g</F> is used as-is, so the settings which
+#!  describe the manual — its source files, scaffolding and
+#!  <A>extract_examples</A> options — are not duplicated. Only the parts of
+#!  the manual needed to collect the examples are built, no HTML or PDF is
+#!  produced, and everything is written to a temporary directory, so this
+#!  works even when the package directory is read-only.
+#!
+#!  This is meant to be used from a package's <F>tst/testall.g</F>, so that
+#!  extracted tests need not be committed to the repository:
+#!  <Listing><![CDATA[
+#!  LoadPackage( "mypkg" );
+#!  dirs := DirectoriesPackageLibrary( "mypkg", "tst" );
+#!  Add( dirs, AutoDocExtractExamples( "mypkg" ) );
+#!  TestDirectory( dirs, rec( exitGAP := true ) );]]></Listing>
+#!
+#!  Note that <F>makedoc.g</F> is read in the usual way, so any other work it
+#!  performs still happens; and a script ending in <C>QUIT</C> cannot be used
+#!  this way.
+#! @Returns a directory
+#! @Arguments pkg[, makedoc]
+DeclareGlobalFunction( "AutoDocExtractExamples" );
+
+DeclareGlobalFunction( "AUTODOC_ExtractOnlyDirectory" );

gap/Examples.gi

@@ -0,0 +1,87 @@
+# AutoDoc: Generate documentation from GAP source code
+#
+# Copyright of AutoDoc belongs to its developers.
+# Please refer to the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Environment variable equivalent of the AutoDocExtractOnly global option. It
+# exists so that a makedoc.g ending in QUIT can still be driven, by running it
+# as a separate GAP process.
+BindGlobal( "AUTODOC_EXTRACT_ONLY_ENVVAR", "AUTODOC_EXTRACT_ONLY" );
+
+BindGlobal( "AUTODOC_DEFAULT_MAKEDOC_FILE", "makedoc.g" );
+
+# The scratch directory to work in, or fail if AutoDoc should build the manual
+# normally. Both spellings carry the directory; passing just `true` or `1`
+# asks for extract-only mode without naming one.
+InstallGlobalFunction( "AUTODOC_ExtractOnlyDirectory",
+function()
+    local value;
+
+    value := ValueOption( "AutoDocExtractOnly" );
+    if value = fail and
+       IsBound( GAPInfo.SystemEnvironment.( AUTODOC_EXTRACT_ONLY_ENVVAR ) ) then
+        value := GAPInfo.SystemEnvironment.( AUTODOC_EXTRACT_ONLY_ENVVAR );
+    fi;
+
+    if value = fail or value = false then
+        return fail;
+    fi;
+
+    if IsDirectory( value ) then
+        return value;
+    fi;
+
+    if value = true or value = "" or value = "1" then
+        return DirectoryTemporary();
+    fi;
+
+    if not IsString( value ) then
+        Error( "AutoDocExtractOnly must be true, a path, or a directory object" );
+    fi;
+
+    AUTODOC_CreateDirIfMissing( value );
+    return Directory( value );
+end );
+
+##
+InstallGlobalFunction( "AutoDocExtractExamples",
+function( pkg, makedoc... )
+    local pkgdir, script, scratch, olddir;
+
+    if Length( makedoc ) > 0 then
+        script := makedoc[ 1 ];
+    else
+        script := AUTODOC_DEFAULT_MAKEDOC_FILE;
+    fi;
+
+    if IsDirectory( pkg ) then
+        pkgdir := pkg;
+    elif IsString( pkg ) then
+        pkgdir := DirectoriesPackageLibrary( pkg, "" );
+        if pkgdir = [ ] then
+            Error( "could not locate package ", pkg );
+        fi;
+        pkgdir := pkgdir[ 1 ];
+    else
+        Error( "pkg must be a package name or a directory object" );
+    fi;
+
+    script := Filename( pkgdir, script );
+    if script = fail or not IsReadableFile( script ) then
+        Error( "could not read ", script );
+    fi;
+
+    scratch := DirectoryTemporary();
+
+    # AutoDoc() with no arguments picks up PackageInfo.g from the working
+    # directory, and makedoc.g scripts name their inputs relative to the
+    # package, so run the script from there.
+    olddir := AUTODOC_CurrentDirectory();
+    ChangeDirectoryCurrent( Filename( pkgdir, "" ) );
+    Read( script : AutoDocExtractOnly := scratch, nopdf );
+    ChangeDirectoryCurrent( olddir );
+
+    return Directory( Filename( scratch, "tst" ) );
+end );

gap/Magic.gd

@@ -441,6 +441,23 @@
 #!     Also, if the environment variable `NOPDF` is set, then &AutoDoc;
 #!     behaves as if the global option <A>nopdf</A> had been enabled.
 #! </Item>
+#! <Mark><A>AutoDocExtractOnly</A></Mark>
+#! <Item>
+#!     If this global option is set, &AutoDoc; builds only as much of the
+#!     manual as is needed to collect its examples: no HTML, PDF or manual
+#!     index is produced, and all output is written below the given scratch
+#!     directory instead of the package. The extracted tests end up in its
+#!     <F>tst</F> subdirectory.
+#!     <P/>
+#!     The value is a directory, or `true` to have one chosen automatically.
+#!     If the environment variable `AUTODOC_EXTRACT_ONLY` is set, &AutoDoc;
+#!     behaves as if this option had been given; that is useful for a
+#!     <F>makedoc.g</F> which ends in <C>QUIT</C> and therefore has to be run
+#!     as a separate process.
+#!     <P/>
+#!     Rather than setting this by hand, use
+#!     <Ref Func="AutoDocExtractExamples"/>.
+#! </Item>
 #! <Mark><A>relativePath</A></Mark>
 #! <Item>
 #!     This has the same effect as <A>gapdoc.gap_root_relative_path</A>, but