removing files superceded by utils0-.tst

CHANGES.md

@@ -1,5 +1,10 @@
 # CHANGES log for the 'Utils' package
 
+## Version 0.99 for GAP 4.16.1 (09/09/26)
+ * (09/09/26) since now using AutoDoc to extract tests from the manual,
+              many of the original tests are no longer needed, so removed.
+ * (08/09/26) extensive changes to the Download function
+
 ## Version 0.98 for GAP 4.16.0 (04/08/26)
  * (04/08/26) changed Fitting series examples to just show StructureDescription
 

doc/download.xml

@@ -69,7 +69,31 @@ The following components are supported.
   and the function writes the downloaded contents to this file;
   the returned record does not have a <C>result</C> component in this case.
   <P/>
-  If the download fails then this file is not left behind.
+  If the download fails then a file created by the attempt is not left
+  behind, unless <C>resume</C> is set.  A file that was already there
+  before the call is not removed, since it is not ours to delete &ndash;
+  but note that a failed attempt may well have overwritten it, so its old
+  contents are not guaranteed either.
+</Item>
+<Mark><C>resume</C></Mark>
+<Item>
+  If this component is bound and has the value <K>true</K>,
+  and <C>target</C> is given,
+  then a partially downloaded file is continued rather than fetched again,
+  and it is kept if the download fails again.
+  <P/>
+  Methods that cannot resume decline the request, since they would discard
+  the partial file that a method which can resume needs; currently the
+  methods based on the external programs <C>curl</C> and <C>wget</C> resume.
+  If none is available the download fails, and the caller can retry without
+  <C>resume</C>.
+  <P/>
+  It is the caller's responsibility that an existing <C>target</C> really is
+  a prefix of what <A>url</A> delivers; otherwise the two get concatenated.
+  <P/>
+  If the server does not support range requests then no data is lost:
+  <C>wget</C> fetches the file again from the start, and <C>curl</C> fails
+  and leaves the partial file alone.
 </Item>
 <Mark><C>verifyCert</C></Mark>
 <Item>

doc/groups.xml

@@ -401,24 +401,21 @@ The first of these is the zero map, the last is the identity.
 <P/>
 <Example><![CDATA[
 gap> gens := [ (1,2,3,4), (1,2)(3,4) ];; 
-gap> d8 := Group( gens );;
-gap> SetName( d8, "d8" );
+gap> d8 := Group( gens );;  SetName( d8, "d8" );
 gap> c2 := Subgroup( d8, [ (2,4) ] );;
 gap> SortedList( IdempotentEndomorphismsWithImage( gens, c2 ) );
 [ [ (), (2,4) ], [ (2,4), () ] ]
 gap> data:= IdempotentEndomorphismsData( d8 );;
-gap> data.images[1];
-[ [ (), () ] ]
-gap> List( data.images, Length );
-[ 1, 2, 2, 2, 2, 1 ]
+gap> Length( data.images );
+6
 gap> all:= IdempotentEndomorphisms( d8 );;
-gap> Length( all );
-10
-gap> all[1];
-[ (1,2,3,4), (1,2)(3,4) ] -> [ (), () ]
-gap> Size( Image( all[1] ) );
-1
-gap> Last( all ) = IdentityMapping( d8 );
+gap> allim := List( all, m -> MappingGeneratorsImages(m)[2] );;
+gap> ## the list of 10 sets of images sorted: 
+gap> allimpairs := 
+> [ [ (), () ], [ (), (2,4) ], [ (2,4), () ], [ (), (1,3) ], [ (1,3), () ], 
+>   [ (), (1,2)(3,4) ], [ (1,2)(3,4), (1,2)(3,4) ], [ (), (1,4)(2,3) ], 
+>   [ (1,4)(2,3), (1,4)(2,3) ], [ (1,2,3,4), (1,2)(3,4) ] ];; 
+gap> ForAll( allim, m -> ( m in allimpairs ) ); 
 true
 ]]></Example>
 </Description>

doc/intro.xml

@@ -2,7 +2,7 @@
 <!--                                                                     -->
 <!--  intro.xml            Utils documentation            Chris Wensley  -->
 <!--                                                                     -->
-<!--  Copyright (C) 2015-2018, The GAP Group                             --> 
+<!--  Copyright (C) 2015-2026, The GAP Group                             --> 
 <!--                                                                     -->
 <!-- ------------------------------------------------------------------- -->
 
@@ -78,8 +78,16 @@ may be rebuilt as follows:
 gap> ReadPackage( "utils", "makedoc.g" ); 
 ]]></Log>
 <P/>
-It is possible to check that the package has been installed correctly
-by running the test files (which terminates the &GAP; session): 
+
+This package now uses the <Package>AutoDoc</Package> package
+to extract test files from the examples in the manual.
+Many of the original test files,
+which duplicated these examples, have now been deleted.
+The user can ensure that the full set of test files exists on their system
+by rebuilding the manual, as described above.
+<P/>
+It is then possible to check that the package has been installed correctly
+by running the test suite (this terminates the &GAP; session): 
 <P/>
 <Log><![CDATA[
 gap> ReadPackage( "utils", "tst/testall.g" );

doc/iterator.xml

@@ -2,7 +2,7 @@
 <!--                                                                     -->
 <!--  iterator.xml          Utils documentation                          -->
 <!--                                                                     -->
-<!--  Copyright (C) 2015-2019, The GAP Group                             --> 
+<!--  Copyright (C) 2015-2026, The GAP Group                             --> 
 <!--                                                                     -->
 <!-- ------------------------------------------------------------------- -->
 
@@ -54,6 +54,11 @@ gap> AllIsomorphisms( G, s3 );
 gap> iter := AllIsomorphismsIterator( G, s3 );;
 gap> for h in iter do Print( ImageElm( h, G.1 ) = (6,7), ", " ); od;
 true, false, false, true, false, false,
+gap> s4 := Group( (1,2),(2,3),(3,4) );;
+gap> AllIsomorphismsNumber( s3, s4 );
+0
+gap> AllIsomorphisms( s3, s4 );
+[ ]
 ]]></Example>
 </Description>
 </ManSection>
@@ -156,6 +161,24 @@ gap> NextIterator( pairs4 );
 gap> IsDoneIterator( pairs4 );
 true
 ]]></Example>
+
+Some trivial cases:
+<Example><![CDATA[
+gap> iter0 := IteratorList( [ ] );;
+gap> iter4 := IteratorList( [ 4 ] );; 
+gap> cart := CartesianIterator( iter0, iter0 );;
+gap> IsDoneIterator( cart );
+true
+gap> cart := CartesianIterator( iter0, iter4 );;
+gap> IsDoneIterator( cart );
+true
+gap> cart := CartesianIterator( iter4, iter0 );;
+gap> IsDoneIterator( cart );
+true
+gap> pairs0 := UnorderedPairsIterator( iter0 );;
+gap> IsDoneIterator( pairs0 );
+true
+]]></Example>
 </Description>
 </ManSection>
 

doc/matrix.xml

@@ -2,7 +2,7 @@
 <!--                                                                     -->
 <!--  matrix.xml          Utils documentation                            -->
 <!--                                                                     -->
-<!--  Copyright (C) 2023, The GAP Group                                  --> 
+<!--  Copyright (C) 2015-2026, The GAP Group                                  --> 
 <!--                                                                     -->
 <!-- ------------------------------------------------------------------- -->
 
@@ -83,6 +83,14 @@ gap> Display( M8 );
 gap> L8 := DirectSumDecompositionMatrices( M8 );;
 gap> Length( L8 ); 
 16
+
+gap> M3 := [ [0,0,7,0,0,0,0], [0,0,0,8,0,0,0], [0,0,0,0,9,0,0] ];;
+gap> Display( M3 );  
+[ [  0,  0,  7,  0,  0,  0,  0 ],
+  [  0,  0,  0,  8,  0,  0,  0 ],
+  [  0,  0,  0,  0,  9,  0,  0] ]
+gap> L3 := DirectSumDecompositionMatrices( M3 );                  
+[ [ [ [ 0, 0, 7 ] ], [ [ 8 ] ], [ [ 9, 0, 0 ] ] ] ]
 ]]></Example>
 <P/>
 

lib/download.gi

@@ -36,15 +36,29 @@ Add( Download_Methods, rec(
   download:= function( url, opt )
     local res;
 
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      # Declining matters: this method would truncate the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
+    fi;
+
     opt:= ShallowCopy( opt );
     if not IsBound( opt.failOnError ) then
       opt.failOnError:= true;
     fi;
-    # 'DownloadURL' handles the options 'verifyCert' and 'maxTime'.
+    # 'DownloadURL' handles the options 'verifyCert' and 'maxTime'.  Newer
+    # versions can write the body to a file themselves, which keeps it out of
+    # memory; they call that option 'targetFile'.
+    if IsBound( opt.target ) and IsString( opt.target ) then
+      opt.targetFile:= opt.target;
+    fi;
     res:= ValueGlobal( "DownloadURL" )( url, opt );
 
     if res.success = true and
-       IsBound( opt.target ) and IsString( opt.target ) then
+       IsBound( opt.target ) and IsString( opt.target ) and
+       IsBound( res.result ) then
+      # an older 'DownloadURL' ignored 'targetFile' and returned the contents
       FileString( opt.target, res.result );
       Unbind( res.result );
     fi;
@@ -61,6 +75,11 @@ Add( Download_Methods, rec(
       return rec( success:= false, error:= "protocol is not http" );
     elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then
       return rec( success:= false, error:= "no support for given timeout" );
+    elif IsBound( opt.resume ) and opt.resume = true and
+         IsBound( opt.target ) and IsString( opt.target ) then
+      # No range request, so this would overwrite the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
     fi;
 
     # Split the URL after 'http://' into the authority and HTTP request target.
@@ -164,6 +183,10 @@ Add( Download_Methods, rec(
     else
       args:= [ "--quiet", "-O", "-", url ];
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-c" );
+    fi;
     if IsBound( opt.verifyCert ) and opt.verifyCert = false then
       Add( args, "--no-check-certificate" );
     fi;
@@ -173,11 +196,6 @@ Add( Download_Methods, rec(
     code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
     CloseStream( outstream );
     if code <> 0 then
-      # wget may have created the target file; try to remove it
-      if IsBound( opt.target ) and IsString( opt.target ) and
-         IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then
-        Error( "Download cannot remove unwanted file ", opt.target );
-      fi;
       return rec( success:= false,
                   error:= Concatenation( "Process returned ", String( code ) ) );
     elif not ( IsBound( opt.target ) and IsString( opt.target ) ) then
@@ -211,6 +229,11 @@ Add( Download_Methods, rec(
     else
       Add( args, "-" );
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-C" );
+      Add( args, "-" );
+    fi;
     if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
       Add( args, "--max-time" );
       Add( args, opt.maxTime );
@@ -245,7 +268,7 @@ InstallMethod( Download,
 InstallMethod( Download,
     [ "IsString", "IsRecord" ],
     function( url, opt )
-    local timeout, errors, r, res;
+    local timeout, hadTarget, errors, r, res;
 
     # Do not modify the caller's record when filling in the defaults below.
     opt:= ShallowCopy( opt );
@@ -264,6 +287,11 @@ InstallMethod( Download,
       fi;
     fi;
 
+    # Whether the caller brought the target file, as opposed to a method
+    # creating it below.  We remove only what we created.
+    hadTarget:= IsBound( opt.target ) and IsString( opt.target ) and
+                IsExistingFile( opt.target );
+
     # Run over the methods.
     errors:= [];
     for r in Download_Methods do
@@ -275,8 +303,12 @@ InstallMethod( Download,
         fi;
         # A failed method may have left a partial or bogus target file behind.
         # Remove it here, so that the guarantee holds for every method,
-        # including ones added to 'Download_Methods' from outside.
-        if IsBound( opt.target ) and IsString( opt.target ) and
+        # including ones added to 'Download_Methods' from outside.  Two
+        # files are not ours to delete: one the caller brought, and the
+        # partial one that a resumed download continues from.
+        if not hadTarget and
+           not ( IsBound( opt.resume ) and opt.resume = true ) and
+           IsBound( opt.target ) and IsString( opt.target ) and
            IsExistingFile( opt.target ) then
           RemoveFile( opt.target );
         fi;

tst/download.tst

@@ -1,4 +1,4 @@
-#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref
+#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name
 ############################################################################
 ##
 #W  download.tst               Utils Package                   Thomas Breuer
@@ -164,6 +164,66 @@ true
 gap> IsExistingFile( file );
 false
 
+##  'resume' continues a partial file rather than fetching it again.  The
+##  test server answers a Range request with the remainder in upper case, so
+##  a resumed download is distinguishable from a restarted one.
+gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );;
+gap> for r in resumers do
+>      FileString( file, "abcde" );;
+>      res1:= r.download( Concatenation( baseurl, "/resumable" ),
+>                         rec( target:= file, resume:= true ) );
+>      if res1.success <> true then
+>        Print( "resume failed for ", r.name, ": ", res1.error, "\n" );
+>      elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then
+>        Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" );
+>      fi;
+>      RemoveFile( file );
+>    od;
+
+##  Without 'resume' the target is replaced, not appended to.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file ) );;
+gap> StringFile( file );
+"abcdefghijklmnopqrst"
+
+##  Methods that cannot resume decline, rather than discarding the partial
+##  file that a method which can resume needs.
+gap> iometh.download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file, resume:= true ) ).error;
+"no support for resuming"
+
+##  With 'resume', a failed download keeps the partial file to continue from.
+##  Without this, the clean-up in 'Download' would throw away the very thing
+##  the next attempt is meant to continue.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/missing" ),
+>                     rec( target:= file, resume:= true ) );;
+gap> res1.success;
+false
+gap> StringFile( file );
+"abcde"
+gap> RemoveFile( file );;
+
+##  'resume' is only declined when it is actually requested: a method must
+##  not be skipped merely because the component is present.
+gap> res1:= Download( Concatenation( baseurl, "/file" ),
+>                     rec( target:= file, resume:= false ) );;
+gap> res1.success;
+true
+gap> RemoveFile( file );;
+
+##  A file that was already there is not removed by a failed download:
+##  'Download' only cleans up after itself.
+gap> FileString( file, "mine" );;
+gap> res1:= Download( Concatenation( baseurl, "/missing" ),
+>                     rec( target:= file ) );;
+gap> res1.success;
+false
+gap> IsExistingFile( file );
+true
+gap> RemoveFile( file );;
+
 ##  test errors and redirects
 gap> res1:= Download( Concatenation( baseurl, "/missing" ) );;
 gap> res1.success = false;

tst/gslp.tst

@@ -5,6 +5,9 @@
 ##
 #Y  Copyright (C) 2026, The GAP Group
 ##
+##
+##  some tests other than those in the manual
+
 gap> START_TEST( "gslp.tst" );
 
 #