--- a/src/fetch.c	Tue Jul 25 13:12:50 2000 +0100
+++ b/src/fetch.c	Tue Jul 25 13:14:54 2000 +0100
@@ -1,7 +1,7 @@
 /*
   fetch.c
 
-  $Id: fetch.c 174 2000-07-19 07:02:45Z enz $
+  $Id: fetch.c 183 2000-07-25 12:14:54Z bears $
 */
 
 #if HAVE_CONFIG_H
@@ -31,6 +31,7 @@
 #include "fetchlist.h"
 #include "request.h"
 #include "group.h"
+#include "lock.h"
 #include "log.h"
 #include "outgoing.h"
 #include "protocol.h"
@@ -38,6 +39,8 @@
 #include "util.h"
 #include "portable.h"
 
+#define	MAX_ARTICLE_CMDS_QUEUED		20
+
 struct Fetch
 {
     Bool ready;
@@ -63,7 +66,8 @@
     Str file;
 
     ASSERT( fetch.ready );
-    snprintf( file, MAXCHAR, "%s/groupinfo.lastupdate", Cfg_spoolDir() );
+    snprintf( file, MAXCHAR, "%s/lastupdate.%s",
+	      Cfg_spoolDir(), fetch.serv );
     if ( ! Utl_getStamp( &t, file ) )
     {
         Log_err( "Cannot read %s. Please run noffle --query groups", file );
@@ -71,21 +75,22 @@
     }
     Log_inf( "Updating groupinfo" );
     Client_getNewgrps( &t );
-    Utl_stamp( file );
 }
 
-void
-Fetch_getNewArts( const char *name, FetchMode mode )
+/* Databases open on entry, closed on exit. */
+static void
+fetchNewArts( const char *name, FetchMode mode )
 {
     int next, first, last;
 
     if ( ! Client_changeToGrp( name ) )
     {
         Log_err( "Could not change to group %s", name );
+	Lock_closeDatabases();
         return;
     }
+    Client_rmtFirstLast( &first, &last );
     Cont_read( name );
-    Client_rmtFirstLast( &first, &last );
     next = Grp_rmtNext( name );
     if ( next == GRP_RMT_NEXT_NOT_SUBSCRIBED )
 	next = first;
@@ -94,6 +99,7 @@
         Log_inf( "No new articles in %s", name );
         Cont_write();
         Grp_setFirstLast( name, Cont_first(), Cont_last() );
+	Lock_closeDatabases();
         return;
     }
     if ( first == 0 && last == 0 )
@@ -101,6 +107,7 @@
         Log_inf( "No articles in %s", name );
         Cont_write();
         Grp_setFirstLast( name, Cont_first(), Cont_last() );
+	Lock_closeDatabases();
         return;
     }
     if ( next > last + 1 )
@@ -124,9 +131,19 @@
     }
     Log_inf( "Getting remote overviews %lu-%lu for group %s",
              first, last, name );
-    Client_getOver( first, last, mode );
-    Cont_write();
-    Grp_setFirstLast( name, Cont_first(), Cont_last() );
+    Lock_closeDatabases();
+    Client_getOver( name, first, last, mode );
+}
+
+void
+Fetch_getNewArts( const char *name, FetchMode mode )
+{
+    if ( ! Lock_openDatabases() )
+    {
+        Log_err( "Could not open message base" );
+        return;
+    }
+    fetchNewArts( name, mode );
 }
 
 void
@@ -134,17 +151,43 @@
 {
     FetchMode mode;
     int i, size;
-    const char* name;
+    const char *name;
 
     ASSERT( fetch.ready );
+    if ( ! Lock_openDatabases() )
+    {
+        Log_err( "Could not open message base" );
+        return;
+    }
     Fetchlist_read();
     size = Fetchlist_size();
     for ( i = 0; i < size; ++i )
     {
         Fetchlist_element( &name, &mode, i );
         if ( strcmp( Grp_server( name ), fetch.serv ) == 0 )
-            Fetch_getNewArts( name, mode );
+	{
+            fetchNewArts( name, mode );
+	    if ( ! Lock_openDatabases() )
+	    {
+		Log_err( "Could not open message base" );
+		return;
+	    }
+	}
     }
+    Lock_closeDatabases();
+}
+
+static void
+fetchMessageList( const char *list, int *artcnt, int artmax )
+{
+    const char *p;
+    Str msgId;
+
+    ASSERT( Lock_gotLock() );
+    Client_retrieveArtList( list, artcnt, artmax );
+    p = list;
+    while ( ( p = Utl_getLn( msgId, p ) ) )
+        Req_remove( fetch.serv, msgId );
 }
 
 void
@@ -152,33 +195,60 @@
 {
     Str msgId;
     DynStr *list;
+    DynStr *fetchList;
     const char *p;
     int count = 0, artcnt = 0, artmax = 0;
 
     ASSERT( fetch.ready );
     Log_dbg( "Retrieving articles marked for download" );
     list = new_DynStr( 10000 );
-    if ( Req_first( fetch.serv, msgId ) ) do { artmax++; } while ( Req_next( msgId ) );
-    Log_inf( "%d TOTAL messages to download", artmax);
+    fetchList = new_DynStr( 1000 );
+    if ( list == NULL || fetchList == NULL )
+    {
+	if ( list != NULL )
+	    del_DynStr( list );
+        Log_err( "Out of memory in Fetch_get_Req_");
+	return;
+    }
+
+    /*
+     * Get all waiting message IDs for this server. We copy into a master
+     * list as the requests file will be closed and re-opened during the
+     * fetch and the position therein will be lost.
+     */
+    if ( ! Lock_openDatabases() )
+    {
+        Log_err( "Could not open message base" );
+        return;
+    }
+
     if ( Req_first( fetch.serv, msgId ) )
+    {
         do
         {
             DynStr_appLn( list, msgId );
-            if ( ++count % 20 == 0 ) /* Send max. 20 ARTICLE cmds at once */
-            {
-                p = DynStr_str( list );
-                Client_retrieveArtList( p, &artcnt, artmax );
-                while ( ( p = Utl_getLn( msgId, p ) ) )
-                    Req_remove( fetch.serv, msgId );
-                DynStr_clear( list );
-            }
+	    artmax++;
         }
         while ( Req_next( msgId ) );
+	Log_inf( "%d TOTAL messages to download", artmax);
+    }
+
+    /* Retrieve in groups of up to size MAX_ARTICLE_CMDS_QUEUED. */
     p = DynStr_str( list );
-    Client_retrieveArtList( p, &artcnt, artmax );
-    while ( ( p = Utl_getLn( msgId, p ) ) )
-        Req_remove( fetch.serv, msgId );
+    while ( ( p = Utl_getLn( msgId, p ) ) != NULL )
+    {
+	DynStr_appLn( fetchList, msgId );
+	if ( ++count % MAX_ARTICLE_CMDS_QUEUED == 0 )
+	{
+	    fetchMessageList( DynStr_str( fetchList ), &artcnt, artmax );
+	    DynStr_clear( fetchList );
+	}
+    }
+    fetchMessageList( DynStr_str( fetchList ), &artcnt, artmax );
+
+    del_DynStr( fetchList );
     del_DynStr( list );
+    Lock_closeDatabases();
 }
 
 static void
@@ -256,6 +326,7 @@
 Bool
 Fetch_init( const char *serv )
 {
+    Lock_closeDatabases();
     if ( ! connectToServ( serv ) )
         return FALSE;
     Utl_cpyStr( fetch.serv, serv );
@@ -269,4 +340,5 @@
     Client_disconnect();
     fetch.ready = FALSE;
     Log_inf( "Fetch from '%s' finished", fetch.serv );
+    Lock_openDatabases();
 }