--- a/src/database.c	Wed Jun 26 14:14:56 2002 +0100
+++ b/src/database.c	Wed Jun 26 14:15:44 2002 +0100
@@ -1,7 +1,7 @@
 /*
   database.c
 
-  $Id: database.c 379 2002-03-26 17:52:01Z mirkol $
+  $Id: database.c 387 2002-06-26 13:15:44Z bears $
 
   Uses GNU gdbm library. Using Berkeley db (included in libc6) was
   cumbersome. It is based on Berkeley db 1.85, which has severe bugs
@@ -16,19 +16,23 @@
 #include <stdio.h>
 #include <ctype.h>
 #include <errno.h>
-#include <fcntl.h>
 #include <gdbm.h>
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include "configfile.h"
+#include "content.h"
 #include "database.h"
-#include "itemlist.h"
+#include "group.h"
 #include "log.h"
 #include "protocol.h"
 #include "util.h"
 #include "portable.h"
 
+static const char ARTICLE_FILENAME_FMT[] = "%s/data/articles.gdbm";
+static const char ARTICLE_NEW_FILENAME_FMT[] = "%s/data/articles.gdbm.new";
+
+
 static struct Db
 {
     GDBM_FILE dbf;
@@ -60,7 +64,7 @@
 static const char *
 errMsg( void )
 {
-    if ( errno != 0 )
+    if ( gdbm_errno == GDBM_NO_ERROR )
         return strerror( errno );
     return gdbm_strerror( gdbm_errno );
 }
@@ -72,7 +76,7 @@
     int flags;
 
     ASSERT( db.dbf == NULL );
-    snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
+    snprintf( name, MAXCHAR, ARTICLE_FILENAME_FMT, Cfg_spoolDir() );
     flags = GDBM_WRCREAT | GDBM_FAST;
 
     if ( ! ( db.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
@@ -582,123 +586,170 @@
     return ( cursor.dptr != NULL );
 }
 
-static int
-calcExpireDays( const char *msgId )
+void
+Db_compact( void )
+{
+    ASSERT( db.dbf );
+    if ( gdbm_reorganize( db.dbf ) != 0 )
+	Log_err( "Error compacting article base: %s", errMsg() );
+}
+
+/*
+  Helper functions for database rebuild.
+*/
+
+static struct DbNew
+{
+    GDBM_FILE dbf;
+
+} dbNew = { NULL };
+
+
+static Bool
+newOpen( void )
 {
-    const char *xref;
-    ItemList *refs;
-    const char *ref;
-    int res;
+    Str name;
+    int flags;
+
+    ASSERT( dbNew.dbf == NULL );
+    snprintf( name, MAXCHAR, ARTICLE_NEW_FILENAME_FMT, Cfg_spoolDir() );
+    flags = GDBM_WRCREAT | GDBM_FAST;
+
+    if ( ! ( dbNew.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
+    {
+        Log_err( "Error opening %s for r/w (%s)", name, errMsg() );
+        return FALSE;
+    }
+    Log_dbg( LOG_DBG_NEWSBASE, "%s opened for r/w", name );
+    return TRUE;
+}
 
-    xref = Db_xref( msgId );
-    if ( xref[ 0 ] == '\0' )
-	return -1;
+static Bool
+newClose( Bool makeMain )
+{
+    Str newName;
+    
+    ASSERT( dbNew.dbf );
+    Log_dbg( LOG_DBG_NEWSBASE, "Closing new database" );
+    gdbm_close( dbNew.dbf );
+    dbNew.dbf = NULL;
+
+    snprintf( newName, MAXCHAR, ARTICLE_NEW_FILENAME_FMT, Cfg_spoolDir() );
+
+    if ( makeMain )
+    {
+	Str name;
 
-    res = -1;
-    refs = new_Itl( xref, " :" );
-    for ( ref = Itl_first( refs ); ref != NULL; ref = Itl_next( refs ) )
+	ASSERT( db.dbf );
+	Db_close();
+	snprintf( name, MAXCHAR, ARTICLE_FILENAME_FMT, Cfg_spoolDir() );
+	if ( rename( newName, name ) != 0 )
+	{
+	    Log_err( "Rename %s to %s failed: %s",
+		     newName, name, strerror( errno ) );
+	    return FALSE;
+	}
+	Log_dbg( LOG_DBG_NEWSBASE, "Renamed %s to %s", newName, name );
+	return Db_open();
+    }
+    else
     {
-	int days;
+	if ( unlink( newName ) != 0 )
+	{
+	    Log_err( "Unlink %s failed: %s", newName, strerror( errno ) );
+	    return FALSE;
+	}
+	Log_dbg( LOG_DBG_NEWSBASE, "Deleted %s", newName );
+	return TRUE;
+    }
+}
+
+static Bool
+newCopyArt( const char *msgId )
+{
+    datum key, val;
 
-	days = Cfg_expire( ref );
-	if ( days == 0
-	     || ( days > res && res != 0 ) )
-	    res = days;
+    ASSERT( db.dbf );
+    ASSERT( dbNew.dbf );
+    key.dptr = (void *)msgId;
+    key.dsize = strlen( msgId ) + 1;
+
+    val = gdbm_fetch( db.dbf, key );
+    if ( val.dptr != NULL )
+    {
+	Bool res;
 	
-	Itl_next( refs );	/* Throw away group number */
+	res = ( gdbm_store( dbNew.dbf, key, val, GDBM_INSERT ) == 0 );
+	if ( ! res )
+	    Log_err( "Could not store %s in new database (%s)",
+		     msgId, errMsg() );
+	free( val.dptr );
+	return res;
     }
-    del_Itl( refs );
+    Log_err( "%s not found in database", msgId );
+    return FALSE;
+}
 
-    return res;
+static Bool
+newContains( const char *msgId )
+{
+    datum key;
+
+    ASSERT( dbNew.dbf );
+    key.dptr = (void*)msgId;
+    key.dsize = strlen( msgId ) + 1;
+    return gdbm_exists( dbNew.dbf, key );
 }
 
 Bool
-Db_expire( void )
+Db_rebuild( void )
 {
-    int cntDel, cntLeft, flags, expDays;
-    time_t nowTime, lastAccess;
+    const Over *ov;
+    int i;
+    Str grp;
     const char *msgId;
-    Str name, tmpName;
-    GDBM_FILE tmpDbf;
-    datum key, val;
-    Str expires;
-    time_t texpires;
+    Bool err;
 
-    if ( ! Db_open() )
+    if ( ! Cont_firstGrp( grp ) )
         return FALSE;
-    snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
-    snprintf( tmpName, MAXCHAR, "%s/data/articles.gdbm.new", Cfg_spoolDir() );
-    flags = GDBM_NEWDB | GDBM_FAST;
-    if ( ! ( tmpDbf = gdbm_open( tmpName, 512, flags, 0644, NULL ) ) )
+    if ( ! newOpen() )
+	return FALSE;
+    
+    Log_inf( "Rebuilding article database" );
+    err = FALSE;
+    do
     {
-        Log_err( "Error opening %s for read/write (%s)", tmpName, errMsg() );
-        Db_close();
-        return FALSE;
-    }
-    Log_inf( "Expiring articles" );
-    cntDel = 0;
-    cntLeft = 0;
-    nowTime = time( NULL );
-    if ( Db_first( &msgId ) )
-        do
+	if ( ! Grp_exists( grp ) )
+            Log_err( "Overview file for unknown group %s exists", grp );
+        else
         {
-	    expDays = calcExpireDays( msgId );
-            lastAccess = Db_lastAccess( msgId );
-	    if ( Prt_searchHeader( Db_header( msgId ), "Expires", expires ) )
-		texpires = Utl_parseNewsDate( expires );
-	    else
-		texpires = (time_t) -1;
-	    
-	    if ( expDays == -1 )
-		Log_err( "Internal error: Failed expiry calculation on %s",
-			 msgId );
-	    else if ( lastAccess == -1 )
-                Log_err( "Internal error: Getting lastAccess of %s failed",
-                         msgId );
-            else if ( expDays > 0
-		      && difftime( nowTime, lastAccess ) >
-		          ( (double) expDays * 24 * 3600 ) )
-            {
-#ifdef DEBUG
-		Str last, now;
+            Cont_read( grp );
+            for ( i = Cont_first(); i <= Cont_last(); ++i )
+	    {
+		if ( ! Cont_validNumb( i ) )
+		    continue;
+		
+                if ( ( ov = Cont_get( i ) ) )
+                {
+                    msgId = Ov_msgId( ov );
+		    if ( msgId == NULL )
+		    {
+			err = TRUE;
+			Log_err( "Overview in %s has no msg id", grp );
+		    }
+		    else if ( ! newContains( msgId ) )
+			err |= ! newCopyArt( msgId );
+                }
+		else
+		{    
+		    err = TRUE;
+		    Log_err( "Overview %d not available in group %s", i, grp );
+		}
+	    }
+        }
+    }
+    while ( Cont_nextGrp( grp ) );
 
-		Utl_cpyStr( last, ctime( &lastAccess ) );
-		last[ strlen( last ) - 1 ] = '\0';
-		Utl_cpyStr( now, ctime( &nowTime ) );
-		last[ strlen( now ) - 1 ] = '\0';
-                Log_dbg( LOG_DBG_EXPIRE,
-			 "Expiring %s: last access %s, time now %s",
-			 msgId, last, now );
-#endif
-                ++cntDel;
-            }
-	    else if ( ( texpires != (time_t) -1 )
-		      && nowTime > texpires )
-	    {
-		Log_dbg( LOG_DBG_EXPIRE,
-			 "Expiring %s: Expires header activated", msgId );
-		++cntDel;
-	    }
-            else
-            {
-                ++cntLeft;
-                key.dptr = (void *)msgId;
-                key.dsize = strlen( msgId ) + 1;
+    return newClose( ! err );
+}
 
-                val = gdbm_fetch( db.dbf, key );
-                if ( val.dptr != NULL )
-                {
-                    if ( gdbm_store( tmpDbf, key, val, GDBM_INSERT ) != 0 )
-                        Log_err( "Could not store %s in new database (%s)",
-                                 errMsg() );
-                    free( val.dptr );
-                }
-            }
-        }
-        while ( Db_next( &msgId ) );
-    Log_inf( "%lu articles deleted, %lu left", cntDel, cntLeft );
-    gdbm_close( tmpDbf );
-    Db_close();
-    rename( tmpName, name );
-    return TRUE;
-}