Mercurial > noffle

comparison src/database.c @ 255:52f467c7213b noffle

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn] * docs/noffle.1,src/Makefile.am,src/Makefile.in,src/content.c, src/content.h,src/database.c,src/database.h,src/expire.c, src/expire.h,src/noffle.c: Split out expire code from database.c, change to remove articles in place (rather than rebuild article database) and add separate command to rebuild article database from articles listed in overviews. This may help if the article database gets corrupted.
author bears
date Wed, 26 Jun 2002 14:15:44 +0100
parents 7a830ce3211e
children 087e7039b569
comparison
equal deleted inserted replaced
254:4c0f54d51591 255:52f467c7213b
1 /* 1 /*
2 database.c 2 database.c
3 3
4 $Id: database.c 379 2002-03-26 17:52:01Z mirkol $ 4 $Id: database.c 387 2002-06-26 13:15:44Z bears $
5 5
6 Uses GNU gdbm library. Using Berkeley db (included in libc6) was 6 Uses GNU gdbm library. Using Berkeley db (included in libc6) was
7 cumbersome. It is based on Berkeley db 1.85, which has severe bugs 7 cumbersome. It is based on Berkeley db 1.85, which has severe bugs
8 (e.g. it is not recommended to delete or overwrite entries with 8 (e.g. it is not recommended to delete or overwrite entries with
9 overflow pages). 9 overflow pages).
14 #endif 14 #endif
15 15
16 #include <stdio.h> 16 #include <stdio.h>
17 #include <ctype.h> 17 #include <ctype.h>
18 #include <errno.h> 18 #include <errno.h>
19 #include <fcntl.h>
20 #include <gdbm.h> 19 #include <gdbm.h>
21 #include <unistd.h> 20 #include <unistd.h>
22 #include <sys/types.h> 21 #include <sys/types.h>
23 #include <sys/stat.h> 22 #include <sys/stat.h>
24 #include "configfile.h" 23 #include "configfile.h"
24 #include "content.h"
25 #include "database.h" 25 #include "database.h"
26 #include "itemlist.h" 26 #include "group.h"
27 #include "log.h" 27 #include "log.h"
28 #include "protocol.h" 28 #include "protocol.h"
29 #include "util.h" 29 #include "util.h"
30 #include "portable.h" 30 #include "portable.h"
31
32 static const char ARTICLE_FILENAME_FMT[] = "%s/data/articles.gdbm";
33 static const char ARTICLE_NEW_FILENAME_FMT[] = "%s/data/articles.gdbm.new";
34
31 35
32 static struct Db 36 static struct Db
33 { 37 {
34 GDBM_FILE dbf; 38 GDBM_FILE dbf;
35 39
58 } db = { NULL, "(unknown)", "", 0, 0, "", "", "", "", "", 0, 0, NULL }; 62 } db = { NULL, "(unknown)", "", 0, 0, "", "", "", "", "", 0, 0, NULL };
59 63
60 static const char * 64 static const char *
61 errMsg( void ) 65 errMsg( void )
62 { 66 {
63 if ( errno != 0 ) 67 if ( gdbm_errno == GDBM_NO_ERROR )
64 return strerror( errno ); 68 return strerror( errno );
65 return gdbm_strerror( gdbm_errno ); 69 return gdbm_strerror( gdbm_errno );
66 } 70 }
67 71
68 Bool 72 Bool
70 { 74 {
71 Str name, host; 75 Str name, host;
72 int flags; 76 int flags;
73 77
74 ASSERT( db.dbf == NULL ); 78 ASSERT( db.dbf == NULL );
75 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() ); 79 snprintf( name, MAXCHAR, ARTICLE_FILENAME_FMT, Cfg_spoolDir() );
76 flags = GDBM_WRCREAT | GDBM_FAST; 80 flags = GDBM_WRCREAT | GDBM_FAST;
77 81
78 if ( ! ( db.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) ) 82 if ( ! ( db.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
79 { 83 {
80 Log_err( "Error opening %s for r/w (%s)", name, errMsg() ); 84 Log_err( "Error opening %s for r/w (%s)", name, errMsg() );
580 free( oldDptr ); 584 free( oldDptr );
581 *msgId = cursor.dptr; 585 *msgId = cursor.dptr;
582 return ( cursor.dptr != NULL ); 586 return ( cursor.dptr != NULL );
583 } 587 }
584 588
585 static int 589 void
586 calcExpireDays( const char *msgId ) 590 Db_compact( void )
587 { 591 {
588 const char *xref; 592 ASSERT( db.dbf );
589 ItemList *refs; 593 if ( gdbm_reorganize( db.dbf ) != 0 )
590 const char *ref; 594 Log_err( "Error compacting article base: %s", errMsg() );
591 int res; 595 }
592 596
593 xref = Db_xref( msgId ); 597 /*
594 if ( xref[ 0 ] == '\0' ) 598 Helper functions for database rebuild.
595 return -1; 599 */
596 600
597 res = -1; 601 static struct DbNew
598 refs = new_Itl( xref, " :" ); 602 {
599 for ( ref = Itl_first( refs ); ref != NULL; ref = Itl_next( refs ) ) 603 GDBM_FILE dbf;
600 { 604
601 int days; 605 } dbNew = { NULL };
602 606
603 days = Cfg_expire( ref ); 607
604 if ( days == 0 608 static Bool
605 || ( days > res && res != 0 ) ) 609 newOpen( void )
606 res = days; 610 {
611 Str name;
612 int flags;
613
614 ASSERT( dbNew.dbf == NULL );
615 snprintf( name, MAXCHAR, ARTICLE_NEW_FILENAME_FMT, Cfg_spoolDir() );
616 flags = GDBM_WRCREAT | GDBM_FAST;
617
618 if ( ! ( dbNew.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
619 {
620 Log_err( "Error opening %s for r/w (%s)", name, errMsg() );
621 return FALSE;
622 }
623 Log_dbg( LOG_DBG_NEWSBASE, "%s opened for r/w", name );
624 return TRUE;
625 }
626
627 static Bool
628 newClose( Bool makeMain )
629 {
630 Str newName;
631
632 ASSERT( dbNew.dbf );
633 Log_dbg( LOG_DBG_NEWSBASE, "Closing new database" );
634 gdbm_close( dbNew.dbf );
635 dbNew.dbf = NULL;
636
637 snprintf( newName, MAXCHAR, ARTICLE_NEW_FILENAME_FMT, Cfg_spoolDir() );
638
639 if ( makeMain )
640 {
641 Str name;
642
643 ASSERT( db.dbf );
644 Db_close();
645 snprintf( name, MAXCHAR, ARTICLE_FILENAME_FMT, Cfg_spoolDir() );
646 if ( rename( newName, name ) != 0 )
647 {
648 Log_err( "Rename %s to %s failed: %s",
649 newName, name, strerror( errno ) );
650 return FALSE;
651 }
652 Log_dbg( LOG_DBG_NEWSBASE, "Renamed %s to %s", newName, name );
653 return Db_open();
654 }
655 else
656 {
657 if ( unlink( newName ) != 0 )
658 {
659 Log_err( "Unlink %s failed: %s", newName, strerror( errno ) );
660 return FALSE;
661 }
662 Log_dbg( LOG_DBG_NEWSBASE, "Deleted %s", newName );
663 return TRUE;
664 }
665 }
666
667 static Bool
668 newCopyArt( const char *msgId )
669 {
670 datum key, val;
671
672 ASSERT( db.dbf );
673 ASSERT( dbNew.dbf );
674 key.dptr = (void *)msgId;
675 key.dsize = strlen( msgId ) + 1;
676
677 val = gdbm_fetch( db.dbf, key );
678 if ( val.dptr != NULL )
679 {
680 Bool res;
607 681
608 Itl_next( refs ); /* Throw away group number */ 682 res = ( gdbm_store( dbNew.dbf, key, val, GDBM_INSERT ) == 0 );
609 } 683 if ( ! res )
610 del_Itl( refs ); 684 Log_err( "Could not store %s in new database (%s)",
611 685 msgId, errMsg() );
612 return res; 686 free( val.dptr );
687 return res;
688 }
689 Log_err( "%s not found in database", msgId );
690 return FALSE;
691 }
692
693 static Bool
694 newContains( const char *msgId )
695 {
696 datum key;
697
698 ASSERT( dbNew.dbf );
699 key.dptr = (void*)msgId;
700 key.dsize = strlen( msgId ) + 1;
701 return gdbm_exists( dbNew.dbf, key );
613 } 702 }
614 703
615 Bool 704 Bool
616 Db_expire( void ) 705 Db_rebuild( void )
617 { 706 {
618 int cntDel, cntLeft, flags, expDays; 707 const Over *ov;
619 time_t nowTime, lastAccess; 708 int i;
709 Str grp;
620 const char *msgId; 710 const char *msgId;
621 Str name, tmpName; 711 Bool err;
622 GDBM_FILE tmpDbf; 712
623 datum key, val; 713 if ( ! Cont_firstGrp( grp ) )
624 Str expires; 714 return FALSE;
625 time_t texpires; 715 if ( ! newOpen() )
626 716 return FALSE;
627 if ( ! Db_open() ) 717
628 return FALSE; 718 Log_inf( "Rebuilding article database" );
629 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() ); 719 err = FALSE;
630 snprintf( tmpName, MAXCHAR, "%s/data/articles.gdbm.new", Cfg_spoolDir() ); 720 do
631 flags = GDBM_NEWDB | GDBM_FAST; 721 {
632 if ( ! ( tmpDbf = gdbm_open( tmpName, 512, flags, 0644, NULL ) ) ) 722 if ( ! Grp_exists( grp ) )
633 { 723 Log_err( "Overview file for unknown group %s exists", grp );
634 Log_err( "Error opening %s for read/write (%s)", tmpName, errMsg() ); 724 else
635 Db_close();
636 return FALSE;
637 }
638 Log_inf( "Expiring articles" );
639 cntDel = 0;
640 cntLeft = 0;
641 nowTime = time( NULL );
642 if ( Db_first( &msgId ) )
643 do
644 { 725 {
645 expDays = calcExpireDays( msgId ); 726 Cont_read( grp );
646 lastAccess = Db_lastAccess( msgId ); 727 for ( i = Cont_first(); i <= Cont_last(); ++i )
647 if ( Prt_searchHeader( Db_header( msgId ), "Expires", expires ) )
648 texpires = Utl_parseNewsDate( expires );
649 else
650 texpires = (time_t) -1;
651
652 if ( expDays == -1 )
653 Log_err( "Internal error: Failed expiry calculation on %s",
654 msgId );
655 else if ( lastAccess == -1 )
656 Log_err( "Internal error: Getting lastAccess of %s failed",
657 msgId );
658 else if ( expDays > 0
659 && difftime( nowTime, lastAccess ) >
660 ( (double) expDays * 24 * 3600 ) )
661 {
662 #ifdef DEBUG
663 Str last, now;
664
665 Utl_cpyStr( last, ctime( &lastAccess ) );
666 last[ strlen( last ) - 1 ] = '\0';
667 Utl_cpyStr( now, ctime( &nowTime ) );
668 last[ strlen( now ) - 1 ] = '\0';
669 Log_dbg( LOG_DBG_EXPIRE,
670 "Expiring %s: last access %s, time now %s",
671 msgId, last, now );
672 #endif
673 ++cntDel;
674 }
675 else if ( ( texpires != (time_t) -1 )
676 && nowTime > texpires )
677 { 728 {
678 Log_dbg( LOG_DBG_EXPIRE, 729 if ( ! Cont_validNumb( i ) )
679 "Expiring %s: Expires header activated", msgId ); 730 continue;
680 ++cntDel; 731
732 if ( ( ov = Cont_get( i ) ) )
733 {
734 msgId = Ov_msgId( ov );
735 if ( msgId == NULL )
736 {
737 err = TRUE;
738 Log_err( "Overview in %s has no msg id", grp );
739 }
740 else if ( ! newContains( msgId ) )
741 err |= ! newCopyArt( msgId );
742 }
743 else
744 {
745 err = TRUE;
746 Log_err( "Overview %d not available in group %s", i, grp );
747 }
681 } 748 }
682 else
683 {
684 ++cntLeft;
685 key.dptr = (void *)msgId;
686 key.dsize = strlen( msgId ) + 1;
687
688 val = gdbm_fetch( db.dbf, key );
689 if ( val.dptr != NULL )
690 {
691 if ( gdbm_store( tmpDbf, key, val, GDBM_INSERT ) != 0 )
692 Log_err( "Could not store %s in new database (%s)",
693 errMsg() );
694 free( val.dptr );
695 }
696 }
697 } 749 }
698 while ( Db_next( &msgId ) ); 750 }
699 Log_inf( "%lu articles deleted, %lu left", cntDel, cntLeft ); 751 while ( Cont_nextGrp( grp ) );
700 gdbm_close( tmpDbf ); 752
701 Db_close(); 753 return newClose( ! err );
702 rename( tmpName, name ); 754 }
703 return TRUE; 755
704 }