Mercurial > noffle

comparison src/database.c @ 43:2842f50feb55 noffle

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn] * client.c, client.h, common.h, config.c, config.h, content.c, content.h, control.c, control.h, database.c, database.h, dynamicstring.c, dynamicstring.h, fetch.c, fetch.h, fetchlist.c, fetchlist.h, group.c, group.h, itemlist.c, itemlist.h, lock.c, lock.h, log.c, log.h, noffle.c, online.c, online.h, outgoing.c, outgoing.h, over.c, over.h, post.c, post.h, protocol.c, protocol.h, pseudo.c, pseudo.h, request.c, request.h, server.c, server.h, util.c, util.h, wildmat.c, wildmat.h: Moved files to the subdirectory src/ * Makefile.am, acconfig.h, configure.in, docs/Makefile.am, src/Makefile.am, Makefile.in, aclocal.m4, config.h.in, configure, install-sh, missing, mkinstalldirs, stamp-h.in, docs/Makefile.in, src/Makefile.in: Added files. They are used by aclocal, autoheader, autoconf and automake. * src/config.c, src/config.h: Renamed to configfile.c and configfile.h, because configure will generate a config.h file itself. * src/client.c, src/content.c, src/database.c, src/fetch.c, src/fetchlist.c, src/group.c, src/lock.c, src/noffle.c, src/online.c, src/outgoing.c, src/over.c, src/pseudo.c, src/request.c, src/server.c, src/util.c: Changed '#include "config.h"' to '#include "configfile.h"'. * src/client.c, src/content.c, src/database.c, src/fetch.c, src/fetchlist.c, src/group.c, src/lock.c, src/online.c, src/outgoing.c, src/post.c, src/protocol.c, src/request.c, src/server.c: Files now #include <config.h>. Added missing <stdio.h>. This removes the warnings about snprintf() not being declared. * Makefile: Removed. This is now generated by configure.
author uh1763
date Fri, 05 May 2000 22:45:56 +0100
parents
children 125d79c9e586
comparison
equal deleted inserted replaced
42:2467ff423c15 43:2842f50feb55
1 /*
2 database.c
3
4 $Id: database.c 49 2000-05-05 21:45:56Z uh1763 $
5
6 Uses GNU gdbm library. Using Berkeley db (included in libc6) was
7 cumbersome. It is based on Berkeley db 1.85, which has severe bugs
8 (e.g. it is not recommended to delete or overwrite entries with
9 overflow pages).
10 */
11
12 #if HAVE_CONFIG_H
13 #include <config.h>
14 #endif
15
16 #include <stdio.h>
17 #include "database.h"
18 #include <ctype.h>
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <gdbm.h>
22 #include <unistd.h>
23 #include <sys/types.h>
24 #include <sys/stat.h>
25 #include "configfile.h"
26 #include "itemlist.h"
27 #include "log.h"
28 #include "protocol.h"
29 #include "util.h"
30 #include "wildmat.h"
31
32 static struct Db
33 {
34 GDBM_FILE dbf;
35
36 /* Start string for Xref header line: "Xref: <host>" */
37 Str xrefHost;
38
39 /* Msg Id of presently loaded article, empty if none loaded */
40 Str msgId;
41
42 /* Status of loaded article */
43 int stat; /* Flags */
44 time_t lastAccess;
45
46 /* Overview of loaded article */
47 Str subj;
48 Str from;
49 Str date;
50 Str ref;
51 Str xref;
52 size_t bytes;
53 size_t lines;
54
55 /* Article text (except for overview header lines) */
56 DynStr *txt;
57
58 } db = { NULL, "(unknown)", "", 0, 0, "", "", "", "", "", 0, 0, NULL };
59
60 static const char *
61 errMsg( void )
62 {
63 if ( errno != 0 )
64 return strerror( errno );
65 return gdbm_strerror( gdbm_errno );
66 }
67
68 Bool
69 Db_open( void )
70 {
71 Str name, host;
72 int flags;
73
74 ASSERT( db.dbf == NULL );
75 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
76 flags = GDBM_WRCREAT | GDBM_FAST;
77
78 if ( ! ( db.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
79 {
80 Log_err( "Error opening %s for r/w (%s)", name, errMsg() );
81 return FALSE;
82 }
83 Log_dbg( "%s opened for r/w", name );
84
85 if ( db.txt == NULL )
86 db.txt = new_DynStr( 5000 );
87
88 gethostname( host, MAXCHAR );
89 snprintf( db.xrefHost, MAXCHAR, "Xref: %s", host );
90
91 return TRUE;
92 }
93
94 void
95 Db_close( void )
96 {
97 ASSERT( db.dbf );
98 Log_dbg( "Closing database" );
99 gdbm_close( db.dbf );
100 db.dbf = NULL;
101 del_DynStr( db.txt );
102 db.txt = NULL;
103 Utl_cpyStr( db.msgId, "" );
104 }
105
106 static Bool
107 loadArt( const char *msgId )
108 {
109 static void *dptr = NULL;
110
111 datum key, val;
112 Str t = "";
113 const char *p;
114
115 ASSERT( db.dbf );
116
117 if ( strcmp( msgId, db.msgId ) == 0 )
118 return TRUE;
119
120 key.dptr = (void *)msgId;
121 key.dsize = strlen( msgId ) + 1;
122 if ( dptr != NULL )
123 {
124 free( dptr );
125 dptr = NULL;
126 }
127 val = gdbm_fetch( db.dbf, key );
128 dptr = val.dptr;
129 if ( dptr == NULL )
130 {
131 Log_dbg( "database.c loadArt: gdbm_fetch found no entry" );
132 return FALSE;
133 }
134
135 Utl_cpyStr( db.msgId, msgId );
136 p = Utl_getLn( t, (char *)dptr );
137 if ( ! p || sscanf( t, "%x", &db.stat ) != 1 )
138 {
139 Log_err( "Entry in database '%s' is corrupt (status)", msgId );
140 return FALSE;
141 }
142 p = Utl_getLn( t, p );
143 if ( ! p || sscanf( t, "%lu", &db.lastAccess ) != 1 )
144 {
145 Log_err( "Entry in database '%s' is corrupt (lastAccess)", msgId );
146 return FALSE;
147 }
148 p = Utl_getLn( db.subj, p );
149 p = Utl_getLn( db.from, p );
150 p = Utl_getLn( db.date, p );
151 p = Utl_getLn( db.ref, p );
152 p = Utl_getLn( db.xref, p );
153 if ( ! p )
154 {
155 Log_err( "Entry in database '%s' is corrupt (overview)", msgId );
156 return FALSE;
157 }
158 p = Utl_getLn( t, p );
159 if ( ! p || sscanf( t, "%u", &db.bytes ) != 1 )
160 {
161 Log_err( "Entry in database '%s' is corrupt (bytes)", msgId );
162 return FALSE;
163 }
164 p = Utl_getLn( t, p );
165 if ( ! p || sscanf( t, "%u", &db.lines ) != 1 )
166 {
167 Log_err( "Entry in database '%s' is corrupt (lines)", msgId );
168 return FALSE;
169 }
170 DynStr_clear( db.txt );
171 DynStr_app( db.txt, p );
172 return TRUE;
173 }
174
175 static Bool
176 saveArt( void )
177 {
178 DynStr *s;
179 Str t = "";
180 datum key, val;
181
182 if ( strcmp( db.msgId, "" ) == 0 )
183 return FALSE;
184 s = new_DynStr( 5000 );
185 snprintf( t, MAXCHAR, "%x", db.stat );
186 DynStr_appLn( s, t );
187 snprintf( t, MAXCHAR, "%lu", db.lastAccess );
188 DynStr_appLn( s, t );
189 DynStr_appLn( s, db.subj );
190 DynStr_appLn( s, db.from );
191 DynStr_appLn( s, db.date );
192 DynStr_appLn( s, db.ref );
193 DynStr_appLn( s, db.xref );
194 snprintf( t, MAXCHAR, "%u", db.bytes );
195 DynStr_appLn( s, t );
196 snprintf( t, MAXCHAR, "%u", db.lines );
197 DynStr_appLn( s, t );
198 DynStr_appDynStr( s, db.txt );
199
200 key.dptr = (void *)db.msgId;
201 key.dsize = strlen( db.msgId ) + 1;
202 val.dptr = (void *)DynStr_str( s );
203 val.dsize = DynStr_len( s ) + 1;
204 if ( gdbm_store( db.dbf, key, val, GDBM_REPLACE ) != 0 )
205 {
206 Log_err( "Could not store %s in database (%s)", errMsg() );
207 return FALSE;
208 }
209
210 del_DynStr( s );
211 return TRUE;
212 }
213
214 Bool
215 Db_prepareEntry( const Over *ov, const char *grp, int numb )
216 {
217 const char *msgId;
218
219 ASSERT( db.dbf );
220 ASSERT( ov );
221 ASSERT( grp );
222
223 msgId = Ov_msgId( ov );
224 Log_dbg( "Preparing entry %s", msgId );
225 if ( Db_contains( msgId ) )
226 Log_err( "Preparing article twice: %s", msgId );
227
228 db.stat = DB_NOT_DOWNLOADED;
229 db.lastAccess = time( NULL );
230
231 Utl_cpyStr( db.msgId, msgId );
232 Utl_cpyStr( db.subj, Ov_subj( ov ) );
233 Utl_cpyStr( db.from, Ov_from( ov ) );
234 Utl_cpyStr( db.date, Ov_date( ov ) );
235 Utl_cpyStr( db.ref, Ov_ref( ov ) );
236 snprintf( db.xref, MAXCHAR, "%s:%i", grp, numb );
237 db.bytes = Ov_bytes( ov );
238 db.lines = Ov_lines( ov );
239
240 DynStr_clear( db.txt );
241
242 return saveArt();
243 }
244
245 Bool
246 Db_storeArt( const char *msgId, const char *artTxt )
247 {
248 Str line, lineEx, field, value;
249 const char *startPos;
250
251 ASSERT( db.dbf );
252
253 Log_dbg( "Store article %s", msgId );
254 if ( ! loadArt( msgId ) )
255 {
256 Log_err( "Cannot find info about '%s' in database", msgId );
257 return FALSE;
258 }
259 if ( ! ( db.stat & DB_NOT_DOWNLOADED ) )
260 {
261 Log_err( "Trying to store already retrieved article '%s'", msgId );
262 return FALSE;
263 }
264 db.stat &= ~DB_NOT_DOWNLOADED;
265 db.stat &= ~DB_RETRIEVING_FAILED;
266 db.lastAccess = time( NULL );
267
268 DynStr_clear( db.txt );
269
270 /* Read header */
271 startPos = artTxt;
272 while ( TRUE )
273 {
274 artTxt = Utl_getHeaderLn( lineEx, artTxt );
275 if ( lineEx[ 0 ] == '\0' )
276 {
277 DynStr_appLn( db.txt, lineEx );
278 break;
279 }
280 /* Remove fields already in overview and handle x-noffle
281 headers correctly in case of cascading NOFFLEs */
282 if ( Prt_getField( field, value, lineEx ) )
283 {
284 if ( strcmp( field, "x-noffle-status" ) == 0 )
285 {
286 if ( strstr( value, "NOT_DOWNLOADED" ) != 0 )
287 db.stat |= DB_NOT_DOWNLOADED;
288 }
289 else if ( strcmp( field, "message-id" ) != 0
290 && strcmp( field, "xref" ) != 0
291 && strcmp( field, "references" ) != 0
292 && strcmp( field, "subject" ) != 0
293 && strcmp( field, "from" ) != 0
294 && strcmp( field, "date" ) != 0
295 && strcmp( field, "bytes" ) != 0
296 && strcmp( field, "lines" ) != 0
297 && strcmp( field, "x-noffle-lastaccess" ) != 0 )
298 DynStr_appLn( db.txt, lineEx );
299 }
300 }
301
302 /* Read body */
303 while ( ( artTxt = Utl_getLn( line, artTxt ) ) )
304 if ( ! ( db.stat & DB_NOT_DOWNLOADED ) )
305 DynStr_appLn( db.txt, line );
306
307 return saveArt();
308 }
309
310 void
311 Db_setStat( const char *msgId, int stat )
312 {
313 if ( loadArt( msgId ) )
314 {
315 db.stat = stat;
316 saveArt();
317 }
318 }
319
320 void
321 Db_updateLastAccess( const char *msgId )
322 {
323 if ( loadArt( msgId ) )
324 {
325 db.lastAccess = time( NULL );
326 saveArt();
327 }
328 }
329
330 void
331 Db_setXref( const char *msgId, const char *xref )
332 {
333 if ( loadArt( msgId ) )
334 {
335 Utl_cpyStr( db.xref, xref );
336 saveArt();
337 }
338 }
339
340 /* Search best position for breaking a line */
341 static const char *
342 searchBreakPos( const char *line, int wantedLength )
343 {
344 const char *lastSpace = NULL;
345 Bool lastWasSpace = FALSE;
346 int len = 0;
347
348 while ( *line != '\0' )
349 {
350 if ( isspace( *line ) )
351 {
352 if ( len > wantedLength && lastSpace != NULL )
353 return lastSpace;
354 if ( ! lastWasSpace )
355 lastSpace = line;
356 lastWasSpace = TRUE;
357 }
358 else
359 lastWasSpace = FALSE;
360 ++len;
361 ++line;
362 }
363 if ( len > wantedLength && lastSpace != NULL )
364 return lastSpace;
365 return line;
366 }
367
368 /* Append header line by breaking long line into multiple lines */
369 static void
370 appendLongHeader( DynStr *target, const char *field, const char *value )
371 {
372 const int wantedLength = 78;
373 const char *breakPos, *old;
374 int len;
375
376 len = strlen( field );
377 DynStr_appN( target, field, len );
378 DynStr_appN( target, " ", 1 );
379 old = value;
380 while ( isspace( *old ) )
381 ++old;
382 breakPos = searchBreakPos( old, wantedLength - len - 1 );
383 DynStr_appN( target, old, breakPos - old );
384 if ( *breakPos == '\0' )
385 {
386 DynStr_appN( target, "\n", 1 );
387 return;
388 }
389 DynStr_appN( target, "\n ", 2 );
390 while ( TRUE )
391 {
392 old = breakPos;
393 while ( isspace( *old ) )
394 ++old;
395 breakPos = searchBreakPos( old, wantedLength - 1 );
396 DynStr_appN( target, old, breakPos - old );
397 if ( *breakPos == '\0' )
398 {
399 DynStr_appN( target, "\n", 1 );
400 return;
401 }
402 DynStr_appN( target, "\n ", 2 );
403 }
404 }
405
406 const char *
407 Db_header( const char *msgId )
408 {
409 static DynStr *s = NULL;
410
411 Str date, t;
412 int stat;
413 const char *p;
414
415 if ( s == NULL )
416 s = new_DynStr( 5000 );
417 else
418 DynStr_clear( s );
419 ASSERT( db.dbf );
420 if ( ! loadArt( msgId ) )
421 return NULL;
422 strftime( date, MAXCHAR, "%Y-%m-%d %H:%M:%S",
423 localtime( &db.lastAccess ) );
424 stat = db.stat;
425 snprintf( t, MAXCHAR,
426 "Message-ID: %s\n"
427 "X-NOFFLE-Status:%s%s%s\n"
428 "X-NOFFLE-LastAccess: %s\n",
429 msgId,
430 stat & DB_INTERESTING ? " INTERESTING" : "",
431 stat & DB_NOT_DOWNLOADED ? " NOT_DOWNLOADED" : "",
432 stat & DB_RETRIEVING_FAILED ? " RETRIEVING_FAILED" : "",
433 date );
434 DynStr_app( s, t );
435 appendLongHeader( s, "Subject:", db.subj );
436 appendLongHeader( s, "From:", db.from );
437 appendLongHeader( s, "Date:", db.date );
438 appendLongHeader( s, "References:", db.ref );
439 DynStr_app( s, "Bytes: " );
440 snprintf( t, MAXCHAR, "%u", db.bytes );
441 DynStr_appLn( s, t );
442 DynStr_app( s, "Lines: " );
443 snprintf( t, MAXCHAR, "%u", db.lines );
444 DynStr_appLn( s, t );
445 appendLongHeader( s, db.xrefHost, db.xref );
446 p = strstr( DynStr_str( db.txt ), "\n\n" );
447 if ( ! p )
448 DynStr_appDynStr( s, db.txt );
449 else
450 DynStr_appN( s, DynStr_str( db.txt ), p - DynStr_str( db.txt ) + 1 );
451 return DynStr_str( s );
452 }
453
454 const char *
455 Db_body( const char *msgId )
456 {
457 const char *p;
458
459 if ( ! loadArt( msgId ) )
460 return "";
461 p = strstr( DynStr_str( db.txt ), "\n\n" );
462 if ( ! p )
463 return "";
464 return ( p + 2 );
465 }
466
467 int
468 Db_stat( const char *msgId )
469 {
470 if ( ! loadArt( msgId ) )
471 return 0;
472 return db.stat;
473 }
474
475 time_t
476 Db_lastAccess( const char *msgId )
477 {
478 if ( ! loadArt( msgId ) )
479 return -1;
480 return db.lastAccess;
481 }
482
483 const char *
484 Db_ref( const char *msgId )
485 {
486 if ( ! loadArt( msgId ) )
487 return "";
488 return db.ref;
489 }
490
491 const char *
492 Db_xref( const char *msgId )
493 {
494 if ( ! loadArt( msgId ) )
495 return "";
496 return db.xref;
497 }
498
499 const char *
500 Db_from( const char *msgId )
501 {
502 if ( ! loadArt( msgId ) )
503 return "";
504 return db.from;
505 }
506
507 const char *
508 Db_date( const char *msgId )
509 {
510 if ( ! loadArt( msgId ) )
511 return "";
512 return db.date;
513 }
514
515 Bool
516 Db_contains( const char *msgId )
517 {
518 datum key;
519
520 ASSERT( db.dbf );
521 if ( strcmp( msgId, db.msgId ) == 0 )
522 return TRUE;
523 key.dptr = (void*)msgId;
524 key.dsize = strlen( msgId ) + 1;
525 return gdbm_exists( db.dbf, key );
526 }
527
528 void
529 Db_delete( const char *msgId )
530 {
531 datum key;
532
533 ASSERT( db.dbf );
534 if ( strcmp( msgId, db.msgId ) == 0 )
535 db.msgId[ 0 ] = '\0';
536 key.dptr = (void*)msgId;
537 key.dsize = strlen( msgId ) + 1;
538 gdbm_delete( db.dbf, key );
539 }
540
541 static datum cursor = { NULL, 0 };
542
543 Bool
544 Db_first( const char** msgId )
545 {
546 ASSERT( db.dbf );
547 if ( cursor.dptr != NULL )
548 {
549 free( cursor.dptr );
550 cursor.dptr = NULL;
551 }
552 cursor = gdbm_firstkey( db.dbf );
553 *msgId = cursor.dptr;
554 return ( cursor.dptr != NULL );
555 }
556
557 Bool
558 Db_next( const char** msgId )
559 {
560 void *oldDptr = cursor.dptr;
561
562 ASSERT( db.dbf );
563 if ( cursor.dptr == NULL )
564 return FALSE;
565 cursor = gdbm_nextkey( db.dbf, cursor );
566 free( oldDptr );
567 *msgId = cursor.dptr;
568 return ( cursor.dptr != NULL );
569 }
570
571 static int
572 calcExpireDays( const char *msgId )
573 {
574 const char *xref;
575 ItemList *refs;
576 const char *ref;
577 int res;
578
579 xref = Db_xref( msgId );
580 if ( xref[ 0 ] == '\0' )
581 return -1;
582
583 res = -1;
584 refs = new_Itl( xref, " :" );
585 for ( ref = Itl_first( refs ); ref != NULL; ref = Itl_next( refs ) )
586 {
587 Str pattern;
588 int days;
589
590 Cfg_beginExpireEnum();
591 while ( ( days = Cfg_nextExpire( pattern ) ) != -1 )
592 if ( Wld_match( ref, pattern )
593 && ( ( days > res && res != 0 ) ||
594 days == 0 ) )
595 {
596 res = days;
597 Log_dbg ( "Custom expiry %d for %s in group %s",
598 days, msgId, ref );
599 break;
600 }
601
602 Itl_next( refs ); /* Throw away group number */
603 }
604
605 if ( res == -1 )
606 res = Cfg_expire();
607 return res;
608 }
609
610 Bool
611 Db_expire( void )
612 {
613 int cntDel, cntLeft, flags, expDays;
614 time_t nowTime, lastAccess;
615 const char *msgId;
616 Str name, tmpName;
617 GDBM_FILE tmpDbf;
618 datum key, val;
619
620 if ( ! Db_open() )
621 return FALSE;
622 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
623 snprintf( tmpName, MAXCHAR, "%s/data/articles.gdbm.new", Cfg_spoolDir() );
624 flags = GDBM_NEWDB | GDBM_FAST;
625 if ( ! ( tmpDbf = gdbm_open( tmpName, 512, flags, 0644, NULL ) ) )
626 {
627 Log_err( "Error opening %s for read/write (%s)", errMsg() );
628 Db_close();
629 return FALSE;
630 }
631 Log_inf( "Expiring articles" );
632 cntDel = 0;
633 cntLeft = 0;
634 nowTime = time( NULL );
635 if ( Db_first( &msgId ) )
636 do
637 {
638 expDays = calcExpireDays( msgId );
639 lastAccess = Db_lastAccess( msgId );
640 if ( expDays == -1 )
641 Log_err( "Internal error: Failed expiry calculation on %s",
642 msgId );
643 else if ( lastAccess == -1 )
644 Log_err( "Internal error: Getting lastAccess of %s failed",
645 msgId );
646 else if ( expDays > 0
647 && difftime( nowTime, lastAccess ) >
648 ( (double) expDays * 24 * 3600 ) )
649 {
650 #ifdef DEBUG
651 Str last, now;
652
653 Utl_cpyStr( last, ctime( &lastAccess ) );
654 last[ strlen( last ) - 1 ] = '\0';
655 Utl_cpyStr( now, ctime( &nowTime ) );
656 last[ strlen( now ) - 1 ] = '\0';
657 Log_dbg( "Expiring %s: last access %s, time now %s",
658 msgId, last, now );
659 #endif
660 ++cntDel;
661 }
662 else
663 {
664 ++cntLeft;
665 key.dptr = (void *)msgId;
666 key.dsize = strlen( msgId ) + 1;
667
668 val = gdbm_fetch( db.dbf, key );
669 if ( val.dptr != NULL )
670 {
671 if ( gdbm_store( tmpDbf, key, val, GDBM_INSERT ) != 0 )
672 Log_err( "Could not store %s in new database (%s)",
673 errMsg() );
674 free( val.dptr );
675 }
676 }
677 }
678 while ( Db_next( &msgId ) );
679 Log_inf( "%lu articles deleted, %lu left", cntDel, cntLeft );
680 gdbm_close( tmpDbf );
681 Db_close();
682 rename( tmpName, name );
683 return TRUE;
684 }