Mercurial > noffle

comparison database.c @ 0:04124a4423d4 noffle

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn] Initial revision
author enz
date Tue, 04 Jan 2000 11:35:42 +0000
parents
children 526a4c34ee2e
comparison
equal deleted inserted replaced
-1:000000000000 0:04124a4423d4
1 /*
2 database.c
3
4 $Id: database.c 3 2000-01-04 11:35:42Z enz $
5
6 Uses GNU gdbm library. Using Berkeley db (included in libc6) was
7 cumbersome. It is based on Berkeley db 1.85, which has severe bugs
8 (e.g. it is not recommended to delete or overwrite entries with
9 overflow pages).
10 */
11
12 #include "database.h"
13 #include <ctype.h>
14 #include <errno.h>
15 #include <fcntl.h>
16 #include <gdbm.h>
17 #include <unistd.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include "config.h"
21 #include "log.h"
22 #include "protocol.h"
23 #include "util.h"
24
25 static struct Db
26 {
27 GDBM_FILE dbf;
28
29 /* Start string for Xref header line: "Xref: <host>" */
30 Str xrefHost;
31
32 /* Msg Id of presently loaded article, empty if none loaded */
33 Str msgId;
34
35 /* Status of loaded article */
36 int stat; /* Flags */
37 time_t lastAccess;
38
39 /* Overview of loaded article */
40 Str subj;
41 Str from;
42 Str date;
43 Str ref;
44 Str xref;
45 size_t bytes;
46 size_t lines;
47
48 /* Article text (except for overview header lines) */
49 DynStr *txt;
50
51 } db = { NULL, "(unknown)", "", 0, 0, "", "", "", "", "", 0, 0, NULL };
52
53 static const char *
54 errMsg( void )
55 {
56 if ( errno != 0 )
57 return strerror( errno );
58 return gdbm_strerror( gdbm_errno );
59 }
60
61 Bool
62 Db_open( void )
63 {
64 Str name, host;
65 int flags;
66
67 ASSERT( db.dbf == NULL );
68 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
69 flags = GDBM_WRCREAT | GDBM_FAST;
70
71 if ( ! ( db.dbf = gdbm_open( name, 512, flags, 0644, NULL ) ) )
72 {
73 Log_err( "Error opening %s for r/w (%s)", name, errMsg() );
74 return FALSE;
75 }
76 Log_dbg( "%s opened for r/w", name );
77
78 if ( db.txt == NULL )
79 db.txt = new_DynStr( 5000 );
80
81 gethostname( host, MAXCHAR );
82 snprintf( db.xrefHost, MAXCHAR, "Xref: %s", host );
83
84 return TRUE;
85 }
86
87 void
88 Db_close( void )
89 {
90 ASSERT( db.dbf );
91 Log_dbg( "Closing database" );
92 gdbm_close( db.dbf );
93 db.dbf = NULL;
94 del_DynStr( db.txt );
95 db.txt = NULL;
96 Utl_cpyStr( db.msgId, "" );
97 }
98
99 static Bool
100 loadArt( const char *msgId )
101 {
102 static void *dptr = NULL;
103
104 datum key, val;
105 Str t = "";
106 const char *p;
107
108 ASSERT( db.dbf );
109
110 if ( strcmp( msgId, db.msgId ) == 0 )
111 return TRUE;
112
113 key.dptr = (void *)msgId;
114 key.dsize = strlen( msgId ) + 1;
115 if ( dptr != NULL )
116 {
117 free( dptr );
118 dptr = NULL;
119 }
120 val = gdbm_fetch( db.dbf, key );
121 dptr = val.dptr;
122 if ( dptr == NULL )
123 {
124 Log_dbg( "database.c loadArt: gdbm_fetch found no entry" );
125 return FALSE;
126 }
127
128 Utl_cpyStr( db.msgId, msgId );
129 p = Utl_getLn( t, (char *)dptr );
130 if ( ! p || sscanf( t, "%x", &db.stat ) != 1 )
131 {
132 Log_err( "Entry in database '%s' is corrupt (status)", msgId );
133 return FALSE;
134 }
135 p = Utl_getLn( t, p );
136 if ( ! p || sscanf( t, "%lu", &db.lastAccess ) != 1 )
137 {
138 Log_err( "Entry in database '%s' is corrupt (lastAccess)", msgId );
139 return FALSE;
140 }
141 p = Utl_getLn( db.subj, p );
142 p = Utl_getLn( db.from, p );
143 p = Utl_getLn( db.date, p );
144 p = Utl_getLn( db.ref, p );
145 p = Utl_getLn( db.xref, p );
146 if ( ! p )
147 {
148 Log_err( "Entry in database '%s' is corrupt (overview)", msgId );
149 return FALSE;
150 }
151 p = Utl_getLn( t, p );
152 if ( ! p || sscanf( t, "%u", &db.bytes ) != 1 )
153 {
154 Log_err( "Entry in database '%s' is corrupt (bytes)", msgId );
155 return FALSE;
156 }
157 p = Utl_getLn( t, p );
158 if ( ! p || sscanf( t, "%u", &db.lines ) != 1 )
159 {
160 Log_err( "Entry in database '%s' is corrupt (lines)", msgId );
161 return FALSE;
162 }
163 DynStr_clear( db.txt );
164 DynStr_app( db.txt, p );
165 return TRUE;
166 }
167
168 static Bool
169 saveArt( void )
170 {
171 DynStr *s;
172 Str t = "";
173 datum key, val;
174
175 if ( strcmp( db.msgId, "" ) == 0 )
176 return FALSE;
177 s = new_DynStr( 5000 );
178 snprintf( t, MAXCHAR, "%x", db.stat );
179 DynStr_appLn( s, t );
180 snprintf( t, MAXCHAR, "%lu", db.lastAccess );
181 DynStr_appLn( s, t );
182 DynStr_appLn( s, db.subj );
183 DynStr_appLn( s, db.from );
184 DynStr_appLn( s, db.date );
185 DynStr_appLn( s, db.ref );
186 DynStr_appLn( s, db.xref );
187 snprintf( t, MAXCHAR, "%u", db.bytes );
188 DynStr_appLn( s, t );
189 snprintf( t, MAXCHAR, "%u", db.lines );
190 DynStr_appLn( s, t );
191 DynStr_appDynStr( s, db.txt );
192
193 key.dptr = (void *)db.msgId;
194 key.dsize = strlen( db.msgId ) + 1;
195 val.dptr = (void *)DynStr_str( s );
196 val.dsize = DynStr_len( s ) + 1;
197 if ( gdbm_store( db.dbf, key, val, GDBM_REPLACE ) != 0 )
198 {
199 Log_err( "Could not store %s in database (%s)", errMsg() );
200 return FALSE;
201 }
202
203 del_DynStr( s );
204 return TRUE;
205 }
206
207 Bool
208 Db_prepareEntry( const Over *ov, const char *grp, int numb )
209 {
210 const char *msgId;
211
212 ASSERT( db.dbf );
213 ASSERT( ov );
214 ASSERT( grp );
215
216 msgId = Ov_msgId( ov );
217 Log_dbg( "Preparing entry %s", msgId );
218 if ( Db_contains( msgId ) )
219 Log_err( "Preparing article twice: %s", msgId );
220
221 db.stat = DB_NOT_DOWNLOADED;
222 db.lastAccess = time( NULL );
223
224 Utl_cpyStr( db.msgId, msgId );
225 Utl_cpyStr( db.subj, Ov_subj( ov ) );
226 Utl_cpyStr( db.from, Ov_from( ov ) );
227 Utl_cpyStr( db.date, Ov_date( ov ) );
228 Utl_cpyStr( db.ref, Ov_ref( ov ) );
229 snprintf( db.xref, MAXCHAR, "%s:%i", grp, numb );
230 db.bytes = Ov_bytes( ov );
231 db.lines = Ov_lines( ov );
232
233 DynStr_clear( db.txt );
234
235 return saveArt();
236 }
237
238 Bool
239 Db_storeArt( const char *msgId, const char *artTxt )
240 {
241 Str line, lineEx, field, value;
242 const char *startPos;
243
244 ASSERT( db.dbf );
245
246 Log_dbg( "Store article %s", msgId );
247 if ( ! loadArt( msgId ) )
248 {
249 Log_err( "Cannot find info about '%s' in database", msgId );
250 return FALSE;
251 }
252 if ( ! ( db.stat & DB_NOT_DOWNLOADED ) )
253 {
254 Log_err( "Trying to store alrady retrieved article '%s'", msgId );
255 return FALSE;
256 }
257 db.stat &= ~DB_NOT_DOWNLOADED;
258 db.stat &= ~DB_RETRIEVING_FAILED;
259 db.lastAccess = time( NULL );
260
261 DynStr_clear( db.txt );
262
263 /* Read header */
264 startPos = artTxt;
265 while ( TRUE )
266 {
267 artTxt = Utl_getLn( lineEx, artTxt );
268 if ( lineEx[ 0 ] == '\0' )
269 {
270 DynStr_appLn( db.txt, lineEx );
271 break;
272 }
273 /* Get other lines if field is split over multiple lines */
274 while ( ( artTxt = Utl_getLn( line, artTxt ) ) )
275 if ( isspace( line[ 0 ] ) )
276 {
277 strncat( lineEx, "\n", MAXCHAR );
278 strncat( lineEx, line, MAXCHAR );
279 }
280 else
281 {
282 artTxt = Utl_ungetLn( startPos, artTxt );
283 break;
284 }
285 /* Remove fields already in overview and handle x-noffle
286 headers correctly in case of cascading NOFFLEs */
287 if ( Prt_getField( field, value, lineEx ) )
288 {
289 if ( strcmp( field, "x-noffle-status" ) == 0 )
290 {
291 if ( strstr( value, "NOT_DOWNLOADED" ) != 0 )
292 db.stat |= DB_NOT_DOWNLOADED;
293 }
294 else if ( strcmp( field, "message-id" ) != 0
295 && strcmp( field, "xref" ) != 0
296 && strcmp( field, "references" ) != 0
297 && strcmp( field, "subject" ) != 0
298 && strcmp( field, "from" ) != 0
299 && strcmp( field, "date" ) != 0
300 && strcmp( field, "bytes" ) != 0
301 && strcmp( field, "lines" ) != 0
302 && strcmp( field, "x-noffle-lastaccess" ) != 0 )
303 DynStr_appLn( db.txt, lineEx );
304 }
305 }
306
307 /* Read body */
308 while ( ( artTxt = Utl_getLn( line, artTxt ) ) )
309 if ( ! ( db.stat & DB_NOT_DOWNLOADED ) )
310 DynStr_appLn( db.txt, line );
311
312 return saveArt();
313 }
314
315 void
316 Db_setStat( const char *msgId, int stat )
317 {
318 if ( loadArt( msgId ) )
319 {
320 db.stat = stat;
321 saveArt();
322 }
323 }
324
325 void
326 Db_updateLastAccess( const char *msgId )
327 {
328 if ( loadArt( msgId ) )
329 {
330 db.lastAccess = time( NULL );
331 saveArt();
332 }
333 }
334
335 void
336 Db_setXref( const char *msgId, const char *xref )
337 {
338 if ( loadArt( msgId ) )
339 {
340 Utl_cpyStr( db.xref, xref );
341 saveArt();
342 }
343 }
344
345 /* Search best position for breaking a line */
346 static const char *
347 searchBreakPos( const char *line, int wantedLength )
348 {
349 const char *lastSpace = NULL;
350 Bool lastWasSpace = FALSE;
351 int len = 0;
352
353 while ( *line != '\0' )
354 {
355 if ( isspace( *line ) )
356 {
357 if ( len > wantedLength && lastSpace != NULL )
358 return lastSpace;
359 if ( ! lastWasSpace )
360 lastSpace = line;
361 lastWasSpace = TRUE;
362 }
363 else
364 lastWasSpace = FALSE;
365 ++len;
366 ++line;
367 }
368 if ( len > wantedLength && lastSpace != NULL )
369 return lastSpace;
370 return line;
371 }
372
373 /* Append header line by breaking long line into multiple lines */
374 static void
375 appendLongHeader( DynStr *target, const char *field, const char *value )
376 {
377 const int wantedLength = 78;
378 const char *breakPos, *old;
379 int len;
380
381 len = strlen( field );
382 DynStr_appN( target, field, len );
383 DynStr_appN( target, " ", 1 );
384 old = value;
385 while ( isspace( *old ) )
386 ++old;
387 breakPos = searchBreakPos( old, wantedLength - len - 1 );
388 DynStr_appN( target, old, breakPos - old );
389 if ( *breakPos == '\0' )
390 {
391 DynStr_appN( target, "\n", 1 );
392 return;
393 }
394 DynStr_appN( target, "\n ", 2 );
395 while ( TRUE )
396 {
397 old = breakPos;
398 while ( isspace( *old ) )
399 ++old;
400 breakPos = searchBreakPos( old, wantedLength - 1 );
401 DynStr_appN( target, old, breakPos - old );
402 if ( *breakPos == '\0' )
403 {
404 DynStr_appN( target, "\n", 1 );
405 return;
406 }
407 DynStr_appN( target, "\n ", 2 );
408 }
409 }
410
411 const char *
412 Db_header( const char *msgId )
413 {
414 static DynStr *s = NULL;
415
416 Str date, t;
417 int stat;
418 const char *p;
419
420 if ( s == NULL )
421 s = new_DynStr( 5000 );
422 else
423 DynStr_clear( s );
424 ASSERT( db.dbf );
425 if ( ! loadArt( msgId ) )
426 return NULL;
427 strftime( date, MAXCHAR, "%Y-%m-%d %H:%M:%S",
428 localtime( &db.lastAccess ) );
429 stat = db.stat;
430 snprintf( t, MAXCHAR,
431 "Message-ID: %s\n"
432 "X-NOFFLE-Status:%s%s%s\n"
433 "X-NOFFLE-LastAccess: %s\n",
434 msgId,
435 stat & DB_INTERESTING ? " INTERESTING" : "",
436 stat & DB_NOT_DOWNLOADED ? " NOT_DOWNLOADED" : "",
437 stat & DB_RETRIEVING_FAILED ? " RETRIEVING_FAILED" : "",
438 date );
439 DynStr_app( s, t );
440 appendLongHeader( s, "Subject:", db.subj );
441 appendLongHeader( s, "From:", db.from );
442 appendLongHeader( s, "Date:", db.date );
443 appendLongHeader( s, "References:", db.ref );
444 DynStr_app( s, "Bytes: " );
445 snprintf( t, MAXCHAR, "%u", db.bytes );
446 DynStr_appLn( s, t );
447 DynStr_app( s, "Lines: " );
448 snprintf( t, MAXCHAR, "%u", db.lines );
449 DynStr_appLn( s, t );
450 appendLongHeader( s, db.xrefHost, db.xref );
451 p = strstr( DynStr_str( db.txt ), "\n\n" );
452 if ( ! p )
453 DynStr_appDynStr( s, db.txt );
454 else
455 DynStr_appN( s, DynStr_str( db.txt ), p - DynStr_str( db.txt ) + 1 );
456 return DynStr_str( s );
457 }
458
459 const char *
460 Db_body( const char *msgId )
461 {
462 const char *p;
463
464 if ( ! loadArt( msgId ) )
465 return "";
466 p = strstr( DynStr_str( db.txt ), "\n\n" );
467 if ( ! p )
468 return "";
469 return ( p + 2 );
470 }
471
472 int
473 Db_stat( const char *msgId )
474 {
475 if ( ! loadArt( msgId ) )
476 return 0;
477 return db.stat;
478 }
479
480 time_t
481 Db_lastAccess( const char *msgId )
482 {
483 if ( ! loadArt( msgId ) )
484 return -1;
485 return db.lastAccess;
486 }
487
488 const char *
489 Db_ref( const char *msgId )
490 {
491 if ( ! loadArt( msgId ) )
492 return "";
493 return db.ref;
494 }
495
496 const char *
497 Db_xref( const char *msgId )
498 {
499 if ( ! loadArt( msgId ) )
500 return "";
501 return db.xref;
502 }
503
504 Bool
505 Db_contains( const char *msgId )
506 {
507 datum key;
508
509 ASSERT( db.dbf );
510 if ( strcmp( msgId, db.msgId ) == 0 )
511 return TRUE;
512 key.dptr = (void*)msgId;
513 key.dsize = strlen( msgId ) + 1;
514 return gdbm_exists( db.dbf, key );
515 }
516
517 static datum cursor = { NULL, 0 };
518
519 Bool
520 Db_first( const char** msgId )
521 {
522 ASSERT( db.dbf );
523 if ( cursor.dptr != NULL )
524 {
525 free( cursor.dptr );
526 cursor.dptr = NULL;
527 }
528 cursor = gdbm_firstkey( db.dbf );
529 *msgId = cursor.dptr;
530 return ( cursor.dptr != NULL );
531 }
532
533 Bool
534 Db_next( const char** msgId )
535 {
536 void *oldDptr = cursor.dptr;
537
538 ASSERT( db.dbf );
539 if ( cursor.dptr == NULL )
540 return FALSE;
541 cursor = gdbm_nextkey( db.dbf, cursor );
542 free( oldDptr );
543 *msgId = cursor.dptr;
544 return ( cursor.dptr != NULL );
545 }
546
547 Bool
548 Db_expire( unsigned int days )
549 {
550 double limit;
551 int cntDel, cntLeft, flags;
552 time_t nowTime, lastAccess;
553 const char *msgId;
554 Str name, tmpName;
555 GDBM_FILE tmpDbf;
556 datum key, val;
557
558 if ( ! Db_open() )
559 return FALSE;
560 snprintf( name, MAXCHAR, "%s/data/articles.gdbm", Cfg_spoolDir() );
561 snprintf( tmpName, MAXCHAR, "%s/data/articles.gdbm.new", Cfg_spoolDir() );
562 flags = GDBM_NEWDB | GDBM_FAST;
563 if ( ! ( tmpDbf = gdbm_open( tmpName, 512, flags, 0644, NULL ) ) )
564 {
565 Log_err( "Error opening %s for read/write (%s)", errMsg() );
566 Db_close();
567 return FALSE;
568 }
569 Log_inf( "Expiring articles that have not been accessed for %u days",
570 days );
571 limit = days * 24. * 3600.;
572 cntDel = 0;
573 cntLeft = 0;
574 nowTime = time( NULL );
575 if ( Db_first( &msgId ) )
576 do
577 {
578 lastAccess = Db_lastAccess( msgId );
579 if ( lastAccess == -1 )
580 Log_err( "Internal error: Getting lastAccess of %s failed",
581 msgId );
582 else if ( difftime( nowTime, lastAccess ) > limit )
583 {
584 Log_dbg( "Expiring %s", msgId );
585 ++cntDel;
586 }
587 else
588 {
589 ++cntLeft;
590 key.dptr = (void *)msgId;
591 key.dsize = strlen( msgId ) + 1;
592
593 val = gdbm_fetch( db.dbf, key );
594 if ( val.dptr != NULL )
595 {
596 if ( gdbm_store( tmpDbf, key, val, GDBM_INSERT ) != 0 )
597 Log_err( "Could not store %s in new database (%s)",
598 errMsg() );
599 free( val.dptr );
600 }
601 }
602 }
603 while ( Db_next( &msgId ) );
604 Log_inf( "%lu articles deleted, %lu left", cntDel, cntLeft );
605 gdbm_close( tmpDbf );
606 Db_close();
607 rename( tmpName, name );
608 return TRUE;
609 }