Mercurial > noffle

view src/client.c @ 191:28488e0e3630 noffle

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn] * src/group.h,src/group.c,src/noffle.c,src/server.c: Grp_setLastAccess is only ever called with last param as time(NULL), so remove it and call time() inside the implementation of Grp_setLastAccess. * src/client.c,src/group.h,src/group.c,src/noffle.c,src/post.c: Groups are automatically unsubscribed when the last access to the group is older than a particular threshold. However, for very low traffic groups, the last access may exceed the threshold simply because there has been no new article posted. In this case, rather than unsubscribe, update the group last access time. This means that groups are now only unsubscribed if the last access exceeds the threshold AND articles have arrived in the group since. Add Grp_setLastPostTime() to track the last time an article arrived in the group.
author bears
date Sat, 20 Oct 2001 14:23:46 +0100
parents f1bacee93ca6
children 24d4cd032da5
line wrap: on
line source

/*
  client.c

  $Id: client.c 310 2001-10-20 13:23:46Z bears $
*/

#if HAVE_CONFIG_H
#include <config.h>
#endif

#include "client.h"

#include <stdio.h>
#include <arpa/inet.h>
#include <ctype.h>
#include <netdb.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdarg.h>
#include <sys/socket.h>
#include <unistd.h>
#include "configfile.h"
#include "content.h"
#include "control.h"
#include "dynamicstring.h"
#include "filter.h"
#include "group.h"
#include "itemlist.h"
#include "lock.h"
#include "log.h"
#include "over.h"
#include "protocol.h"
#include "pseudo.h"
#include "request.h"
#include "util.h"
#include "wildmat.h"
#include "portable.h"

/*
  Some newsgroups names are reserved for server-specific or server
  pseudo groups. We don't want to fetch them. For example, INN
  keeps all its control messages in a 'control' hierarchy, and
  used the "to." hierarchy for dark and mysterious purposes I think
  are to do with newsfeeds. The recommended restrictions are documented
  in C.Lindsay, "News Article Format", <draft-ietf-usefor-article-03.txt>.
*/

struct ForbiddenGroupName
{
    const char *pattern;
    Bool match;
} forbiddenGroupNames[] =
{
    { "*.*", FALSE },			/* Single component */
    { "control.*", TRUE },		/* control.* groups */
    { "to.*", TRUE },			/* to.* groups */
    { "*.all", TRUE },			/* 'all' as a component */
    { "*.all.*", TRUE },
    { "all.*", TRUE },
    { "*.ctl", TRUE },			/* 'ctl' as a component */
    { "*.ctl.*", TRUE },
    { "ctl.*", TRUE }
};

struct
{
    FILE* in;     /* Receiving socket from server */
    FILE* out;    /* Sending socket to server */
    Str lastCmd;  /* Last command line */
    Str lastStat; /* Response from server to last command */
    Str grp;      /* Selected group */
    int rmtFirst;  /* First article of current group at server */
    int rmtLast;   /* Last article of current group at server */
    Bool auth;    /* Authetication already done? */
    Str serv;     /* Remote server name */
} client = { NULL, NULL, "", "", "", 1, 0, FALSE, "" };

static void
logBreakDown( void )
{
    Log_err( "Connection to remote server lost "
             "(article numbers could be inconsistent)" );
}

static Bool
getLn( Str line )
{
    Bool r;

    r = Prt_getLn( line, client.in, Cfg_connectTimeout() );
    if ( ! r )
        logBreakDown();
    return r; 
}

static Bool
getTxtLn( Str line, Bool *err )
{
    Bool r;

    r = Prt_getTxtLn( line, err, client.in, Cfg_connectTimeout() );
    if ( *err )
        logBreakDown();
    return r; 
}

static void
putTxtBuf( const char *buf )
{
    Prt_putTxtBuf( buf, client.out );
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
}

static void
putEndOfTxt( void )
{
    Prt_putEndOfTxt( client.out );
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
}

static Bool
putCmdLn( const char *line )
{
    Bool err;
    unsigned int n;

    strcpy( client.lastCmd, line );
    strcpy( client.lastStat, "[no status available]" );
    Log_dbg( LOG_DBG_PROTOCOL, "[S] %s", line );
    n = fprintf( client.out, "%s\r\n", line );
    err = ( n != strlen( line ) + 2 );
    if ( err )
        logBreakDown();;
    return ! err;
}

static Bool
putCmd( const char *fmt, ... )
{
    Str line;
    va_list ap;

    va_start( ap, fmt );
    vsnprintf( line, MAXCHAR, fmt, ap );
    va_end( ap );
    if ( ! putCmdLn( line ) )
        return FALSE;
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
    return TRUE;
}

static Bool
putCmdNoFlush( const char *fmt, ... )
{
    Str line;
    va_list ap;

    va_start( ap, fmt );
    vsnprintf( line, MAXCHAR, fmt, ap );
    va_end( ap );
    return putCmdLn( line );
}

static int getStat( void );

static int
performAuth( void )
{
    int stat;
    Str user, pass;
    
    Cfg_authInfo( client.serv, user, pass );
    if ( strcmp( user, "" ) == 0 )
    {
        Log_err( "No username for authentication set" );
        return STAT_AUTH_REQUIRED;
    }    
    putCmd( "AUTHINFO USER %s", user );
    stat = getStat();
    if ( stat == STAT_AUTH_ACCEPTED
         || stat == STAT_AUTH_ACCEPTED_DEPREC )
        return stat;
    else if ( stat != STAT_MORE_AUTH_REQUIRED_DEPREC
              && stat != STAT_AUTH_REQUIRED )
    {
        Log_err( "Username rejected. Server stat: %s", client.lastStat );
        return stat;
    }    
    if ( strcmp( pass, "" ) == 0 )
    {
        Log_err( "No password for authentication set" );
        return STAT_AUTH_REQUIRED;
    }
    putCmd( "AUTHINFO PASS %s", pass );
    stat = getStat();
    if ( stat != STAT_AUTH_ACCEPTED
         && stat != STAT_AUTH_ACCEPTED_DEPREC)
        Log_err( "Password rejected. Server status: %s", client.lastStat );
    return stat;    
}

static int
getStat( void )
{
    int result;
    Str lastCmd;

    if ( ! getLn( client.lastStat ) )
        result = STAT_CONNECTION_LOST;
    else if ( sscanf( client.lastStat, "%d", &result ) != 1 )
    {
        Log_err( "Invalid server status: %s", client.lastStat );
        result = STAT_PROGRAM_FAULT;
    }
    if ( ( result == STAT_AUTH_REQUIRED
           || result == STAT_AUTH_REQUIRED_DEPREC )
         && ! client.auth )
    {
        client.auth = TRUE;
        strcpy( lastCmd, client.lastCmd );
	result = performAuth();
        if ( result == STAT_AUTH_ACCEPTED
             || result == STAT_AUTH_ACCEPTED_DEPREC )
        {
            putCmd( lastCmd );
            return getStat();
        }
    }
    return result;
}

static void
connectAlarm( int sig )
{
    UNUSED( sig );
    
    return;
}

static Bool
connectWithTimeout( int sock, const struct sockaddr *servAddr,
                    socklen_t addrLen )
{
    sig_t oldHandler;
    int r, to;

    oldHandler = Utl_installSignalHandler( SIGALRM, connectAlarm );
    if ( oldHandler == SIG_ERR )
    {
        Log_err( "client.c:connectWithTimeout: signal failed." );
        return FALSE;
    }
    to = Cfg_connectTimeout();
    if ( alarm( ( unsigned int ) to ) != 0 )
        Log_err( "client.c:connectWithTimeout: Alarm was already set." );
    r = connect( sock, servAddr, addrLen );
    alarm( 0 );
    Utl_installSignalHandler( SIGALRM, oldHandler );
    return ( r >= 0 );
}

static DynStr *
collectTxt( void )
{
    DynStr *res;
    Str line;
    Bool err;

    res = new_DynStr(2048);
    if ( res == NULL )
	return NULL;

    while ( getTxtLn( line, &err ) && ! err )
	DynStr_appLn( res, line );

    if ( err )
    {
	del_DynStr( res );
	return NULL;
    }
    else
	return res;
}

Bool
Client_connect( const char *serv )
{
    unsigned short int port;
    int sock, i;
    unsigned int stat;
    struct hostent *hp;
    char *pStart, *pColon;
    Str host, s;
    Str user, pass;
    struct sockaddr_in sIn;

    ASSERT( client.in == NULL && client.out == NULL );
    client.auth = FALSE;
    Utl_cpyStr( s, serv );
    pStart = Utl_stripWhiteSpace( s );
    pColon = strstr( pStart, ":" );
    if ( pColon == NULL )
    {
        strcpy( host, pStart );
        port = 119;
    }
    else
    {
        *pColon = '\0';
        strcpy( host, pStart );
        if ( sscanf( pColon + 1, "%hi", &port ) != 1 )
        {
            Log_err( "Syntax error in server name: '%s'", serv );
            return FALSE;;
        }
        if ( port <= 0 || port > 65535 )
        {
            Log_err( "Invalid port number %hi. Must be in [1, 65535]", port );
            return FALSE;;
        }
    }
    memset( (void *)&sIn, 0, sizeof( sIn ) );
    hp = gethostbyname( host );
    if ( hp )
    {
        for ( i = 0; (hp->h_addr_list)[ i ]; ++i )
        {
            sIn.sin_family = hp->h_addrtype;
            sIn.sin_port = htons( port );
            sIn.sin_addr = *( (struct in_addr *)hp->h_addr_list[ i ] );
            sock = socket( AF_INET, SOCK_STREAM, 0 );
            if ( sock < 0 )
                break;
            if ( ! connectWithTimeout( sock, (struct sockaddr *)&sIn,
                                       sizeof( sIn ) ) )
            {
                close( sock );
                break;
            }
            if ( ! ( client.out = fdopen( sock, "w" ) )
                 || ! ( client.in  = fdopen( dup( sock ), "r" ) ) )
            {
		if ( client.out != NULL )
		    fclose( client.out );
                close( sock );
		client.in = client.out = NULL;
                break;
            }
            Utl_cpyStr( client.serv, serv );
            stat = getStat();
            switch( stat ) {
            case STAT_READY_POST_ALLOW:
            case STAT_READY_NO_POST_ALLOW: 
                Log_inf( "Connected to %s:%d",
                         inet_ntoa( sIn.sin_addr ), port );
		/* INN needs a MODE READER before it will permit POST. */
		putCmd( "MODE READER" );
		getStat();
                Cfg_authInfo( client.serv, user, pass );
                if ( strcmp( user, "" ) != 0 )
                    performAuth();
                return TRUE;
            default:
                Log_err( "Bad server stat %d", stat ); 
            }
            shutdown( fileno( client.out ), 0 );
	    fclose( client.in );
	    fclose( client.out );
	    close( sock );
	    client.in = client.out = NULL;
        }
    }
    return FALSE;
}

static Bool
isGetGroup( const char *name )
{
    GroupEnum *ge;
    Bool emptyList;
    const char *pattern;

    emptyList = TRUE;
    ge = new_GetGrEn( client.serv );
    while ( ( pattern = GrEn_next( ge ) ) != NULL )
    {
	emptyList = FALSE;
	if ( Wld_match( name, pattern ) )
	{
	    del_GrEn( ge );
	    return TRUE;
	}
    }
    
    del_GrEn( ge );
    return emptyList;
}

static Bool
isOmitGroup( const char *name )
{
    GroupEnum *ge;
    const char *pattern;

    ge = new_OmitGrEn( client.serv );
    while ( ( pattern = GrEn_next( ge ) ) != NULL )
	if ( Wld_match( name, pattern ) )
	{
	    del_GrEn( ge );
	    return TRUE;
	}
    
    del_GrEn( ge );
    return FALSE;   
}

static Bool
isForbiddenGroupName( const char *name )
{
    size_t i;

    for ( i = 0;
	  i < sizeof( forbiddenGroupNames ) /
	      sizeof( struct ForbiddenGroupName );
	  i++ )
    {
	/* Negate result of Wld_match to ensure it is 1 or 0. */
	if ( forbiddenGroupNames[i].match !=
	     ( ! Wld_match( name, forbiddenGroupNames[i].pattern ) ) )
	    return TRUE;
    }

    return FALSE;
}

static void
processGrps( const char *lines, Bool noServerPattern )
{
    char postAllow;
    int first, last;
    Str grp, line, file;
    Bool groupupdate;

    ASSERT( ! Lock_gotLock() );
    if ( ! Lock_openDatabases() )
	return;

    groupupdate = FALSE;
    while ( ( lines = Utl_getLn( line, lines) ) != NULL )
    {
        if ( sscanf( line, "%s %d %d %c",
                     grp, &last, &first, &postAllow ) != 4 )
        {
            Log_err( "Unknown reply to LIST or NEWGROUPS: %s", line );
            continue;
        }
	if ( isForbiddenGroupName( grp ) )
	{
	    Log_inf( "Group %s forbidden", grp );
	    continue;
	}
	if ( noServerPattern && ! isGetGroup( grp ) )
	    continue;
	if ( isOmitGroup( grp ) )
	    continue;
        if ( ! Grp_exists( grp ) )
        {
            Log_inf( "Registering new group '%s'", grp );
            Grp_create( grp );
            /* Start local numbering with remote first number to avoid
               new numbering at the readers if noffle is re-installed */
            if ( first != 0 )
                Grp_setFirstLast( grp, first, first - 1 );
            else
                Grp_setFirstLast( grp, 1, 0 );
            Grp_setServ( grp, client.serv );
	    Grp_setPostAllow( grp, postAllow );
	    groupupdate = TRUE;
        }
        else
        {
            if ( Cfg_servIsPreferential( client.serv, Grp_server( grp ) ) )
            {
                Log_inf( "Changing server for '%s': '%s'->'%s'",
                         grp, Grp_server( grp ), client.serv );
                Grp_setServ( grp, client.serv );
                Grp_setRmtNext( grp, first );
		Grp_setPostAllow( grp, postAllow );
		groupupdate = TRUE;
            }
            else
                Log_dbg( LOG_DBG_FETCH,
			 "Group %s is already fetched from %s",
			 grp, Grp_server( grp ) );            
        }
    }

    snprintf( file, MAXCHAR, "%s/lastupdate.%s",
	      Cfg_spoolDir(), client.serv );
    Utl_stamp( file );
    if ( groupupdate )
    {
	snprintf( file, MAXCHAR, "%s/groupinfo.lastupdate",
		  Cfg_spoolDir() );
	Utl_stamp( file );
    }
    Lock_closeDatabases();
}

void
Client_disconnect( void )
{
    if ( putCmd( "QUIT" ) )
        getStat();
    fclose( client.in );
    fclose( client.out );
    client.in = client.out = NULL;
}

static int
doGetGrps( const char *pattern, Bool *noServerPattern )
{
    Str cmd;
    int stat;
    DynStr *response;

    Utl_cpyStr( cmd, "LIST ACTIVE" );
    if ( pattern[ 0 ] != '\0' )
    {
	Utl_catStr( cmd, " " );
	Utl_catStr( cmd, pattern );
    }

    *noServerPattern = FALSE;
    if ( ! putCmd( cmd ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
        return stat;

    /*
     * Try LIST instead of LIST ACTIVE in case server doesn't
     * support LIST ACTIVE.
     */
    if ( pattern[ 0 ] != '\0' && stat != STAT_GRPS_FOLLOW )
    {
	*noServerPattern = TRUE;
	if ( ! putCmd( "LIST" ) )
	    return STAT_CONNECTION_LOST;
	stat = getStat();
    }    
    if ( stat != STAT_GRPS_FOLLOW )    
    {
	Log_err( "%s failed: %s", cmd, client.lastStat );
	return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    processGrps( DynStr_str( response ), *noServerPattern );
    del_DynStr( response );
    return STAT_OK;
}

int
Client_getGrps( void )
{
    GroupEnum *ge;
    const char *pattern;
    Bool doneOne, noServerPattern;
    int res;

    Log_inf( "Getting groups" );

    doneOne = FALSE;
    res = STAT_OK;
    ge = new_GetGrEn( client.serv );
    while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
    {
	res = doGetGrps( pattern, &noServerPattern );
	doneOne = TRUE;
	if ( noServerPattern )
	    break;
    }

    if ( ! doneOne )
	res = doGetGrps( "", &noServerPattern );

    del_GrEn( ge );
    return res;
}

static int
doGetDsc( const char *pattern, Bool *noServerPattern )
{
    Str name, line, dsc, cmd;
    int stat;
    DynStr *response;
    const char *lines;
    Bool result;

    ASSERT( ! Lock_gotLock() );
    Utl_cpyStr( cmd, "LIST NEWSGROUPS" );
    if ( pattern[ 0 ] != '\0' )
    {
	Utl_catStr( cmd, " " );
	Utl_catStr( cmd, pattern );
    }

    *noServerPattern = FALSE;
    if ( ! putCmd( cmd ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
        return stat;

    /* Try without pattern in case server doesn't support patterns. */
    if ( pattern[ 0 ] != '\0' && stat != STAT_GRPS_FOLLOW )
    {
	*noServerPattern = TRUE;
	if ( !putCmd( "LIST NEWSGROUPS" ) )
	     return STAT_CONNECTION_LOST;
	stat = getStat();
    }
    if ( stat != STAT_GRPS_FOLLOW )
    {
        Log_err( "%s failed: %s", cmd, client.lastStat );
        return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    if ( ! Lock_openDatabases() )
	return STAT_NEWSBASE_FATAL;
    
    lines = DynStr_str( response );
    result = STAT_OK;
    while ( ( lines = Utl_getLn( line, lines) ) != NULL )
    {
        if ( sscanf( line, "%s", name ) != 1 )
        {
            Log_err( "Unknown reply to LIST NEWSGROUPS: %s", line );
	    result = STAT_PROGRAM_FAULT;
	    break;
        }
	if ( *noServerPattern && ! isGetGroup( name ) )
	    continue;
        strcpy( dsc, Utl_restOfLn( line, 1 ) );
        if ( Grp_exists( name ) )
        {
            Log_dbg( LOG_DBG_FETCH, "Description of %s: %s", name, dsc );
            Grp_setDsc( name, dsc );
        }
    }
    Lock_closeDatabases();
    del_DynStr( response );
    return result;
}

int
Client_getDsc( void )
{
    GroupEnum *ge;
    const char *pattern;
    Bool doneOne, noServerPattern;
    int res;

    Log_inf( "Querying group descriptions" );

    doneOne = FALSE;
    res = STAT_OK;
    ge = new_GetGrEn( client.serv );
    while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
    {
	res = doGetDsc( pattern, &noServerPattern );
	doneOne = TRUE;
	if ( noServerPattern )
	    break;
    }

    if ( ! doneOne )
	res = doGetDsc( "", &noServerPattern );

    del_GrEn( ge );
    return res;
}

int
Client_getNewgrps( const time_t *lastTime )
{
    Str s;
    const char *p;
    DynStr *response;
    int stat;

    ASSERT( *lastTime > 0 );
    strftime( s, MAXCHAR, "%Y%m%d %H%M00", gmtime( lastTime ) );
    /*
      Do not use century for working with old server software until 2000.
      According to newest IETF draft, this is still valid after 2000.
      (directly using %y in fmt string causes a Y2K compiler warning)
    */
    p = s + 2;
    if ( ! putCmd( "NEWGROUPS %s GMT", p ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( stat != STAT_NEW_GRP_FOLLOW )
    {
        Log_err( "NEWGROUPS command failed: %s", client.lastStat );
        return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    processGrps( DynStr_str( response ), TRUE );
    del_DynStr( response );
    return STAT_OK;
}

static const char *
readField( Str result, const char *p )
{
    size_t len;
    char *r;

    if ( ! p )
        return NULL;
    r = result;
    *r = '\0';
    len = 0;
    while ( *p != '\t' && *p != '\n' )
    {
        if ( ! *p )
            return p;
        *(r++) = *(p++);
        ++len;
        if ( len >= MAXCHAR - 1 )
        {
            *r = '\0';
            Log_err( "Field in overview too long: %s", r );
            return ++p;
        }
    }
    *r = '\0';
    return ++p;
}

static Bool
parseOvLn( Str line, int *numb, Str subj, Str from,
           Str date, Str msgId, Str ref, size_t *bytes, size_t *lines )
{
    const char *p;
    Str t;
    
    p = readField( t, line );
    if ( sscanf( t, "%d", numb ) != 1 )
        return FALSE;
    p = readField( subj, p );
    p = readField( from, p );
    p = readField( date, p );
    p = readField( msgId, p );
    p = readField( ref, p );
    p = readField( t, p );
    *bytes = 0;
    *lines = 0;
    if ( sscanf( t, "%d", bytes ) != 1 )
        return TRUE;
    p = readField( t, p );
    if ( sscanf( t, "%d", lines ) != 1 )
        return TRUE;
    return TRUE;
}

static const char*
nextXref( const char *pXref, Str grp, int *numb )
{
    Str s;
    const char *pColon, *src;
    char *dst;

    src = pXref;
    while ( *src && isspace( *src ) )
        ++src;
    dst = s;
    while ( *src && ! isspace( *src ) )
        *(dst++) = *(src++);
    *dst = '\0';
    if ( strlen( s ) == 0 )
        return NULL;
    pColon = strstr( s, ":" );
    if ( ! pColon || sscanf( pColon + 1, "%d", numb ) != 1 )
    {
        Log_err( "Corrupt Xref at position '%s'", pXref );
        return NULL;
    }
    Utl_cpyStrN( grp, s, pColon - s );
    Log_dbg( LOG_DBG_FETCH,
	     "client.c: nextXref: grp '%s' numb %lu",
	     grp, numb );
    return src;
}

static Bool
needsMark( const char *ref )
{
    Bool interesting, result;
    const char *msgId;
    int status;
    time_t lastAccess, nowTime;
    double threadFollowTime, maxTime, timeSinceLastAccess;
    ItemList *itl;
    const double secPerDay = 24.0 * 3600.0;

    ASSERT( Lock_gotLock() );
    Log_dbg( LOG_DBG_FETCH, "Checking references '%s' for thread mode", ref );
    result = FALSE;
    itl = new_Itl( ref, " \t" );
    nowTime = time( NULL );
    threadFollowTime = (double)Cfg_threadFollowTime();
    maxTime = threadFollowTime * secPerDay;
    Log_dbg( LOG_DBG_FETCH, "Max time = %.0f", maxTime );
    for ( msgId = Itl_first( itl ); msgId != NULL; msgId = Itl_next( itl ) )
    {
        /*
          References does not have to contain only Message IDs,
          but often it does, so we look up every item in the database.
        */          
        if ( Db_contains( msgId ) )
        {
            status = Db_status( msgId );
            lastAccess = Db_lastAccess( msgId );
            interesting = ( status & DB_INTERESTING );
            timeSinceLastAccess = difftime( nowTime, lastAccess );
            Log_dbg( LOG_DBG_FETCH,
		     "Msg ID '%s': since last access = %.0f, interesting = %s",
                     msgId, timeSinceLastAccess, ( interesting ? "y" : "n" ) );
            if ( interesting && timeSinceLastAccess <= maxTime )
            {
                result = TRUE;
                break;
            }
        }
        else
        {
            Log_dbg( LOG_DBG_FETCH, "MsgID '%s': not in database.", msgId );
        }
    }
    del_Itl( itl );
    Log_dbg( LOG_DBG_FETCH,
	     "Article %s marking for download.",
             ( result ? "needs" : "doesn't need" ) );
    return result;
}

static void
prepareEntry( Over *ov )
{
    Str g, t;
    const char *msgId, *p, *xref;
    int n;

    ASSERT( Lock_gotLock() );
    msgId = Ov_msgId( ov );
    if ( Pseudo_isGeneralInfo( msgId ) )
        Log_dbg( LOG_DBG_FETCH, "Skipping general info '%s'", msgId );
    else if ( Db_contains( msgId ) )
    {
        xref = Db_xref( msgId );
        Log_dbg( LOG_DBG_FETCH,
		 "Entry '%s' already in db with Xref '%s'",
		 msgId, xref );
        p = nextXref( xref, g, &n );
        if ( p == NULL )
            Log_err( "Overview with no group in Xref '%s'", msgId );
        else
        {
            /* TODO: This code block seems unnessesary. Can we remove it? */
            if ( Cfg_servIsPreferential( client.serv, Grp_server( g ) ) )
            {
                Log_dbg( LOG_DBG_FETCH,
			 "Changing first server for '%s' from '%s' to '%s'",
                         msgId, Grp_server( g ), client.serv );
                snprintf( t, MAXCHAR, "%s:%d %s",
                          client.grp, Ov_numb( ov ), xref );
                Db_setXref( msgId, t );
            }
            else
            {
                Log_dbg( LOG_DBG_FETCH,
			 "Adding '%s' to Xref of '%s'", g, msgId );
                snprintf( t, MAXCHAR, "%s %s:%d",
                          xref, client.grp, Ov_numb( ov ) );
                Db_setXref( msgId, t );
            }
        }
    }
    else
    {
        Log_dbg( LOG_DBG_FETCH, "Preparing '%s' in database", msgId );
        Db_prepareEntry( ov, client.grp, Ov_numb( ov ) );
    }
}

int
Client_getOver( const char *grp, int rmtFirst, int rmtLast, FetchMode mode )
{
    size_t nbytes, nlines;
    int rmtNumb, groupsNumb, oldLast, cntMarked;
    Over *ov;
    Str line, subj, from, date, msgId, ref, groups;
    DynStr *response, *newsgroups;
    const char *lines, *groupLines;
    char *p;
    FilterAction action;
    int stat;

    ASSERT( ! Lock_gotLock() );
    ASSERT( strcmp( grp, "" ) != 0 );

    /* Do we need the article Newsgroups: for filtering? */
    if ( Flt_getNewsgroups() )
    {
	if ( ! putCmd( "XHDR Newsgroups %lu-%lu", rmtFirst, rmtLast ) )
	    return STAT_CONNECTION_LOST;
	stat = getStat();
	if ( stat != STAT_HEAD_FOLLOWS )
	{
	    Log_err( "XHDR command failed: %s", client.lastStat );
	    return stat;
	}

	Log_dbg( LOG_DBG_FETCH,
		 "Requesting Newsgroups headers for remote %lu-%lu",
		 rmtFirst, rmtLast );

	newsgroups = collectTxt();
	if ( newsgroups == NULL )
	    return STAT_CONNECTION_LOST;
	
	groupLines = DynStr_str( newsgroups );
    }
    else
    {
	groupLines = NULL;
	newsgroups = NULL;
    }
    
    if ( ! putCmd( "XOVER %lu-%lu", rmtFirst, rmtLast ) )
    {
	del_DynStr( newsgroups );
        return STAT_CONNECTION_LOST;
    }
    
    stat = getStat();
    if ( stat != STAT_OVERS_FOLLOW )
    {
	del_DynStr( newsgroups );
        Log_err( "XOVER command failed: %s", client.lastStat );
        return stat;
    }
    Log_dbg( LOG_DBG_FETCH,
	     "Requesting overview for remote %lu-%lu",
	     rmtFirst, rmtLast );

    response = collectTxt();
    if ( response == NULL )
    {
	del_DynStr( newsgroups );
	return STAT_CONNECTION_LOST;
    }

    if ( ! Lock_openDatabases() )
    {
	del_DynStr( newsgroups );
	del_DynStr( response );
	return STAT_NEWSBASE_FATAL;
    }
    
    Cont_read( grp );
    oldLast = Cont_last();
    cntMarked = 0;
    lines = DynStr_str( response );
    while ( ( lines = Utl_getLn( line, lines ) ) != NULL )
    {
        if ( ! parseOvLn( line, &rmtNumb, subj, from, date, msgId, ref,
                          &nbytes, &nlines ) )
            Log_err( "Bad overview line: %s", line );
	else if ( Cont_find( msgId ) >= 0 )
	    Log_inf( "Already have '%s'", msgId );
        else
        {
            ov = new_Over( subj, from, date, msgId, ref, nbytes, nlines );
	    groupsNumb = 0;
	    p = NULL;
	    if ( groupLines != NULL )
	    {
		do
		{
		    groupLines = Utl_getLn( groups, groupLines );
		    groupsNumb = strtoul( groups, &p, 10 );
		} while ( groupLines != NULL
			  && p > groups
			  && groupsNumb < rmtNumb );
		if ( groupsNumb != rmtNumb )
		    p = NULL;
	    }

	    action = Flt_checkFilters( grp, p, ov, mode );
	    if ( action == FILTER_DISCARD )
		continue;
            Cont_app( ov );
            prepareEntry( ov );
            if ( action == FILTER_FULL
		 || ( action == FILTER_THREAD && needsMark( ref ) ) )
            {
                Req_add( client.serv, msgId );
                ++cntMarked;
            }
        }
        Grp_setRmtNext( client.grp, rmtNumb + 1 );
    }
    if ( oldLast != Cont_last() )
    {
        Log_inf( "Added %s %lu-%lu", client.grp, oldLast + 1, Cont_last() );
	Log_inf( "%u articles marked for download in %s",
		 cntMarked, client.grp  );
	Cont_write();
	Grp_setFirstLast( grp, Cont_first(), Cont_last() );
	Grp_setLastPostTime( grp );
    }
    Lock_closeDatabases();
    del_DynStr( response );
    del_DynStr( newsgroups );
    return STAT_OK;
}

static void
retrievingFailed( const char* msgId, const char *reason )
{
    int status;

    ASSERT( ! Lock_gotLock() );
    Log_err( "Retrieving of %s failed: %s", msgId, reason );
    if ( ! Lock_openDatabases() )
	return;
    status = Db_status( msgId );
    Pseudo_retrievingFailed( msgId, reason );
    Db_setStatus( msgId, status | DB_RETRIEVING_FAILED );
    Lock_closeDatabases();
    return;
}

static int
retrieveAndStoreArt( const char *msgId, int artcnt, int artmax )
{
    Bool err;
    DynStr *s = NULL;

    ASSERT( ! Lock_gotLock() );
    Log_inf( "[%d/%d] Retrieving %s", artcnt, artmax, msgId );
    err = TRUE;

    s = collectTxt();
    if ( s != NULL )
    {
	const char *txt;
	
	txt = DynStr_str( s );
	if ( ! Lock_openDatabases() )
	{
	    del_DynStr( s );
	    retrievingFailed( msgId, "Can't open message base" );
	    return STAT_NEWSBASE_FATAL;
	}
	
        err = ! Db_storeArt( msgId, txt );
	if ( ! err )
	{
	    Str supersedeIds;

	    if ( Prt_searchHeader( txt, "Supersedes", supersedeIds ) )
	    {
		ItemList *ids;
		const char *supersededMsgId;

		ids = new_Itl( supersedeIds, " \n\t" );
		for ( supersededMsgId = Itl_first( ids );
		      supersededMsgId != NULL;
		      supersededMsgId = Itl_next( ids ) )
		    Ctrl_cancel( supersededMsgId );
		del_Itl( ids );
	    }
	}
	Lock_closeDatabases();
	del_DynStr( s );
    }
    else
    {
        retrievingFailed( msgId, "Connection broke down" );
	return STAT_CONNECTION_LOST;
    }
    return err ? STAT_NEWSBASE_FATAL : STAT_OK;
}

int
Client_retrieveArt( const char *msgId )
{
    int res;
    
    ASSERT( Lock_gotLock() );
    if ( ! Db_contains( msgId ) )
    {
        Log_err( "Article '%s' not prepared in database. Skipping.", msgId );
        return STAT_PROGRAM_FAULT;
    }
    if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
    {
        Log_inf( "Article '%s' already retrieved. Skipping.", msgId );
        return STAT_OK;
    }

    Lock_closeDatabases();
    if ( ! putCmd( "ARTICLE %s", msgId ) )
    {
        retrievingFailed( msgId, "Connection broke down" );
	res = STAT_CONNECTION_LOST;
    }
    else if ( ( res = getStat() ) != STAT_ART_FOLLOWS )
        retrievingFailed( msgId, client.lastStat );
    else
        res = retrieveAndStoreArt( msgId, 0, 0 );
    if ( ! Lock_openDatabases() )
	res = STAT_NEWSBASE_FATAL;
    return res;
}

int
Client_retrieveArtList( const char *list, int *artcnt, int artmax )
{
    Str msgId;
    DynStr *s;
    const char *p;
    int res, msgStat;
    
    ASSERT( Lock_gotLock() );
    Log_inf( "Retrieving article list" );
    s = new_DynStr( (int)strlen( list ) );
    p = list;
    res = STAT_OK;
    while ( ( p = Utl_getLn( msgId, p ) ) )
        if ( ! Db_contains( msgId ) )
	{
            Log_err( "[%d/%d] Skipping retrieving of %s "
		     "(not prepared in database)",
                     ++(*artcnt), artmax, msgId );
	    res = STAT_PROGRAM_FAULT;
	}
        else if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
            Log_inf( "[%d/%d] Skipping %s (already retrieved)",
		     ++(*artcnt), artmax, msgId );
        else if ( ! putCmdNoFlush( "ARTICLE %s", msgId ) )
        {
            retrievingFailed( msgId, "Connection broke down" );
            del_DynStr( s );
            return STAT_CONNECTION_LOST;
        }
        else
            DynStr_appLn( s, msgId );

    Lock_closeDatabases();
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
    
    /*
     * We got something. Try to process all messages and return the
     * 'worst' error encountered (note we may have already hit a
     * STAT_PROGRAM_FAULT).
     */
    p = DynStr_str( s );
    while ( ! IS_FATAL( res ) && ( p = Utl_getLn( msgId, p ) ) )
    {
	msgStat = getStat();
	if ( msgStat == STAT_ART_FOLLOWS )
	    msgStat = retrieveAndStoreArt( msgId, ++(*artcnt), artmax );
	else
            retrievingFailed( msgId, client.lastStat );
	    
	if ( res == STAT_OK || ( ! IS_FATAL( res ) && IS_FATAL( msgStat ) ) )
	    res = msgStat;
    }
    del_DynStr( s );
    if ( ! Lock_openDatabases() && ! IS_FATAL( res ) )
	res = STAT_NEWSBASE_FATAL;
    return res;
}

int
Client_changeToGrp( const char* name )
{
    unsigned int stat;
    int estimatedNumb, first, last, res;

    ASSERT( Lock_gotLock() );
    if ( ! Grp_exists( name ) )
        return STAT_NEWSBASE_FATAL;
    Lock_closeDatabases();
    stat = STAT_OK;
    if ( ! putCmd( "GROUP %s", name ) )
	res = STAT_CONNECTION_LOST;
    if ( stat == STAT_OK )
	stat = getStat();
    if ( ! Lock_openDatabases() )
	return STAT_NEWSBASE_FATAL;
    if ( stat != STAT_GRP_SELECTED )
	return stat;
    if ( sscanf( client.lastStat, "%u %d %d %d",
                 &stat, &estimatedNumb, &first, &last ) != 4 )
    {
        Log_err( "Bad server response to GROUP: %s", client.lastStat );
        return STAT_PROGRAM_FAULT;
    }
    Utl_cpyStr( client.grp, name );
    client.rmtFirst = first;
    client.rmtLast = last;
    return STAT_OK;
}

void
Client_rmtFirstLast( int *first, int *last )
{
    ASSERT( Lock_gotLock() );
    *first = client.rmtFirst;
    *last = client.rmtLast;
}

int
Client_postArt( const char *msgId, const char *artTxt, Str errStr )
{
    int stat;
    
    errStr[0] = '\0';
    
    if ( ! putCmd( "POST" ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
	return stat;
    else if ( stat != STAT_SEND_ART )
    {
        Log_err( "Posting of %s not allowed: %s", msgId, client.lastStat );
        strcpy( errStr, client.lastStat );
        return stat;
    }
    putTxtBuf( artTxt );
    putEndOfTxt();
    stat = getStat();
    if ( IS_FATAL( stat ) )
	return stat;
    else if ( stat != STAT_POST_OK )
    {
        Log_err( "Posting of %s failed: %s", msgId, client.lastStat );
        strcpy( errStr, client.lastStat );
        return stat;
    }
    Log_inf( "Posted %s (Status: %s)", msgId, client.lastStat );
    return STAT_OK;
}