/*
  client.c

  $Id: client.c 316 2001-10-31 11:44:53Z bears $
*/

#if HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <arpa/inet.h>
#include <ctype.h>
#include <netdb.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdarg.h>
#include <sys/socket.h>
#include <unistd.h>
#include "client.h"
#include "configfile.h"
#include "content.h"
#include "control.h"
#include "dynamicstring.h"
#include "filter.h"
#include "group.h"
#include "itemlist.h"
#include "lock.h"
#include "log.h"
#include "over.h"
#include "protocol.h"
#include "pseudo.h"
#include "request.h"
#include "util.h"
#include "wildmat.h"
#include "portable.h"

/*
  Some newsgroups names are reserved for server-specific or server
  pseudo groups. We don't want to fetch them. For example, INN
  keeps all its control messages in a 'control' hierarchy, and
  used the "to." hierarchy for dark and mysterious purposes I think
  are to do with newsfeeds. The recommended restrictions are documented
  in C.Lindsay, "News Article Format", <draft-ietf-usefor-article-03.txt>.
*/

struct ForbiddenGroupName
{
    const char *pattern;
    Bool match;
} forbiddenGroupNames[] =
{
    { "*.*", FALSE },			/* Single component */
    { "control.*", TRUE },		/* control.* groups */
    { "to.*", TRUE },			/* to.* groups */
    { "*.all", TRUE },			/* 'all' as a component */
    { "*.all.*", TRUE },
    { "all.*", TRUE },
    { "*.ctl", TRUE },			/* 'ctl' as a component */
    { "*.ctl.*", TRUE },
    { "ctl.*", TRUE }
};

struct
{
    FILE* in;       /* Receiving socket from server */
    FILE* out;      /* Sending socket to server */
    Str lastCmd;    /* Last command line */
    Str lastStat;   /* Response from server to last command */
    Str grp;        /* Selected group */
    int rmtFirst;   /* First article of current group at server */
    int rmtLast;    /* Last article of current group at server */
    Bool auth;      /* Authentication already done? */
    Str serv;       /* Remote server name */
} client = { NULL, NULL, "", "", "", 1, 0, FALSE, "" };

static void
logBreakDown( void )
{
    Log_err( "Connection to remote server lost "
             "(article numbers could be inconsistent)" );
}

static Bool
getLn( Str line )
{
    Bool r;

    r = Prt_getLn( line, client.in, Cfg_connectTimeout() );
    if ( ! r )
        logBreakDown();
    return r; 
}

static Bool
getTxtLn( Str line, Bool *err )
{
    Bool r;

    r = Prt_getTxtLn( line, err, client.in, Cfg_connectTimeout() );
    if ( *err )
        logBreakDown();
    return r; 
}

static void
putTxtBuf( const char *buf )
{
    Prt_putTxtBuf( buf, client.out );
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
}

static void
putEndOfTxt( void )
{
    Prt_putEndOfTxt( client.out );
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
}

static Bool
putCmdLn( const char *line )
{
    Bool err;
    unsigned int n;

    strcpy( client.lastCmd, line );
    strcpy( client.lastStat, "[no status available]" );
    Log_dbg( LOG_DBG_PROTOCOL, "[S] %s", line );
    n = fprintf( client.out, "%s\r\n", line );
    err = ( n != strlen( line ) + 2 );
    if ( err )
        logBreakDown();;
    return ! err;
}

static Bool
putCmd( const char *fmt, ... )
{
    Str line;
    va_list ap;

    va_start( ap, fmt );
    vsnprintf( line, MAXCHAR, fmt, ap );
    va_end( ap );
    if ( ! putCmdLn( line ) )
        return FALSE;
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
    return TRUE;
}

static Bool
putCmdNoFlush( const char *fmt, ... )
{
    Str line;
    va_list ap;

    va_start( ap, fmt );
    vsnprintf( line, MAXCHAR, fmt, ap );
    va_end( ap );
    return putCmdLn( line );
}

static int getStat( void );

static int
performAuth( void )
{
    int stat;
    Str user, pass;
    
    Cfg_authInfo( client.serv, user, pass );
    if ( strcmp( user, "" ) == 0 )
    {
        Log_err( "No username for authentication set" );
        return STAT_AUTH_REQUIRED;
    }    
    putCmd( "AUTHINFO USER %s", user );
    stat = getStat();
    if ( stat == STAT_AUTH_ACCEPTED
         || stat == STAT_AUTH_ACCEPTED_DEPREC )
        return stat;
    else if ( stat != STAT_MORE_AUTH_REQUIRED_DEPREC
              && stat != STAT_AUTH_REQUIRED )
    {
        Log_err( "Username rejected. Server stat: %s", client.lastStat );
        return stat;
    }    
    if ( strcmp( pass, "" ) == 0 )
    {
        Log_err( "No password for authentication set" );
        return STAT_AUTH_REQUIRED;
    }
    putCmd( "AUTHINFO PASS %s", pass );
    stat = getStat();
    if ( stat != STAT_AUTH_ACCEPTED
         && stat != STAT_AUTH_ACCEPTED_DEPREC)
        Log_err( "Password rejected. Server status: %s", client.lastStat );
    return stat;    
}

static int
getStat( void )
{
    int result;
    Str lastCmd;

    if ( ! getLn( client.lastStat ) )
        result = STAT_CONNECTION_LOST;
    else if ( sscanf( client.lastStat, "%d", &result ) != 1 )
    {
        Log_err( "Invalid server status: %s", client.lastStat );
        result = STAT_PROGRAM_FAULT;
    }
    if ( ( result == STAT_AUTH_REQUIRED
           || result == STAT_AUTH_REQUIRED_DEPREC )
         && ! client.auth )
    {
        client.auth = TRUE;
        strcpy( lastCmd, client.lastCmd );
	result = performAuth();
        if ( result == STAT_AUTH_ACCEPTED
             || result == STAT_AUTH_ACCEPTED_DEPREC )
        {
            putCmd( lastCmd );
            return getStat();
        }
    }
    return result;
}

static void
connectAlarm( int sig )
{
    UNUSED( sig );
    
    return;
}

static Bool
connectWithTimeout( int sock, const struct sockaddr *servAddr,
                    socklen_t addrLen )
{
    sig_t oldHandler;
    int r, to;

    oldHandler = Utl_installSignalHandler( SIGALRM, connectAlarm );
    if ( oldHandler == SIG_ERR )
    {
        Log_err( "client.c:connectWithTimeout: signal failed." );
        return FALSE;
    }
    to = Cfg_connectTimeout();
    if ( alarm( ( unsigned int ) to ) != 0 )
        Log_err( "client.c:connectWithTimeout: Alarm was already set." );
    r = connect( sock, servAddr, addrLen );
    alarm( 0 );
    Utl_installSignalHandler( SIGALRM, oldHandler );
    return ( r >= 0 );
}

static DynStr *
collectTxt( void )
{
    DynStr *res;
    Str line;
    Bool err;

    res = new_DynStr(2048);
    if ( res == NULL )
	return NULL;

    while ( getTxtLn( line, &err ) && ! err )
	DynStr_appLn( res, line );

    if ( err )
    {
	del_DynStr( res );
	return NULL;
    }
    else
	return res;
}

Bool
Client_connect( const char *serv )
{
    unsigned short int port;
    int sock, i;
    unsigned int stat;
    struct hostent *hp;
    char *pStart, *pColon;
    Str host, s;
    Str user, pass;
    struct sockaddr_in sIn;

    ASSERT( client.in == NULL && client.out == NULL );
    client.auth = FALSE;
    Utl_cpyStr( s, serv );
    pStart = Utl_stripWhiteSpace( s );
    pColon = strstr( pStart, ":" );
    if ( pColon == NULL )
    {
        strcpy( host, pStart );
        port = 119;
    }
    else
    {
        *pColon = '\0';
        strcpy( host, pStart );
        if ( sscanf( pColon + 1, "%hi", &port ) != 1 )
        {
            Log_err( "Syntax error in server name: '%s'", serv );
            return FALSE;;
        }
        if ( port <= 0 || port > 65535 )
        {
            Log_err( "Invalid port number %hi. Must be in [1, 65535]", port );
            return FALSE;;
        }
    }
    memset( (void *)&sIn, 0, sizeof( sIn ) );
    hp = gethostbyname( host );
    if ( hp )
    {
        for ( i = 0; (hp->h_addr_list)[ i ]; ++i )
        {
            sIn.sin_family = hp->h_addrtype;
            sIn.sin_port = htons( port );
            sIn.sin_addr = *( (struct in_addr *)hp->h_addr_list[ i ] );
            sock = socket( AF_INET, SOCK_STREAM, 0 );
            if ( sock < 0 )
                break;
            if ( ! connectWithTimeout( sock, (struct sockaddr *)&sIn,
                                       sizeof( sIn ) ) )
            {
                close( sock );
                break;
            }
            if ( ! ( client.out = fdopen( sock, "w" ) )
                 || ! ( client.in  = fdopen( dup( sock ), "r" ) ) )
            {
		if ( client.out != NULL )
		    fclose( client.out );
                close( sock );
		client.in = client.out = NULL;
                break;
            }
            Utl_cpyStr( client.serv, serv );
            stat = getStat();
            switch( stat )
	    {
            case STAT_READY_POST_ALLOW:
            case STAT_READY_NO_POST_ALLOW: 
                Log_inf( "Connected to %s:%d",
                         inet_ntoa( sIn.sin_addr ), port );
		/* INN needs a MODE READER before it will permit POST. */
		putCmd( "MODE READER" );
		getStat();
                Cfg_authInfo( client.serv, user, pass );
                if ( strcmp( user, "" ) != 0 )
                    performAuth();
                return TRUE;
            default:
                Log_err( "Bad server stat %d", stat ); 
            }
            shutdown( fileno( client.out ), 0 );
	    fclose( client.in );
	    fclose( client.out );
	    close( sock );
	    client.in = client.out = NULL;
        }
    }
    return FALSE;
}

static Bool
isGetGroup( const char *name )
{
    GroupEnum *ge;
    Bool emptyList;
    const char *pattern;

    emptyList = TRUE;
    ge = new_GetGrEn( client.serv );
    while ( ( pattern = GrEn_next( ge ) ) != NULL )
    {
	emptyList = FALSE;
	if ( Wld_match( name, pattern ) )
	{
	    del_GrEn( ge );
	    return TRUE;
	}
    }
    
    del_GrEn( ge );
    return emptyList;
}

static Bool
isOmitGroup( const char *name )
{
    GroupEnum *ge;
    const char *pattern;

    ge = new_OmitGrEn( client.serv );
    while ( ( pattern = GrEn_next( ge ) ) != NULL )
	if ( Wld_match( name, pattern ) )
	{
	    del_GrEn( ge );
	    return TRUE;
	}
    
    del_GrEn( ge );
    return FALSE;   
}

static Bool
isForbiddenGroupName( const char *name )
{
    size_t i;

    for ( i = 0;
	  i < sizeof( forbiddenGroupNames ) /
	      sizeof( struct ForbiddenGroupName );
	  i++ )
    {
	/* Negate result of Wld_match to ensure it is 1 or 0. */
	if ( forbiddenGroupNames[i].match !=
	     ( ! Wld_match( name, forbiddenGroupNames[i].pattern ) ) )
	    return TRUE;
    }

    return FALSE;
}

static void
processGrps( const char *lines, Bool noServerPattern )
{
    char postAllow;
    int first, last;
    Str grp, line, file;
    Bool groupupdate;

    ASSERT( ! Lock_gotLock() );
    if ( ! Lock_openDatabases() )
	return;

    groupupdate = FALSE;
    while ( ( lines = Utl_getLn( line, lines) ) != NULL )
    {
        if ( sscanf( line, "%s %d %d %c",
                     grp, &last, &first, &postAllow ) != 4 )
        {
            Log_err( "Unknown reply to LIST or NEWGROUPS: %s", line );
            continue;
        }
	if ( isForbiddenGroupName( grp ) )
	{
	    Log_inf( "Group %s forbidden", grp );
	    continue;
	}
	if ( noServerPattern && ! isGetGroup( grp ) )
	    continue;
	if ( isOmitGroup( grp ) )
	    continue;
        if ( ! Grp_exists( grp ) )
        {
            Log_inf( "Registering new group '%s'", grp );
            Grp_create( grp );
            /* Start local numbering with remote first number to avoid
               new numbering at the readers if noffle is re-installed */
            if ( first != 0 )
                Grp_setFirstLast( grp, first, first - 1 );
            else
                Grp_setFirstLast( grp, 1, 0 );
            Grp_setServ( grp, client.serv );
	    Grp_setPostAllow( grp, postAllow );
	    groupupdate = TRUE;
        }
        else
        {
            if ( Cfg_servIsPreferential( client.serv, Grp_server( grp ) ) )
            {
                Log_inf( "Changing server for '%s': '%s'->'%s'",
                         grp, Grp_server( grp ), client.serv );
                Grp_setServ( grp, client.serv );
                Grp_setRmtNext( grp, first );
		Grp_setPostAllow( grp, postAllow );
		groupupdate = TRUE;
            }
            else
                Log_dbg( LOG_DBG_FETCH,
			 "Group %s is already fetched from %s",
			 grp, Grp_server( grp ) );            
        }
    }

    snprintf( file, MAXCHAR, "%s/lastupdate.%s",
	      Cfg_spoolDir(), client.serv );
    Utl_stamp( file );
    if ( groupupdate )
    {
	snprintf( file, MAXCHAR, "%s/groupinfo.lastupdate",
		  Cfg_spoolDir() );
	Utl_stamp( file );
    }
    Lock_closeDatabases();
}

void
Client_disconnect( void )
{
    if ( putCmd( "QUIT" ) )
        getStat();
    fclose( client.in );
    fclose( client.out );
    client.in = client.out = NULL;
}

static int
doGetGrps( const char *pattern, Bool *noServerPattern )
{
    Str cmd;
    int stat;
    DynStr *response;

    Utl_cpyStr( cmd, "LIST ACTIVE" );
    if ( pattern[ 0 ] != '\0' )
    {
	Utl_catStr( cmd, " " );
	Utl_catStr( cmd, pattern );
    }

    *noServerPattern = FALSE;
    if ( ! putCmd( cmd ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
        return stat;

    /*
     * Try LIST instead of LIST ACTIVE in case server doesn't
     * support LIST ACTIVE.
     */
    if ( pattern[ 0 ] != '\0' && stat != STAT_GRPS_FOLLOW )
    {
	*noServerPattern = TRUE;
	if ( ! putCmd( "LIST" ) )
	    return STAT_CONNECTION_LOST;
	stat = getStat();
    }    
    if ( stat != STAT_GRPS_FOLLOW )    
    {
	Log_err( "%s failed: %s", cmd, client.lastStat );
	return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    processGrps( DynStr_str( response ), *noServerPattern );
    del_DynStr( response );
    return STAT_OK;
}

int
Client_getGrps( void )
{
    GroupEnum *ge;
    const char *pattern;
    Bool doneOne, noServerPattern;
    int res;

    Log_inf( "Getting groups" );

    doneOne = FALSE;
    res = STAT_OK;
    ge = new_GetGrEn( client.serv );
    while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
    {
	res = doGetGrps( pattern, &noServerPattern );
	doneOne = TRUE;
	if ( noServerPattern )
	    break;
    }

    if ( ! doneOne )
	res = doGetGrps( "", &noServerPattern );

    del_GrEn( ge );
    return res;
}

static int
doGetDsc( const char *pattern, Bool *noServerPattern )
{
    Str name, line, dsc, cmd;
    int stat;
    DynStr *response;
    const char *lines;
    Bool result;

    ASSERT( ! Lock_gotLock() );
    Utl_cpyStr( cmd, "LIST NEWSGROUPS" );
    if ( pattern[ 0 ] != '\0' )
    {
	Utl_catStr( cmd, " " );
	Utl_catStr( cmd, pattern );
    }

    *noServerPattern = FALSE;
    if ( ! putCmd( cmd ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
        return stat;

    /* Try without pattern in case server doesn't support patterns. */
    if ( pattern[ 0 ] != '\0' && stat != STAT_GRPS_FOLLOW )
    {
	*noServerPattern = TRUE;
	if ( !putCmd( "LIST NEWSGROUPS" ) )
	     return STAT_CONNECTION_LOST;
	stat = getStat();
    }
    if ( stat != STAT_GRPS_FOLLOW )
    {
        Log_err( "%s failed: %s", cmd, client.lastStat );
        return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    if ( ! Lock_openDatabases() )
	return STAT_NEWSBASE_FATAL;
    
    lines = DynStr_str( response );
    result = STAT_OK;
    while ( ( lines = Utl_getLn( line, lines) ) != NULL )
    {
        if ( sscanf( line, "%s", name ) != 1 )
        {
            Log_err( "Unknown reply to LIST NEWSGROUPS: %s", line );
	    result = STAT_PROGRAM_FAULT;
	    break;
        }
	if ( *noServerPattern && ! isGetGroup( name ) )
	    continue;
        strcpy( dsc, Utl_restOfLn( line, 1 ) );
        if ( Grp_exists( name ) )
        {
            Log_dbg( LOG_DBG_FETCH, "Description of %s: %s", name, dsc );
            Grp_setDsc( name, dsc );
        }
    }
    Lock_closeDatabases();
    del_DynStr( response );
    return result;
}

int
Client_getDsc( void )
{
    GroupEnum *ge;
    const char *pattern;
    Bool doneOne, noServerPattern;
    int res;

    Log_inf( "Querying group descriptions" );

    doneOne = FALSE;
    res = STAT_OK;
    ge = new_GetGrEn( client.serv );
    while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
    {
	res = doGetDsc( pattern, &noServerPattern );
	doneOne = TRUE;
	if ( noServerPattern )
	    break;
    }

    if ( ! doneOne )
	res = doGetDsc( "", &noServerPattern );

    del_GrEn( ge );
    return res;
}

int
Client_getNewgrps( const time_t *lastTime )
{
    Str s;
    const char *p;
    DynStr *response;
    int stat;

    ASSERT( *lastTime > 0 );
    strftime( s, MAXCHAR, "%Y%m%d %H%M00", gmtime( lastTime ) );
    /*
      Do not use century for working with old server software until 2000.
      According to newest IETF draft, this is still valid after 2000.
      (directly using %y in fmt string causes a Y2K compiler warning)
    */
    p = s + 2;
    if ( ! putCmd( "NEWGROUPS %s GMT", p ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( stat != STAT_NEW_GRP_FOLLOW )
    {
        Log_err( "NEWGROUPS command failed: %s", client.lastStat );
        return stat;
    }

    response = collectTxt();
    if ( response == NULL )
	return STAT_CONNECTION_LOST;
    
    processGrps( DynStr_str( response ), TRUE );
    del_DynStr( response );
    return STAT_OK;
}

static const char *
readField( Str result, const char *p )
{
    size_t len;
    char *r;

    if ( ! p )
        return NULL;
    r = result;
    *r = '\0';
    len = 0;
    while ( *p != '\t' && *p != '\n' )
    {
        if ( ! *p )
            return p;
        *(r++) = *(p++);
        ++len;
        if ( len >= MAXCHAR - 1 )
        {
            *r = '\0';
            Log_err( "Field in overview too long: %s", r );
            return ++p;
        }
    }
    *r = '\0';
    return ++p;
}

static Bool
parseOvLn( Str line, int *numb, Str subj, Str from,
           Str date, Str msgId, Str ref, size_t *bytes, size_t *lines )
{
    const char *p;
    Str t;
    
    p = readField( t, line );
    if ( sscanf( t, "%d", numb ) != 1 )
        return FALSE;
    p = readField( subj, p );
    p = readField( from, p );
    p = readField( date, p );
    p = readField( msgId, p );
    p = readField( ref, p );
    p = readField( t, p );
    *bytes = 0;
    *lines = 0;
    if ( sscanf( t, "%d", bytes ) != 1 )
        return TRUE;
    p = readField( t, p );
    if ( sscanf( t, "%d", lines ) != 1 )
        return TRUE;
    return TRUE;
}

static const char*
nextXref( const char *pXref, Str grp, int *numb )
{
    Str s;
    const char *pColon, *src;
    char *dst;

    src = pXref;
    while ( *src && isspace( *src ) )
        ++src;
    dst = s;
    while ( *src && ! isspace( *src ) )
        *(dst++) = *(src++);
    *dst = '\0';
    if ( strlen( s ) == 0 )
        return NULL;
    pColon = strstr( s, ":" );
    if ( ! pColon || sscanf( pColon + 1, "%d", numb ) != 1 )
    {
        Log_err( "Corrupt Xref at position '%s'", pXref );
        return NULL;
    }
    Utl_cpyStrN( grp, s, pColon - s );
    Log_dbg( LOG_DBG_FETCH,
	     "client.c: nextXref: grp '%s' numb %lu",
	     grp, numb );
    return src;
}

static Bool
needsMark( const char *ref )
{
    Bool interesting, result;
    const char *msgId;
    int status;
    time_t lastAccess, nowTime;
    double threadFollowTime, maxTime, timeSinceLastAccess;
    ItemList *itl;
    const double secPerDay = 24.0 * 3600.0;

    ASSERT( Lock_gotLock() );
    Log_dbg( LOG_DBG_FETCH, "Checking references '%s' for thread mode", ref );
    result = FALSE;
    itl = new_Itl( ref, " \t" );
    nowTime = time( NULL );
    threadFollowTime = (double)Cfg_threadFollowTime();
    maxTime = threadFollowTime * secPerDay;
    Log_dbg( LOG_DBG_FETCH, "Max time = %.0f", maxTime );
    for ( msgId = Itl_first( itl ); msgId != NULL; msgId = Itl_next( itl ) )
    {
        /*
          References does not have to contain only Message IDs,
          but often it does, so we look up every item in the database.
        */          
        if ( Db_contains( msgId ) )
        {
            status = Db_status( msgId );
            lastAccess = Db_lastAccess( msgId );
            interesting = ( status & DB_INTERESTING );
            timeSinceLastAccess = difftime( nowTime, lastAccess );
            Log_dbg( LOG_DBG_FETCH,
		     "Msg ID '%s': since last access = %.0f, interesting = %s",
                     msgId, timeSinceLastAccess, ( interesting ? "y" : "n" ) );
            if ( interesting && timeSinceLastAccess <= maxTime )
            {
                result = TRUE;
                break;
            }
        }
        else
        {
            Log_dbg( LOG_DBG_FETCH, "MsgID '%s': not in database.", msgId );
        }
    }
    del_Itl( itl );
    Log_dbg( LOG_DBG_FETCH,
	     "Article %s marking for download.",
             ( result ? "needs" : "doesn't need" ) );
    return result;
}

static void
prepareEntry( Over *ov )
{
    Str g, t;
    const char *msgId, *p, *xref;
    int n;

    ASSERT( Lock_gotLock() );
    msgId = Ov_msgId( ov );
    if ( Pseudo_isGeneralInfo( msgId ) )
        Log_dbg( LOG_DBG_FETCH, "Skipping general info '%s'", msgId );
    else if ( Db_contains( msgId ) )
    {
        xref = Db_xref( msgId );
        Log_dbg( LOG_DBG_FETCH,
		 "Entry '%s' already in db with Xref '%s'",
		 msgId, xref );
        p = nextXref( xref, g, &n );
        if ( p == NULL )
            Log_err( "Overview with no group in Xref '%s'", msgId );
        else
        {
            /* TODO: This code block seems unnessesary. Can we remove it? */
            if ( Cfg_servIsPreferential( client.serv, Grp_server( g ) ) )
            {
                Log_dbg( LOG_DBG_FETCH,
			 "Changing first server for '%s' from '%s' to '%s'",
                         msgId, Grp_server( g ), client.serv );
                snprintf( t, MAXCHAR, "%s:%d %s",
                          client.grp, Ov_numb( ov ), xref );
                Db_setXref( msgId, t );
            }
            else
            {
                Log_dbg( LOG_DBG_FETCH,
			 "Adding '%s' to Xref of '%s'", g, msgId );
                snprintf( t, MAXCHAR, "%s %s:%d",
                          xref, client.grp, Ov_numb( ov ) );
                Db_setXref( msgId, t );
            }
        }
    }
    else
    {
        Log_dbg( LOG_DBG_FETCH, "Preparing '%s' in database", msgId );
        Db_prepareEntry( ov, client.grp, Ov_numb( ov ) );
    }
}

int
Client_getOver( const char *grp, int rmtFirst, int rmtLast, FetchMode mode )
{
    size_t nbytes, nlines;
    int rmtNumb, groupsNumb, oldLast, cntMarked;
    Over *ov;
    Str line, subj, from, date, msgId, ref, groups;
    DynStr *response, *newsgroups;
    const char *lines, *groupLines;
    char *p;
    FilterAction action;
    int stat;

    ASSERT( ! Lock_gotLock() );
    ASSERT( strcmp( grp, "" ) != 0 );

    /* Do we need the article Newsgroups: for filtering? */
    if ( Flt_getNewsgroups() )
    {
	if ( ! putCmd( "XHDR Newsgroups %lu-%lu", rmtFirst, rmtLast ) )
	    return STAT_CONNECTION_LOST;
	stat = getStat();
	if ( stat != STAT_HEAD_FOLLOWS )
	{
	    Log_err( "XHDR command failed: %s", client.lastStat );
	    return stat;
	}

	Log_dbg( LOG_DBG_FETCH,
		 "Requesting Newsgroups headers for remote %lu-%lu",
		 rmtFirst, rmtLast );

	newsgroups = collectTxt();
	if ( newsgroups == NULL )
	    return STAT_CONNECTION_LOST;
	
	groupLines = DynStr_str( newsgroups );
    }
    else
    {
	groupLines = NULL;
	newsgroups = NULL;
    }
    
    if ( ! putCmd( "XOVER %lu-%lu", rmtFirst, rmtLast ) )
    {
	del_DynStr( newsgroups );
        return STAT_CONNECTION_LOST;
    }
    
    stat = getStat();
    if ( stat != STAT_OVERS_FOLLOW )
    {
	del_DynStr( newsgroups );
        Log_err( "XOVER command failed: %s", client.lastStat );
        return stat;
    }
    Log_dbg( LOG_DBG_FETCH,
	     "Requesting overview for remote %lu-%lu",
	     rmtFirst, rmtLast );

    response = collectTxt();
    if ( response == NULL )
    {
	del_DynStr( newsgroups );
	return STAT_CONNECTION_LOST;
    }

    if ( ! Lock_openDatabases() )
    {
	del_DynStr( newsgroups );
	del_DynStr( response );
	return STAT_NEWSBASE_FATAL;
    }
    
    Cont_read( grp );
    oldLast = Cont_last();
    cntMarked = 0;
    lines = DynStr_str( response );
    while ( ( lines = Utl_getLn( line, lines ) ) != NULL )
    {
        if ( ! parseOvLn( line, &rmtNumb, subj, from, date, msgId, ref,
                          &nbytes, &nlines ) )
            Log_err( "Bad overview line: %s", line );
	else if ( Cont_find( msgId ) >= 0 )
	    Log_inf( "Already have '%s'", msgId );
        else
        {
            ov = new_Over( subj, from, date, msgId, ref, nbytes, nlines );
	    groupsNumb = 0;
	    p = NULL;
	    if ( groupLines != NULL )
	    {
		do
		{
		    groupLines = Utl_getLn( groups, groupLines );
		    groupsNumb = strtoul( groups, &p, 10 );
		} while ( groupLines != NULL
			  && p > groups
			  && groupsNumb < rmtNumb );
		if ( groupsNumb != rmtNumb )
		    p = NULL;
	    }

	    action = Flt_checkFilters( grp, p, ov, mode );
	    if ( action == FILTER_DISCARD )
		continue;
            Cont_app( ov );
            prepareEntry( ov );
            if ( action == FILTER_FULL
		 || ( action == FILTER_THREAD && needsMark( ref ) ) )
            {
                Req_add( client.serv, msgId );
                ++cntMarked;
            }
        }
        Grp_setRmtNext( client.grp, rmtNumb + 1 );
    }
    if ( oldLast != Cont_last() )
    {
        Log_inf( "Added %s %lu-%lu", client.grp, oldLast + 1, Cont_last() );
	Log_inf( "%u articles marked for download in %s",
		 cntMarked, client.grp  );
	Cont_write();
	Grp_setFirstLast( grp, Cont_first(), Cont_last() );
	Grp_setLastPostTime( grp );
    }
    Lock_closeDatabases();
    del_DynStr( response );
    del_DynStr( newsgroups );
    return STAT_OK;
}

static void
retrievingFailed( const char* msgId, const char *reason )
{
    int status;

    ASSERT( ! Lock_gotLock() );
    Log_err( "Retrieving of %s failed: %s", msgId, reason );
    if ( ! Lock_openDatabases() )
	return;
    status = Db_status( msgId );
    Pseudo_retrievingFailed( msgId, reason );
    Db_setStatus( msgId, status | DB_RETRIEVING_FAILED );
    Lock_closeDatabases();
    return;
}

static int
retrieveAndStoreArt( const char *msgId, int artcnt, int artmax )
{
    Bool err;
    DynStr *s = NULL;

    ASSERT( ! Lock_gotLock() );
    Log_inf( "[%d/%d] Retrieving %s", artcnt, artmax, msgId );
    err = TRUE;

    s = collectTxt();
    if ( s != NULL )
    {
	const char *txt;
	
	txt = DynStr_str( s );
	if ( ! Lock_openDatabases() )
	{
	    del_DynStr( s );
	    retrievingFailed( msgId, "Can't open message base" );
	    return STAT_NEWSBASE_FATAL;
	}
	
        err = ! Db_storeArt( msgId, txt );
	if ( ! err )
	{
	    Str supersedeIds;

	    if ( Prt_searchHeader( txt, "Supersedes", supersedeIds ) )
	    {
		ItemList *ids;
		const char *supersededMsgId;

		ids = new_Itl( supersedeIds, " \n\t" );
		for ( supersededMsgId = Itl_first( ids );
		      supersededMsgId != NULL;
		      supersededMsgId = Itl_next( ids ) )
		    Ctrl_cancel( supersededMsgId );
		del_Itl( ids );
	    }
	}
	Lock_closeDatabases();
	del_DynStr( s );
    }
    else
    {
        retrievingFailed( msgId, "Connection broke down" );
	return STAT_CONNECTION_LOST;
    }
    return err ? STAT_NEWSBASE_FATAL : STAT_OK;
}

int
Client_retrieveArt( const char *msgId )
{
    int res;
    
    ASSERT( Lock_gotLock() );
    if ( ! Db_contains( msgId ) )
    {
        Log_err( "Article '%s' not prepared in database. Skipping.", msgId );
        return STAT_PROGRAM_FAULT;
    }
    if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
    {
        Log_inf( "Article '%s' already retrieved. Skipping.", msgId );
        return STAT_OK;
    }

    Lock_closeDatabases();
    if ( ! putCmd( "ARTICLE %s", msgId ) )
    {
        retrievingFailed( msgId, "Connection broke down" );
	res = STAT_CONNECTION_LOST;
    }
    else if ( ( res = getStat() ) != STAT_ART_FOLLOWS )
        retrievingFailed( msgId, client.lastStat );
    else
        res = retrieveAndStoreArt( msgId, 0, 0 );
    if ( ! Lock_openDatabases() )
	res = STAT_NEWSBASE_FATAL;
    return res;
}

int
Client_retrieveArtList( const char *list, int *artcnt, int artmax )
{
    Str msgId;
    DynStr *s;
    const char *p;
    int res, msgStat;
    
    ASSERT( Lock_gotLock() );
    Log_inf( "Retrieving article list" );
    s = new_DynStr( (int)strlen( list ) );
    p = list;
    res = STAT_OK;
    while ( ( p = Utl_getLn( msgId, p ) ) )
        if ( ! Db_contains( msgId ) )
	{
            Log_err( "[%d/%d] Skipping retrieving of %s "
		     "(not prepared in database)",
                     ++(*artcnt), artmax, msgId );
	    res = STAT_PROGRAM_FAULT;
	}
        else if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
            Log_inf( "[%d/%d] Skipping %s (already retrieved)",
		     ++(*artcnt), artmax, msgId );
        else if ( ! putCmdNoFlush( "ARTICLE %s", msgId ) )
        {
            retrievingFailed( msgId, "Connection broke down" );
            del_DynStr( s );
            return STAT_CONNECTION_LOST;
        }
        else
            DynStr_appLn( s, msgId );

    Lock_closeDatabases();
    fflush( client.out );
    Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
    
    /*
     * We got something. Try to process all messages and return the
     * 'worst' error encountered (note we may have already hit a
     * STAT_PROGRAM_FAULT).
     */
    p = DynStr_str( s );
    while ( ! IS_FATAL( res ) && ( p = Utl_getLn( msgId, p ) ) )
    {
	msgStat = getStat();
	if ( msgStat == STAT_ART_FOLLOWS )
	    msgStat = retrieveAndStoreArt( msgId, ++(*artcnt), artmax );
	else
            retrievingFailed( msgId, client.lastStat );
	    
	if ( res == STAT_OK || ( ! IS_FATAL( res ) && IS_FATAL( msgStat ) ) )
	    res = msgStat;
    }
    del_DynStr( s );
    if ( ! Lock_openDatabases() && ! IS_FATAL( res ) )
	res = STAT_NEWSBASE_FATAL;
    return res;
}

int
Client_changeToGrp( const char* name )
{
    unsigned int stat;
    int estimatedNumb, first, last, res;

    ASSERT( Lock_gotLock() );
    if ( ! Grp_exists( name ) )
        return STAT_NEWSBASE_FATAL;
    Lock_closeDatabases();
    stat = STAT_OK;
    if ( ! putCmd( "GROUP %s", name ) )
	res = STAT_CONNECTION_LOST;
    if ( stat == STAT_OK )
	stat = getStat();
    if ( ! Lock_openDatabases() )
	return STAT_NEWSBASE_FATAL;
    if ( stat != STAT_GRP_SELECTED )
	return stat;
    if ( sscanf( client.lastStat, "%u %d %d %d",
                 &stat, &estimatedNumb, &first, &last ) != 4 )
    {
        Log_err( "Bad server response to GROUP: %s", client.lastStat );
        return STAT_PROGRAM_FAULT;
    }
    Utl_cpyStr( client.grp, name );
    client.rmtFirst = first;
    client.rmtLast = last;
    return STAT_OK;
}

void
Client_rmtFirstLast( int *first, int *last )
{
    ASSERT( Lock_gotLock() );
    *first = client.rmtFirst;
    *last = client.rmtLast;
}

int
Client_postArt( const char *msgId, const char *artTxt, Str errStr )
{
    int stat;
    
    errStr[0] = '\0';
    
    if ( ! putCmd( "POST" ) )
        return STAT_CONNECTION_LOST;
    stat = getStat();
    if ( IS_FATAL( stat ) )
	return stat;
    else if ( stat != STAT_SEND_ART )
    {
        Log_err( "Posting of %s not allowed: %s", msgId, client.lastStat );
        strcpy( errStr, client.lastStat );
        return stat;
    }
    putTxtBuf( artTxt );
    putEndOfTxt();
    stat = getStat();
    if ( IS_FATAL( stat ) )
	return stat;
    else if ( stat != STAT_POST_OK )
    {
        Log_err( "Posting of %s failed: %s", msgId, client.lastStat );
        strcpy( errStr, client.lastStat );
        return stat;
    }
    Log_inf( "Posted %s (Status: %s)", msgId, client.lastStat );
    return STAT_OK;
}