---

codex_server.cc

/*
 * Copyright 2003 Michael A. Marsh, Cornell University. All rights reserved.
 * This software is released under the modified BSD license.
 * See the file LICENSE in the top-level directory for details.
 */
//
// $Id: codex_server.cc,v 1.5 2004/05/19 15:56:46 mmarsh Exp $
//
// $Log: codex_server.cc,v $
// Revision 1.5  2004/05/19 15:56:46  mmarsh
// *** empty log message ***
//
// Revision 1.4  2003/11/06 21:45:35  mmarsh
// Removed ppk parameter, fixed a typo, and added in a couple of missing
// variables.
//
// Revision 1.3  2003/11/06 18:11:51  mmarsh
// Cleaned up doxygen page label and title.
//
// Revision 1.2  2003/11/04 22:07:35  mmarsh
// General code cleanup and reorganization.
//
//

#include <unistd.h>
#include <signal.h>
#include <openssl/ssl.h>
#include <openssl/rand.h>

// global state and general information
#include "CODEX_Server/ServerState.h"
#include "CODEX_KeyService/StateInfo.h"
#include "CODEX_APSS/StateInfo.h"
#include "CODEX_Events/DelayedEvent.h"

// quorum management
#include "CODEX_Server/InternalServer.h"
#include "CODEX_Server/QuorumBuilderAct.h"
#include "CODEX_Server/ServerMessageDispatcher.h"
#include "CODEX_Server/ServerQuorumDispatcher.h"
#include "CODEX_Server/SignRequestHandler.h"
#include "CODEX_Server/ServerResponseHandler.h"
#include "CODEX_Server/BroadcastRequestHandler.h"
#include "CODEX_Server/UnicastRequestHandler.h"

#include "CODEX_Server/ShareLabelChallengeHandler.h"
#include "CODEX_Server/ShareLabelChallengeEvent.h"

// client management
#include "CODEX_KeyService/ClientServer.h"
#include "CODEX_KeyService/ClientActivity.h"
#include "CODEX_KeyService/ClientMessageEvent.h"
#include "CODEX_KeyService/ClientMessageVerifier.h"

// client delegation
#include "CODEX_KeyService/DelegateRequestDistributor.h"
#include "CODEX_KeyService/ClientMessageParser.h"
#include "CODEX_KeyService/ClientResponseParser.h"
#include "CODEX_KeyService/RoutedClientMessageVerifier.h"
#include "CODEX_KeyService/DelegateResponseVerifier.h"
#include "CODEX_KeyService/ClientMessageSigner.h"
#include "CODEX_KeyService/ClientResponseSigner.h"

// threshold cryptography

// APSS
#include "CODEX_APSS/MessageDeliverer.h"
#include "CODEX_APSS/MessageProcessor.h"
#include "CODEX_APSS/MessageVerifier.h"
#include "CODEX_APSS/MessageParser.h"
#include "CODEX_APSS/TriggerActivity.h"

// exceptions
#include "CODEX_Ciphers/CipherExceptions.h"
#include "CODEX_Server/ServerExceptions.h"
#include "CODEX_Server/ConfigurationExceptions.h"

// This is used for signal handling.
bool running;

// These are used for simulating a slow processor.
bool SlowServer = false;
long int ServerDelayS = 0;
long int ServerDelayUS = 0;
long int ServerSlowdown = 0;

// These are used for simulating a slow link.
bool SlowLink = false;
long int LinkDelayS = 0;
long int LinkDelayUS = 0;
long int CODEX_Events::DelayedEvent::sDelay;
long int CODEX_Events::DelayedEvent::usDelay;

void handle_interrupt(int signal)
{
   running = false;
}

void handle_crash(int signal)
{
   cerr << "Fatal error, signal " << signal << endl;
   ::exit(1);
}

void
configure_server(int argc, char** argv)
{
   int arg = 0;
   string config_file;
   string config_section;
   string usage_string = "Usage: codex_server -c <config_file> [-s <section>]";
   while ( -1 != arg )
   {
      arg = getopt(argc,argv,"c:s:");
      switch(arg)
      {
         case 'c' :
            config_file = optarg;
            break;
         case 's' :
            config_section = optarg;
            break;
         case ':' :
         case '?' :
            cerr << usage_string << endl;
            ::exit(1);
      }
   }
   if ( 0 == config_file.size() )
   {
      cerr << usage_string << endl;
      ::exit(1);
   }

   // Configure the server.
   CONF* conf = 0;
   try
   {
      conf = NCONF_new(NCONF_default());
      if ( 0 == NCONF_load(conf,config_file.c_str(),0) )
      {
         throw CODEX_Exceptions::FileCannotOpenException( __FILE__ ,
                                                          __LINE__ ,
                                                          config_file );
      }
      const char* sec = config_section.c_str();
      CODEX_Server::ServerState::instance()->configure(
         conf, sec, config_file );
      CODEX_KeyService::StateInfo::instance()->configure(
         conf, sec, config_file );
      CODEX_APSS::StateInfo::instance()->configure(
         conf, sec, config_file );

      // This is testing configuration, and shouldn't be used in general.
      // That's why these parameters are not mentioned in the comments
      // for the executable.
      long dummy;
      if ( NCONF_get_number_e(conf,sec,"slow_server",&dummy) )
      {
         if ( dummy != 0 )
         {
            SlowServer = true;

            // Get delays
            if (NCONF_get_number_e(conf,sec,"server_delay_s",&ServerDelayS))
            {
               if ( ServerDelayS < 0 )
               {
                  throw CODEX_Server::BCBadValueException(
                     __FILE__ , __LINE__ , config_file, "server_delay_s" );
               }
            }
            if (NCONF_get_number_e(conf,sec,"server_delay_us",&ServerDelayUS))
            {
               if ( ServerDelayUS < 0 )
               {
                  throw CODEX_Server::BCBadValueException(
                     __FILE__ , __LINE__ , config_file, "server_delay_us" );
               }
            }
            if (NCONF_get_number_e(conf,sec,"server_slowdown",&ServerSlowdown))
            {
               if ( ServerSlowdown < 0 )
               {
                  throw CODEX_Server::BCBadValueException(
                     __FILE__ , __LINE__ , config_file, "server_slowdown" );
               }
            }

            cout << "Simulating a slow processor with a delay of "
                 << ServerDelayS << "s, "
                 << ServerDelayUS << "us "
                 << " and slowdown of " << ServerSlowdown << "%"
                 << endl;
         }
      }
      if ( NCONF_get_number_e(conf,sec,"slow_link",&dummy) )
      {
         if ( dummy != 0 )
         {
            SlowLink = true;

            // Get delays
            if (NCONF_get_number_e(conf,sec,"link_delay_s",&LinkDelayS))
            {
               if ( LinkDelayS < 0 )
               {
                  throw CODEX_Server::BCBadValueException(
                     __FILE__ , __LINE__ , config_file, "link_delay_s" );
               }
            }
            if (NCONF_get_number_e(conf,sec,"link_delay_us",&LinkDelayUS))
            {
               if ( LinkDelayUS < 0 )
               {
                  throw CODEX_Server::BCBadValueException(
                     __FILE__ , __LINE__ , config_file, "link_delay_us" );
               }
            }

            cout << "Simulating a slow link with a delay of "
                 << LinkDelayS << "s, "
                 << LinkDelayUS << "us"
                 << endl;
            CODEX_Events::DelayedEvent::sDelay = LinkDelayS;
            CODEX_Events::DelayedEvent::usDelay = LinkDelayUS;
         }
      }

      // Clean up the configuration file pointer.
      NCONF_free( conf );
   }
   catch ( CODEX_Exceptions::ExceptionBase& e )
   {
      if ( 0 != conf ) NCONF_free(conf);
      e.report();
      throw;
   }
}

void
check_client_servers()
{
   fd_set read_set;
   fd_set write_set;
   FD_ZERO( &read_set );
   FD_ZERO( &write_set );
   int smax=0;
   CODEX_Server::ServerState* serverState =
      CODEX_Server::ServerState::instance();

   CODEX_Server::ServerState::ServerList::const_iterator cServItr =
      serverState->serverListBegin();
   for ( ; cServItr != serverState->serverListEnd() ; ++cServItr )
   {
      int s = (*cServItr)->set_fd( &read_set,
                                   CODEX_Quorum::SocketBase::kRead );
      (*cServItr)->set_fd( &write_set, CODEX_Quorum::SocketBase::kWrite );
      smax = ( s > smax ) ? s : smax;
   }

   struct timeval async_timeout;
   async_timeout.tv_sec = 0;
   async_timeout.tv_usec = 0;

   // ignore errors from select(2)
   if ( 0 < select( smax+1, &read_set, &write_set, 0, &async_timeout ) )
   {
      for ( cServItr = serverState->serverListBegin() ;
            cServItr != serverState->serverListEnd() ;
            ++cServItr )
      {
         CODEX_Quorum::LocalServer* cs = *cServItr;

         // We take a harsh view towards errors involving clients.
         // If there's a problem, we kill the connection.
         try
         {
            // Do we need to read?
            if ( cs->isset_fd(&read_set,CODEX_Quorum::SocketBase::kRead) )
            {
               cs->processRequest(CODEX_Quorum::SocketBase::kRead);
            }

            // Do we need to write?
            if ( cs->isset_fd(&write_set,CODEX_Quorum::SocketBase::kWrite) )
            {
               cs->processRequest(CODEX_Quorum::SocketBase::kWrite);
            }
         }
         catch ( ... )
         {
            // Remove this server from serverList and back up.
            --cServItr;
            serverState->removeServer( cs );
         }
      }
   }
}

void
process_next_event( CODEX_Events::QType& eventQueue )
{
   if ( eventQueue.empty() ) return;
   CODEX_Events::EventBase* event = eventQueue.front();
   eventQueue.pop();

   if ( SlowServer )
   {
      struct timeval tv;
      tv.tv_sec  = ServerDelayS;
      tv.tv_usec = ServerDelayUS;
      select(0,0,0,0,&tv);
   }
   if ( event->handle() )
   {
      delete event;
   }
   else
   {
      // Event wasn't successfully handled yet.
      eventQueue.push(event);
   }
}

void
consume_queue( CODEX_Events::QType& eventQueue )
{
   while ( ! eventQueue.empty() )
   {
      delete eventQueue.front();
      eventQueue.pop();
   }
}

void
move_delayed_events( CODEX_Events::QType& delayedEventQueue,
                     CODEX_Events::QType& eventQueue )
{
   while ( ! delayedEventQueue.empty() )
   {
      eventQueue.push(
         new CODEX_Events::DelayedEvent( delayedEventQueue.front() ) );
      delayedEventQueue.pop();
   }
}

void
garbage_collect_activities( CODEX_Events::DeadPileType& deadPile )
{
   while ( deadPile.size() > 0 )
   {
      CODEX_Events::Activity* act = deadPile.back();
      deadPile.pop_back();
      delete act;
   }
}

void
garbage_collect_servers()
{
   CODEX_Server::ServerState* serverState =
      CODEX_Server::ServerState::instance();
   if ( 0 == serverState ) return;
   serverState->garbageCollectServers();
}


int main( int argc, char** argv )
{
   //============================================================
   // Grab the server state object.
   //============================================================
   CODEX_Server::ServerState* serverState =
      CODEX_Server::ServerState::instance();
   CODEX_KeyService::StateInfo* keyServiceState =
      CODEX_KeyService::StateInfo::instance();
   CODEX_APSS::StateInfo* apssState = CODEX_APSS::StateInfo::instance();


   //============================================================
   // Seed the random number generator.
   //============================================================
   unsigned char seed[40] = 
   {0x38,0x4F,0x67,0x4C,0xA6,0x02,0x3D,0x9E,
    0x69, 0x2c, 0x3f, 0xfd, 0x98, 0x50, 0xcc,
    0x76, 0xfa, 0x78, 0x34, 0x12, 0x23, 0xdf,
    0x8a, 0xd4, 0x34, 0xcd, 0x98, 0x59, 0x19,
    0x35, 0x4f, 0xda, 0xed, 0x4e, 0x99, 0x0f};
   RAND_seed(seed,40);


   //============================================================
   // Load SSLeay algorithms
   //============================================================
   SSL_load_error_strings();
   SSLeay_add_ssl_algorithms();


   //============================================================
   // Catch interrupts, which should halt the main loop.
   //============================================================
   signal(SIGINT,handle_interrupt);
   //signal(SIGSEGV,handle_crash);


   //============================================================
   // Process the command line and configure the server.
   //============================================================
   cout << "Configuring server..." << endl;
   configure_server(argc,argv);
   cout << "...Server configuration done" << endl;


   //============================================================
   // Set up the bookkeeping objects for the concurrency system.
   //============================================================
   CODEX_Events::DeadPileType deadPile;
   CODEX_Events::QType eventQueue;
   CODEX_Events::QType delayedEventQueue;
   CODEX_Events::QType* pLinkQueue;
   if ( SlowLink )
   {
      pLinkQueue = &delayedEventQueue;
   }
   else
   {
      pLinkQueue = &eventQueue;
   }
   CODEX_Events::QType& linkQueue = *pLinkQueue;
   

   CODEX_Server::InternalServer is( serverState->serverPort(),
                                    serverState->serverInSocketBuilder() );
   serverState->addServer( &is );
   try
   {
      is.enable();
   }
   catch ( CODEX_Quorum::QSExceptionBase& e )
   {
      e.report();
      return 1;
   }


   //============================================================
   // Create top-level client servers and activities
   //============================================================
   CODEX_Server::BroadcastRequestHandler broadcastHandler( deadPile,
                                                           eventQueue );
   CODEX_Server::UnicastRequestHandler unicastHandler( deadPile,
                                                       eventQueue );
   CODEX_Server::ShareLabelChallengeHandler challengeHandler( deadPile,
                                                              eventQueue );
   CODEX_Server::SignRequestHandler signHandler( deadPile, eventQueue );
   CODEX_KeyService::DelegateRequestDistributor delegateDistributor(
      deadPile,
      eventQueue,
      &broadcastHandler,
      &signHandler );
   CODEX_KeyService::ClientMessageVerifier* clientMsgVerifier =
      new CODEX_KeyService::ClientMessageVerifier( deadPile,
                                                   eventQueue,
                                                   &delegateDistributor );


   //============================================================
   // Create an activity to connect to other quorum servers
   //============================================================
   CODEX_Server::QuorumBuilderAct* quorumBuilder =
      new CODEX_Server::QuorumBuilderAct( deadPile, eventQueue );

   CODEX_Server::ServerResponseHandler serverResponseHandler( deadPile,
                                                              eventQueue );

   //============================================================
   // Create a message dispatcher and attach parsers to it.
   //============================================================
   CODEX_Server::ServerMessageDispatcher messageDispatcher;

   // Read messages from client delegates.
   const unsigned char  clientMessageDomain = 1;

   CODEX_KeyService::ClientMessageSigner
      delegateMsgSigner( deadPile, eventQueue, &serverResponseHandler );

   CODEX_KeyService::RoutedClientMessageVerifier
      delegateVerifier( deadPile, eventQueue, &delegateMsgSigner );

   CODEX_KeyService::ClientMessageParser
      clientMessageParser( deadPile, linkQueue, &delegateVerifier );

   keyServiceState->setMessageDomain( clientMessageDomain );
   messageDispatcher.addParser( clientMessageDomain, &clientMessageParser );

   // Read supported responses from client delegates.
   const unsigned char  clientDelegateDomain = 2;

   CODEX_KeyService::ClientResponseSigner
      delegateRespSigner( deadPile, eventQueue, &serverResponseHandler );

   CODEX_KeyService::DelegateResponseVerifier
      delegateResponseVerifier( deadPile, eventQueue, &delegateRespSigner );

   CODEX_KeyService::ClientResponseParser
      clientResponseParser( deadPile, linkQueue, &delegateResponseVerifier );

   keyServiceState->setDelegationDomain( clientDelegateDomain );
   messageDispatcher.addParser( clientDelegateDomain, &clientResponseParser );

   // Read APSS protocol messages.
   const unsigned char  apssDomain = 3;

   CODEX_APSS::MessageProcessor
      apssProc( deadPile,
                eventQueue,
                &serverResponseHandler,
                &unicastHandler );

   CODEX_APSS::MessageVerifier
      apssVerifier( deadPile, eventQueue, &apssProc, &serverResponseHandler );

   CODEX_APSS::MessageParser
      apssParser( deadPile, linkQueue, &apssVerifier );

   CODEX_APSS::MessageDeliverer
      apssDeliverer( deadPile,
                     eventQueue,
                     &serverResponseHandler,
                     &broadcastHandler );
   apssState->setMessageDeliverer( &apssDeliverer );

   apssState->setDomain( apssDomain );
   messageDispatcher.addParser( apssDomain, &apssParser );

   CODEX_APSS::TriggerActivity( deadPile, eventQueue );

   //============================================================
   // Create a quorum dispatcher.
   //============================================================
   CODEX_Server::ServerQuorumDispatcher quorumDispatcher;


   //============================================================
   // Set up the server that listens for new client connections.
   //============================================================
   CODEX_KeyService::ClientServer clientServer(
      keyServiceState->clientPort(),
      serverState->clientSocketBuilder(),
      deadPile,
      eventQueue,
      clientMsgVerifier );
   serverState->addServer( &clientServer );
   try
   {
      clientServer.enable();
   }
   catch ( CODEX_Quorum::QSExceptionBase& e )
   {
      e.report();
      return 1;
   }


   //============================================================
   // Get the configured quorum system.
   //============================================================
   CODEX_Quorum::QuorumSystem* quorumSystem = serverState->quorumSystem();
   if ( 0 == quorumSystem )
   {
      return 1;
   }


   //============================================================
   // main loop
   //============================================================
   running = true;
   while(running)
   {
      try
      {
         struct timeval tv;
         if ( SlowServer )
         {
            gettimeofday(&tv,0);
         }

         // Check for server-to-server activity.
         quorumSystem->poll( messageDispatcher, quorumDispatcher );

         // Check for client-server activity.
         check_client_servers();

         // Clean up any terminated Activitys.
         garbage_collect_activities( deadPile );

         // Clean up any terminated LocalServers.
         garbage_collect_servers();

         // Move delayed events into the normal queue.
         move_delayed_events( delayedEventQueue, eventQueue );

         // Process the next event, if any.
         process_next_event( eventQueue );

         if ( SlowServer )
         {
            // Now delay some fraction of the time just taken.
            // This is inexact, since select() is based on the wall clock.
            struct timeval delay;
            gettimeofday(&delay,0);
            long int t =
               ( delay.tv_sec - tv.tv_sec ) * 1000000 +
               ( delay.tv_usec - tv.tv_usec );
            t *= ServerSlowdown;
            t /= 100; // the slowdown is a percentage
            delay.tv_sec = t / 1000000;
            delay.tv_usec = t % 1000000;
            select(0,0,0,0,&delay);
         }
      }
      catch ( CODEX_Exceptions::ExceptionBase& e )
      {
         e.report();
         //backtrace_symbols_fd(0,0,fileno(stdout));
         running = false;
      }
   }


   //============================================================
   // Clean up.
   //============================================================
   garbage_collect_activities( deadPile );
   garbage_collect_servers();
   consume_queue( eventQueue );
   delete clientMsgVerifier;
   delete quorumBuilder;
   CODEX_KeyService::StateInfo::destroy();
   CODEX_APSS::StateInfo::destroy();
   CODEX_Server::ServerState::destroy();
   ERR_free_strings();
   ERR_remove_state(0);
   EVP_cleanup();
   RAND_cleanup();


   //============================================================
   // Now we're done.
   //============================================================
   return 0;
}

---