/*
 * Copyright 1998-2001, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Arun Rodrigues, and Brian Barrett with
 *          Kinis L. Meyer, M. D. McNally, and Andrew Lumsdaine
 * 
 * This file is part of the Notre Dame LAM implementation of MPI.
 * 
 * You should have received a copy of the License Agreement for the Notre
 * Dame LAM implementation of MPI along with the software; see the file
 * LICENSE.  If not, contact Office of Research, University of Notre
 * Dame, Notre Dame, IN 46556.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted subject to the conditions specified in the
 * LICENSE file.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 * 
 * Additional copyrights may follow.
 * 
 *
 * $Id: alltoall.c,v 1.2 2001/01/22 18:03:35 jsquyres Exp $
 *
 * Example program that sends a message from each node each other node
 * in a blocking pattern.
 */

#include <stdio.h>
#include "mpi.h"


int
main(int argc, char* argv[])
{
  int i;
  int rank;
  int size;
  int tag = 201;
  int receive_message;

  /* Normal MPI startup */

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &size);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);

  /* Check to ensure that we have 2 ranks */

  if (size < 2) {
    printf("This program must be run with 2 or more ranks.\n");
    MPI_Finalize();
    return 1;
  }

  /* Loop over all ranks */

  /* Note that we carefully distinguish between the order of sending
     and receiving.  It is *not* sufficient to have a single loop
     where each rank does a simple "MPI_Send(...); MPI_Recv(...);" --
     doing so can lead to deadlock.  This is a subtlety in the MPI
     standard -- MPI_Send() *may* or *may not* block.  In LAM,
     MPI_Send() will block if a message is too long; it will wait
     until the message has been started to be received on the target
     rank.  Under that size, MPI_Send() will [most likely] return
     immediately regardless of what the target rank is doing.

     Note that this is not a LAM-specific issue; every MPI
     implementation is coded in this way -- that MPI_Send tries to
     complete immediately, but above a certain threshhold (usually
     related a combination of message size and the cumulative size of
     unreceived messages to that target), MPI_Send may block until the
     target rank starts receiving. */

  /* Notice also the use of MPI_STATUS_IGNORE.  Since this example
     program doesn't use the MPI_Status object at all, we use the
     special constant MPI_STATUS_IGNORE to tell MPI not to fill it
     in. */

  /* Finally, notice that this program is actually a poor example of
     the all-to-all communication pattern -- there is some degree of
     serialization in the sending of messages.  For example, rank N
     has to wait for rank 0 to send to it before it can continue.
     That is, rank 0 causes a daisy-chain of reactions that allow the
     rest of the ranks to continue -- each rank will not "start" until
     rank 0 contacts it. */

  for (i = 0; i < size; i++) {
    if (i == rank) {

      /* Skip communicating with myself */

      printf("Rank %d not sending to myself\n", i);
      continue;
    } else if (i < rank) {

      /* If our rank number is *greater* than the target rank, we send
         first, receive second. */

      printf("Rank %d sending message \"%d\" to rank %d\n",
	     rank, rank, i);
      MPI_Send(&rank, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
      MPI_Recv(&receive_message, 1, MPI_INT, i, tag, MPI_COMM_WORLD, 
	       MPI_STATUS_IGNORE);
      printf("Rank %d received message \"%d\" from rank %d\n",
	     rank, receive_message, i);
    } else {

      /* If our rank number is *smaller* than the target rank, we
         receive first, send second. */

      MPI_Recv(&receive_message, 1, MPI_INT, i, tag, MPI_COMM_WORLD, 
	       MPI_STATUS_IGNORE);
      printf("Rank %d received message \"%d\" from rank %d\n",
	     rank, receive_message, i);
      printf("Rank %d sending message \"%d\" to rank %d\n",
	     rank, rank, i);
      MPI_Send(&rank, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
    }
  }

  /* All done */

  printf("Rank %d finished\n", rank);
  MPI_Finalize();
  return 0;
}