\documentclass[10pt]{article}
\usepackage{pplmanual}
\input{../pplmanual}

\title{Bluegene Emulator}
\version{0.01}
\credits{BlueGene Emulator was developed by Arun Singla, Neelam Saboo
and Joshua Unger under the guidance of Prof. L. V. Kale.}

\begin{document}
\maketitle

\section{Introduction}

Blue Gene is a proposed one million processor machine from IBM.

The Blue Gene emulator environment is designed with the following
objectives:

\begin{enumerate}
\item To support a realistic Blue Gene API on existing parallel machines

\item To obtain first-order performance estimates of algorithms

\item To facilitate implementations of alternate programming models for
      Blue Gene
\end{enumerate}

The ``Blue Gene'' machine supported by the emulator consists of
three-dimensional grid of 1-chip nodes.  The user may specify the size
of the machine along each dimension (e.g. 34x34x36).  The chip supports
$k$ threads (e.g. 200), each with its own integer unit.  The proximity of
the integer unit with individual memory modules within a chip is not
currently modeled.

The API supported by the emulator can be broken down into several
components:

\begin{enumerate}
\item Low-level API for chip-to-chip communication
\item Mid-level API that supports local micro-tasking with a chip level
scheduler with features such as: read-only variables, reductions, broadcasts,
distributed tables, get/put operations
\item Migratable objects with automatic load balancing support
\end{enumerate}

Of these, the first two have been implemented.  The simple time stamping
algorithm, without error correction, has been implemented.  More
sophisticated timing algorithms, specifically aimed at error correction,
and more sophisticated features (2, 3, and others), as well as libraries
of commonly needed parallel operations are part of the proposed work for
future.

The following sections define the appropriate parts of the API, with
example programs and instructions for executing them.

\section{Blue Gene Programming Environment}

The basic philosophy of the Blue Gene Emulator is to hide intricate details
of Blue Gene machine from
application developer.Thus, the application developer needs to provide
intialization details and handler
functions only and gets the result as though running on a real machine.
Communication, Thread creation,
Time Stamping, etc are done by the emulator.

\subsection{Blue Gene API: Level 0}

\function{void putMessage(PacketMsg *)}
\desc{
        chip-to-chip communication function, invoked by Blue Gene
        environment when a node calls sendPacket
        to put the message in the inBuffer of target node.
}

\function{boolean checkReady()}
\desc{
        invoked by communication thread to see if there is any unattended
        message in inBuffer.
}

\function{PacketMsg *getMessage()}
\desc{
        invoked by communication thread to retrieve the unattended message
        in inBuffer.
}

\subsection{Initialization API: Level 1a}

Execution starts at BgInit(Main *), where Blue Gene machine parameters are
initialized.

\function{void CreateBlueGene(CreateBgNodeMsg *msg)}
\desc{
Specifies the machines configuration in CreateBgNodeMsg.
}

Data required for initialization of Node is specified in a system
defined type CreateBgNodeMsg.

\begin{alltt}
        class CreateBgNodeMsg
        \{
                public:
                  int numCTh ;
                  int numWTh ;
                  int numBgX ;
                  int numBgY ;
                  int numBgZ ;
        \} ;
\end{alltt}

\function{int  getNumArgs()}
\desc{
Return the number of command line arguments
}

\function{const char** getArgs()}
\desc{
Return command line arguments
}

\function{typedef void (*BgHandler)(void*)}
\desc{
This is type defined in BlueGene.h. It represents a handler function
that returns nothing and takes a (void *)
}

The Runtime system calls BgNodeInit(BgNode *) for each node, where
application handlers are registered and
computation is triggered by creating a task at required nodes. The Node
Variables are encapsulated in a struct
definition and a pointer to this (node private variable) is returned.

\function{void  registerHandler(int handlerID, BgHandler h)}
\desc{
Register a Handler with each node
}

\function{void addMessage(PacketMsg *msgPtr, int handlerID, int threadCategory)}
\desc{
Create a micro-task, specifiy the handler function to be
used for this message i.e. handlerID, and
specify the thread category:
\begin{description}
\item[1:] a small piece of work that can be done by
communication thread itself, so NO scheduling overhead.
\item[0:] a large piece of work, so communication thread
schedules it for a worker thread
\end{description}
}

InterNode communication messages are to be inherited from PacketMsg

\begin{alltt}
        class PacketMsg
        \{
            public:
                  int srcX ;
                  int srcY ;
                  int srcZ ;
                  int destX ;
                  int destY ;
                  int destZ ;
                  int numBytes ;
                  double sendTime ;
                  double recvTime ;
        \} ;
\end{alltt}

After completion of execution, Blue Gene environment invokes a user defined
function BgFinish().

\subsection{Handler Function API: Level 1a}

\function{void sendPacket(int x, int y, int z, PacketMsg *msgPtr, int handlerID, int threadCategory)}
\desc{
Sends a PacketMsg pointer to Node[x,y,z] and also specifies the
handler function to be used for this message ie. handlerID,
specify the thread category:
\begin{description}
\item[1:] a small piece of work that can be done by
communication thread itself, so NO scheduling overhead.
\item[0:] a large piece of work, so communication thread
schedules it for a worker thread
\end{description}
}

\function{void getXYZ(int\& x, int\& y, int\& z)}
\desc{
Gets which Blue Gene node do the invoking thread belongs to
}

\function{double getTime()}
\desc{
Returns current time of thread in microseconds (execution time since
the application started)
}


\section{Writing a Blue Gene Application}

\subsection{Application Skeleton}

\begin{alltt}
Handler declarations
Application specific messages, inherited from PacketMsg
Struct definitions encapsulating Node (specific) variables

void  BgInit(Main *)  function
  Make a Blue Gene node creation message, CreateBgNodeMsg
  Initialize number of Communication Threads and 
    number of Worker Threads per node
  Initialize number of Blue Gene nodes in X, Y, and Z dimension.
  Initialize the emulator by calling CreateBlueGene(CreateBgNodeMsg)

void *BgNodeInit(BgNode *) function
  Register handlers, registerHandler(handlerID, handler)
  Send initialization message packets to node, 
    addMessage(messagePointer, handlerID)
  Declare Node Variables (struct) and return a pointer to it.

void  BgFinish()  function
  detects Quiescence

Handler Function 1, void handlerName(ThreadInfo *info)
Hanlder Function 2, void handlerName(ThreadInfo *info)
..
Handler Function N, void handlerName(ThreadInfo *info)

\end{alltt}

\subsection{Sample Application 1}

\begin{alltt}
/* Application: 
 *   Each node starting at [0,0,0] sends a packet to next node in
 *   the ring order.
 *   After node [0,0,0] gets message from last node
 *   in the ring, the application ends.
 */

#include "BlueGene.h"
#define  computeID 1

extern "C" void compute(ThreadInfo *) ;

class MyMsg : public PacketMsg
\{
public:
  int dummy ;
\} ;


void BgInit(Main *main)
\{
  int num_comm = 1, num_work = 2;       // default number of communication
                                        // and worker threads per node
  int num_args = main->getNumArgs();
  if (num_args < 4) \{ 
    // Abort application: insufficient number of arguments
    CkAbort("Usage: ring <x> <y> <z> [<numCommTh> <numWorkTh>]\verb+\n+"); 
  \}

  if (num_args > 5) \{ num_work = atoi(main->getArgs()[5]); \}
  if (num_args > 4) \{ num_comm = atoi(main->getArgs()[4]); \}

  CreateBgNodeMsg *bgNodeMsg = new CreateBgNodeMsg;
  bgNodeMsg->numBgX = atoi(main->getArgs()[1]);
  bgNodeMsg->numBgY = atoi(main->getArgs()[2]);
  bgNodeMsg->numBgZ = atoi(main->getArgs()[3]);
  bgNodeMsg->numCTh = num_comm;
  bgNodeMsg->numWTh = num_work;

  main->CreateBlueGene(bgNodeMsg);
  return;
\}

void* BgNodeInit(BgNode *bgNode)
\{
  bgNode->registerHandler(computeID, compute) ;

  // trigger computation at Node[0,0,0]
  if(bgNode->thisIndex.x==0 \&\& bgNode->thisIndex.y==0 \&\& bgNode->thisIndex.z==0)
  \{
   MyMsg *msg = new MyMsg ;
   msg->dummy = 0 ;
   bgNode->addMessage(msg, computeID, 0) ;
  \}
  return NULL;    // No node variables for this example
\}

void BgFinish() \{\}

void compute(ThreadInfo *info)
\{
  int i, j, k ;
  int ni, nj, nk ;

  info->bgNode->getXYZ(i,j,k) ;

  if(i==info->bgNode->numBgX-1 \&\& j==info->bgNode->numBgY-1 \&\& 
     k==info->bgNode->numBgZ-1)
  \{
           ckout << "Exiting" << endl ;
           info->bgNode->finish() ;
           return ;
  \}

  nk = k + 1 ;
  nj = j ;
  ni = i ;
  if( nk==info->bgNode->numBgZ )
  \{
          nk = 0 ;
         nj = j+1 ;
         if ( nj==info->bgNode->numBgY )
         \{
                nj = 0 ;
                ni = i+1 ;
                if ( ni==info->bgNode->numBgX ) \{
                    ni = 0 ;
                \}
         \}
  \}
  MyMsg *msg = new MyMsg ;
  msg->dummy = i+j+k ;
  info->bgNode->sendPacket(ni, nj, nk, msg, computeID, 0) ;
\}
\end{alltt}


\subsection{Sample Application 2}

\begin{alltt}

/* Application: 
 *   Find the maximum element.
 *   Each node computes maximum of it's elements and
 *   the max values it received from other nodes
 *   and sends the result to next node in the reduction sequence.
 * Reduction Sequence: Reduce max data to X-Y Plane
 *   Reduce max data to Y Axis
 *   Reduce max data to origin.
 */

#include "BlueGene.h"

#define A_SIZE 4

#define reduceID                    1
#define computeMaxID        2
#define contributeID             3

//handler declarations
void reduce(ThreadInfo *) ;
void computeMax(ThreadInfo *) ;
void contribute(ThreadInfo *) ;

//Application specifice messages inherited from PacketMsg
class contributeMsg: public PacketMsg
\{
public:
  int max ;
\} ;

class computeMaxMsg: public PacketMsg
\{\} ;

class reduceMsg: public PacketMsg
\{\} ;

//Node variables encapsulated in a struct definition
typedef struct userDataStruct
\{
  int data[A_SIZE] ;
  int count ;
\} userData ;

void BgInit(Main *main)
\{
  int num_comm = 1, num_work = 2;       // default number of communication
and worker threads
  int num_args = main->getNumArgs();
  if (num_args < 4) \{ 
    CkAbort("Usage: maxReduceNV <x> <y> <z> [<numCommTh> <numWorkTh>]\verb+\n+"); 
  \}
  if (num_args > 5) \{ num_work = atoi(main->getArgs()[5]); \}
  if (num_args > 4) \{ num_comm = atoi(main->getArgs()[4]); \}

  CreateBgNodeMsg *bgNodeMsg = new CreateBgNodeMsg;
  bgNodeMsg->numBgX = atoi(main->getArgs()[1]);
  bgNodeMsg->numBgY = atoi(main->getArgs()[2]);
  bgNodeMsg->numBgZ = atoi(main->getArgs()[3]);
  bgNodeMsg->numCTh = num_comm;
  bgNodeMsg->numWTh = num_work;

  main->CreateBlueGene(bgNodeMsg);
  return;
\}

void* BgNodeInit(BgNode *bgNode)
\{
  //register handlers
  bgNode->registerHandler(reduceID, reduce) ;
  bgNode->registerHandler(computeMaxID, computeMax) ;
  bgNode->registerHandler(contributeID, contribute) ;

  //triger computer at each node
  computeMaxMsg *msg = new computeMaxMsg;
  bgNode->addMessage(msg, computeMaxID, 0);

  //declare node variable and return a pointer
  userData *ud = new userData ;
  ud->count = 0 ;
  for(int i=0; i<A_SIZE; i++)
   ud->data[i] = 0 ;

  return (void*)ud ;
\}

void BgFinish()
\{\}

void computeMax(ThreadInfo *info)
\{
  int A[A_SIZE][A_SIZE];
  int i, j;
  int max = 0;

  int x,y,z;
  info->bgNode->getXYZ(x,y,z);

  // Initialize data in each node
  for (i=0;i<A_SIZE;i++)
    for (j=0;j<A_SIZE;j++)
      A[i][j] = info->bgNode->numBgX * info->bgNode->numBgY * 
                info->bgNode->numBgZ - x*y*z - i*j ;

  // Find Max
  for (i=0;i<A_SIZE;i++)
    for (j=0;j<A_SIZE;j++)
      if (max < A[i][j])
      \{
             max = A[i][j];
      \}

  // contribute the results for reduction
  contributeMsg *msg = new contributeMsg;
  msg->max = max;
  info->bgNode->addMessage(msg,contributeID,1);

  ckout << "computeMax in " << x << ", " << y << ", " << z << endl;
  ckout << "contributed max value " << max << endl;
\}

void contribute(ThreadInfo *info)
\{
  int x,y,z;
  info->bgNode->getXYZ(x,y,z);

  // Accessing node variables : get number of data values received at this node
  int count = ((userData*)(info->bgNode->nvData))->count++ ;
  // Store new contribution to node variables
  ((userData*)(info->bgNode->nvData))->data[count++] =
((contributeMsg*)(info->msg))->max ;

 // Compute the required Count of data values at each node to start reduction 
 // at that node, depends on reduction sequence
  int reqCount ;

  if(z==info->bgNode->numBgZ-1)
  \{
         reqCount = 1 ;
  \}
  else if(z>0 || (z==0 \&\& x==info->bgNode->numBgX-1))
  \{
         reqCount = 2 ;
  \}
  else if(x>0 || (x==0 \&\& y==info->bgNode->numBgY-1))
  \{
         reqCount = 3 ;
  \}
  else
         reqCount = 4 ;

  if(count==reqCount)     //if data for reduction is ready
  \{
       reduceMsg *msg = new reduceMsg ;
       info->bgNode->addMessage(msg, reduceID, 0) ;
       ckout << "contribute in Node " << x << ", " << y << ", " << z << endl;
       ckout << "Values collected " << count << ", calling reduction " << endl;
       return ;
  \}

  ckout << "contribute in Node " << x << ", " << y << ", " << z << endl ;
  ckout << "Values collected " << count << ", reqCount " << reqCount << endl;
\}

void reduce(ThreadInfo *info)
\{
  int x,y,z;
  info->bgNode->getXYZ(x,y,z);
  ckout << "reduce in " << x << ", " << y << ", " << z << endl;

  //do reduction
  int max = 0 ;
  int count = ((userData*)(info->bgNode->nvData))->count ;
  for(int i=0; i<count; i++)
  \{
           if(max<((userData*)(info->bgNode->nvData))->data[i])
           max = ((userData*)(info->bgNode->nvData))->data[i] ;
  \}

  if(x==0 \&\& y==0 \&\& z==0)
  \{
         ckout << "Exiting: max value is " << max << endl ;
         info->bgNode->finish() ;
         return ;
  \}

  //send max to destination, depends on reduction sequence
  if(z>0)
   z-- ;
  else if(x>0)
   x-- ;
  else
   y-- ;

  contributeMsg *msg = new contributeMsg;
  msg->max = max;
  info->bgNode->sendPacket(x,y,z,msg,contributeID,1);

  ckout << "sending max value " << max << " to " << x << ", " << y << ", " << z << endl ;
\}
\end{alltt}

\section{Compiling and Running}

Compile Blue Gene emulator programs using {\tt charmc} as one would
in the case of normal \charmpp{} programs. In order to link the
Blue Gene programs, use \texttt{-language bluegene} as an argument
to the {\tt charmc} linker.

\input{index}
\end{document}
