We needed a way to synchronise a subset of tasks in \charmpp{}
programs. Quiescence detection is one way this can be done, if we want
to synchronize {\bf all} tasks, but barriers provide a more flexible
option with less overhead. 

To make use of barriers, the user initializes a {\em barrier group} in
the main function. A virtual binary tree structure is imposed on the
PEs, with PE 0 at the root.  PEs perform computations and then execute
\kw{atBarrier} when they want to synchronize.  \kw{atBarrier} takes
a single parameter of type {\tt FP *}.  The user must declare and
initialize some variable \uw{fnptr} as follows:

\begin{alltt}
  FP *fnptr = new FP;
  fnptr->fp = theFn;
\end{alltt}

The variable \uw{theFn} above is the void function to be executed when all
PEs have synchronized.

The barrier keeps track of child PEs, and when a given PE has heard
from all its children, it then notifies the parent.  When the root
hears from both children, all PEs are accounted for, and the
respective void function is executed on each PE. 

What follows is a very simple test program illustrating the usage of a
barrier.

\begin{alltt}
// File: test.ci

mainmodule Test \{
  extern module Barrier;
  readonly CkChareID mainhandle;
  readonly int barrierGroup;
  
  mainchare main \{
    entry main();
    entry void Quiescence1(void);
  \};
  
  group busy \{
    entry busy(void);
  \};
\};
\end{alltt}

\begin{verbatim}
// File: test.h

#include "Test.decl.h"
int barrierGroup;
CkChareID mainhandle;

class main : public Chare \{
public:
  main(CkArgMsg *m);
  void Quiescence1(void);
\};

class busy : public Group \{
public:
  busy(void);
\}
\end{alltt}

\begin{verbatim}
// File: test.C

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "charm++.h"

#include "test.h"
#include "Test.def.h"

#include "barrier.h"

main::main(CkArgMsg *m)
\{
  barrierGroup = barrierInit();
  CProxy_busy::ckNew();
  CkStartQD(CProxy_main::ckIdx_Quiescence1(), \&mainhandle);
\}

void main::Quiescence1() 
\{ 
  CkPrintf("All done... Exiting.\n");
  CkExit();
\}

void theFn();

busy::busy()
\{
  int i, j=0;
  FP *fnptr = new FP;

  for (i=0; i<(CkMyPe()+1)*1000000; i++)
    j = j + i;
  CkPrintf("[%d] going to barrier with j=%d\n", CkMyPe(), j);
  fnptr->fp = theFn;
  CProxy_barrier(barrierGroup).ckLocalBranch()->atBarrier(fnptr);
\}

void theFn()
\{
  CkPrintf("[%d] fnptr executing!\n", CkMyPe());
\}
\end{verbatim}
