ck-core/init.C

Go to the documentation of this file.
00001 /*****************************************************************************
00002  * $Source: /cvsroot/charm/src/ck-core/init.C,v $
00003  * $Author: gioachin $
00004  * $Date: 2008-06-03 00:10:09 $
00005  * $Revision: 2.153 $
00006  *****************************************************************************/
00045 
00046 #include "ck.h"
00047 #include "trace.h"
00048 
00049 void CkRestartMain(const char* dirname);
00050 
00051 #define  DEBUGF(x)     //CmiPrintf x;
00052 
00053 UChar _defaultQueueing = CK_QUEUEING_FIFO;
00054 
00055 UInt  _printCS = 0;
00056 UInt  _printSS = 0;
00057 
00058 UInt  _numExpectInitMsgs = 0;
00059 UInt  _numInitMsgs = 0;
00060 CksvDeclare(UInt,_numInitNodeMsgs);
00061 int   _infoIdx;
00062 int   _charmHandlerIdx;
00063 int   _initHandlerIdx;
00064 int   _roHandlerIdx;
00065 int   _roRestartHandlerIdx;
00066 int   _bocHandlerIdx;
00067 int   _nodeBocHandlerIdx;
00068 int   _qdHandlerIdx;
00069 int   _triggerHandlerIdx;
00070 int   _mainDone = 0;
00071 static int   _triggersSent = 0;
00072 
00073 CkOutStream ckout;
00074 CkErrStream ckerr;
00075 CkInStream  ckin;
00076 
00077 CkpvDeclare(void*,       _currentChare);
00078 CkpvDeclare(int,         _currentChareType);
00079 CkpvDeclare(CkGroupID,   _currentGroup);
00080 CkpvDeclare(void*,       _currentNodeGroupObj);
00081 CkpvDeclare(CkGroupID,   _currentGroupRednMgr);
00082 CkpvDeclare(GroupTable*, _groupTable);
00083 CkpvDeclare(GroupIDTable*, _groupIDTable);
00084 CkpvDeclare(CmiImmediateLockType, _groupTableImmLock);
00085 CkpvDeclare(UInt, _numGroups);
00086 
00087 CkpvDeclare(CkCoreState *, _coreState);
00088 
00089 CksvDeclare(UInt, _numNodeGroups);
00090 CksvDeclare(GroupTable*, _nodeGroupTable);
00091 CksvDeclare(GroupIDTable, _nodeGroupIDTable);
00092 CksvDeclare(CmiImmediateLockType, _nodeGroupTableImmLock);
00093 CksvDeclare(CmiNodeLock, _nodeLock);
00094 CksvStaticDeclare(PtrVec*,_nodeBocInitVec);
00095 CkpvDeclare(int, _charmEpoch);
00096 
00097 CkpvDeclare(Stats*, _myStats);
00098 CkpvDeclare(MsgPool*, _msgPool);
00099 
00100 CkpvDeclare(_CkOutStream*, _ckout);
00101 CkpvDeclare(_CkErrStream*, _ckerr);
00102 
00103 CkpvStaticDeclare(int,  _numInitsRecd); /* UInt changed to int */
00104 CkpvStaticDeclare(PtrQ*, _buffQ);
00105 CkpvStaticDeclare(PtrVec*, _bocInitVec);
00106 
00107 /*
00108         FAULT_EVAC
00109 */
00110 CpvDeclare(char *, _validProcessors);
00111 CpvDeclare(char ,startedEvac);
00112 
00113 int    _exitHandlerIdx;
00114 
00115 static Stats** _allStats = 0;
00116 
00117 static int   _numStatsRecd = 0;
00118 static int   _exitStarted = 0;
00119 
00120 static InitCallTable _initCallTable;
00121 
00122 #ifndef CMK_OPTIMIZE
00123 #define _STATS_ON(x) (x) = 1
00124 #else
00125 #define _STATS_ON(x) \
00126           CmiPrintf("stats unavailable in optimized version. ignoring...\n");
00127 #endif
00128 
00129 // fault tolerance
00130 typedef void (*CkFtFn)(const char *, CkArgMsg *);
00131 static CkFtFn  faultFunc = NULL;
00132 static char* _restartDir;
00133 
00134 int _defaultObjectQ = 0;            // for obejct queue
00135 int _ringexit = 0;                  // for charm exit
00136 int _ringtoken = 8;
00137 
00138 
00139 /*
00140         FAULT_EVAC
00141 
00142         flag which marks whether or not to trigger the processor shutdowns
00143 */
00144 static int _raiseEvac=0;
00145 static char *_raiseEvacFile;
00146 void processRaiseEvacFile(char *raiseEvacFile);
00147 
00148 static inline void _parseCommandLineOpts(char **argv)
00149 {
00150   if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
00151       _STATS_ON(_printCS);
00152   if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown"))
00153       _STATS_ON(_printSS);
00154   if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing"))
00155       _defaultQueueing = CK_QUEUEING_FIFO;
00156   if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing"))
00157       _defaultQueueing = CK_QUEUEING_LIFO;
00158   if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing"))
00159       _defaultQueueing = CK_QUEUEING_IFIFO;
00160   if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing"))
00161       _defaultQueueing = CK_QUEUEING_ILIFO;
00162   if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing"))
00163       _defaultQueueing = CK_QUEUEING_BFIFO;
00164   if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing"))
00165       _defaultQueueing = CK_QUEUEING_BLIFO;
00166   if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct"))
00167   {
00168 #if CMK_OBJECT_QUEUE_AVAILABLE
00169       _defaultObjectQ = 1;
00170       if (CkMyPe()==0)
00171         CmiPrintf("Charm++> Create object queue for every Charm object.\n");
00172 #else
00173       CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1.");
00174 #endif
00175   }
00176   if(CmiGetArgString(argv,"+restart",&_restartDir))
00177       faultFunc = CkRestartMain;
00178 #if __FAULT__
00179   if (CmiGetArgFlagDesc(argv,"+restartaftercrash","restarting this processor after a crash")){  
00180 # if CMK_MEM_CHECKPOINT
00181       faultFunc = CkMemRestart;
00182 # endif
00183       CmiPrintf("[%d] Restarting after crash \n",CmiMyPe());
00184   }
00185 #endif
00186   // shut down program in ring fashion to allow projections output w/o IO error
00187   if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion")) 
00188   {
00189     _ringexit = 1;
00190     if (CkMyPe()==0)
00191       CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken);
00192     if (_ringtoken > CkNumPes())  _ringtoken = CkNumPes();
00193   }
00194         /*
00195                 FAULT_EVAC
00196 
00197                 if the argument +raiseevac is present then cause faults
00198         */
00199         if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){
00200                 _raiseEvac = 1;
00201         }
00202 }
00203 
00204 static void _bufferHandler(void *msg)
00205 {
00206   DEBUGF(("[%d] _bufferHandler called.\n", CkMyPe()));
00207   CkpvAccess(_buffQ)->enq(msg);
00208 }
00209 
00210 static void _discardHandler(envelope *env)
00211 {
00212   DEBUGF(("[%d] _discardHandler called.\n", CkMyPe()));
00213   CmiFree(env);
00214 }
00215 
00216 #ifndef CMK_OPTIMIZE
00217 static inline void _printStats(void)
00218 {
00219   DEBUGF(("[%d] _printStats\n", CkMyPe()));
00220   int i;
00221   if(_printSS || _printCS) {
00222     Stats *total = new Stats();
00223     _MEMCHECK(total);
00224     for(i=0;i<CkNumPes();i++)
00225       total->combine(_allStats[i]);
00226     CkPrintf("Charm Kernel Summary Statistics:\n");
00227     for(i=0;i<CkNumPes();i++) {
00228       CkPrintf("Proc %d: [%d created, %d processed]\n", i,
00229                _allStats[i]->getCharesCreated(),
00230                _allStats[i]->getCharesProcessed());
00231     }
00232     CkPrintf("Total Chares: [%d created, %d processed]\n",
00233              total->getCharesCreated(), total->getCharesProcessed());
00234   }
00235   if(_printCS) {
00236     CkPrintf("Charm Kernel Detailed Statistics (R=requested P=processed):\n\n");
00237 
00238     CkPrintf("         Create    Mesgs     Create    Mesgs     Create    Mesgs\n");
00239     CkPrintf("         Chare     for       Group     for       Nodegroup for\n");
00240     CkPrintf("PE   R/P Mesgs     Chares    Mesgs     Groups    Mesgs     Nodegroups\n");
00241     CkPrintf("---- --- --------- --------- --------- --------- --------- ----------\n");
00242 
00243     for(i=0;i<CkNumPes();i++) {
00244       CkPrintf("%4d  R  %9d %9d %9d %9d %9d %9d\n      P  %9d %9d %9d %9d %9d %9d\n",i,
00245                _allStats[i]->getCharesCreated(),
00246                _allStats[i]->getForCharesCreated(),
00247                _allStats[i]->getGroupsCreated(),
00248                _allStats[i]->getGroupMsgsCreated(),
00249                _allStats[i]->getNodeGroupsCreated(),
00250                _allStats[i]->getNodeGroupMsgsCreated(),
00251                _allStats[i]->getCharesProcessed(),
00252                _allStats[i]->getForCharesProcessed(),
00253                _allStats[i]->getGroupsProcessed(),
00254                _allStats[i]->getGroupMsgsProcessed(),
00255                _allStats[i]->getNodeGroupsProcessed(),
00256                _allStats[i]->getNodeGroupMsgsProcessed());
00257     }
00258   }
00259 }
00260 #else
00261 static inline void _printStats(void) {}
00262 #endif
00263 
00264 static inline void _sendStats(void)
00265 {
00266   DEBUGF(("[%d] _sendStats\n", CkMyPe()));
00267 #ifndef CMK_OPTIMIZE
00268   envelope *env = UsrToEnv(CkpvAccess(_myStats));
00269 #else
00270   envelope *env = _allocEnv(StatMsg);
00271 #endif
00272   env->setSrcPe(CkMyPe());
00273   CmiSetHandler(env, _exitHandlerIdx);
00274   CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
00275 }
00276 
00277 static void _exitHandler(envelope *env)
00278 {
00279   DEBUGF(("exitHandler called on %d msgtype: %d\n", CkMyPe(), env->getMsgtype()));
00280   switch(env->getMsgtype()) {
00281     case ExitMsg:
00282       CkAssert(CkMyPe()==0);
00283       if(_exitStarted) {
00284         CmiFree(env);
00285         return;
00286       }
00287       _exitStarted = 1;
00288       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00289       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00290       CkNumberHandler(_nodeBocHandlerIdx, (CmiHandler)_discardHandler);
00291       env->setMsgtype(ReqStatMsg);
00292       env->setSrcPe(CkMyPe());
00293       // if exit in ring, instead of broadcasting, send in ring
00294       if (_ringexit){
00295         DEBUGF(("[%d] Ring Exit \n",CkMyPe()));
00296         const int stride = CkNumPes()/_ringtoken;
00297         int pe = 0;
00298         while (pe<CkNumPes()) {
00299           CmiSyncSend(pe, env->getTotalsize(), (char *)env);
00300           pe += stride;
00301         }
00302         CmiFree(env);
00303       }else{
00304         CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
00305       } 
00306       break;
00307     case ReqStatMsg:
00308       DEBUGF(("ReqStatMsg on %d\n", CkMyPe()));
00309       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00310       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00311       CkNumberHandler(_nodeBocHandlerIdx, (CmiHandler)_discardHandler);
00312         /*FAULT_EVAC*/
00313       if(CmiNodeAlive(CkMyPe())){
00314          _sendStats();
00315       } 
00316       _mainDone = 1; // This is needed because the destructors for
00317                      // readonly variables will be called when the program
00318                      // exits. If the destructor is called while _mainDone
00319                      // is 0, it will assume that the readonly variable was
00320                      // declared locally. On all processors other than 0, 
00321                      // _mainDone is never set to 1 before the program exits.
00322 #ifndef CMK_OPTIMIZE
00323       if (_ringexit) traceClose();
00324 #endif
00325       if (_ringexit) {
00326         int stride = CkNumPes()/_ringtoken;
00327         int pe = CkMyPe()+1;
00328         if (pe < CkNumPes() && pe % stride != 0)
00329           CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
00330         else
00331           CmiFree(env);
00332       }
00333       else
00334         CmiFree(env);
00335       if(CkMyPe()){
00336         DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
00337         ConverseExit();
00338       } 
00339       break;
00340     case StatMsg:
00341       CkAssert(CkMyPe()==0);
00342 #ifndef CMK_OPTIMIZE
00343       _allStats[env->getSrcPe()] = (Stats*) EnvToUsr(env);
00344 #endif
00345       _numStatsRecd++;
00346       DEBUGF(("StatMsg on %d with %d\n", CkMyPe(), _numStatsRecd));
00347                         /*FAULT_EVAC*/
00348       if(_numStatsRecd==CkNumValidPes()) {
00349         _printStats();
00350         DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
00351         ConverseExit();
00352       }
00353       break;
00354     default:
00355       CmiAbort("Internal Error(_exitHandler): Unknown-msg-type. Contact Developers.\n");
00356   }
00357 }
00358 
00359 static inline void _processBufferedBocInits(void)
00360 {
00361   CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
00362         CkpvAccess(_coreState));
00363   register int i = 0;
00364   PtrVec &inits=*CkpvAccess(_bocInitVec);
00365   register int len = inits.size();
00366   for(i=0; i<len; i++) {
00367     envelope *env = inits[i];
00368     if(env==0) continue;
00369     if(env->isPacked())
00370       CkUnpackMessage(&env);
00371     _processBocInitMsg(CkpvAccess(_coreState),env);
00372   }
00373   delete &inits;
00374 }
00375 
00376 static inline void _processBufferedNodeBocInits(void)
00377 {
00378   CkNumberHandlerEx(_nodeBocHandlerIdx,(CmiHandlerEx)_processHandler,
00379         CkpvAccess(_coreState));
00380   register int i = 0;
00381   PtrVec &inits=*CksvAccess(_nodeBocInitVec);
00382   register int len = inits.size();
00383   for(i=0; i<len; i++) {
00384     envelope *env = inits[i];
00385     if(env==0) continue;
00386     if(env->isPacked())
00387       CkUnpackMessage(&env);
00388     _processNodeBocInitMsg(CkpvAccess(_coreState),env);
00389   }
00390   delete &inits;
00391 }
00392 
00393 static inline void _processBufferedMsgs(void)
00394 {
00395   CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler,
00396         CkpvAccess(_coreState));
00397   envelope *env;
00398   while(NULL!=(env=(envelope*)CkpvAccess(_buffQ)->deq())) {
00399     if(env->getMsgtype()==NewChareMsg || env->getMsgtype()==NewVChareMsg) {
00400       if(env->isForAnyPE())
00401         CldEnqueue(CLD_ANYWHERE, env, _infoIdx);
00402       else
00403         CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
00404     } else {
00405       CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
00406     }
00407   }
00408 }
00409 
00410 static int _charmLoadEstimator(void)
00411 {
00412   return CkpvAccess(_buffQ)->length();
00413 }
00414 
00415 static void _sendTriggers(void)
00416 {
00417   int i, num, first;
00418   CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
00419   if (_triggersSent == 0)
00420   {
00421     _triggersSent++;
00422     num = CmiMyNodeSize();
00423     register envelope *env = _allocEnv(RODataMsg);
00424     env->setSrcPe(CkMyPe());
00425     CmiSetHandler(env, _triggerHandlerIdx);
00426     first = CmiNodeFirst(CmiMyNode());
00427     for (i=0; i < num; i++)
00428       if(first+i != CkMyPe())
00429         CmiSyncSend(first+i, env->getTotalsize(), (char *)env);
00430     CmiFree(env);
00431   }
00432   CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
00433 }
00434 
00435 void _initDone(void)
00436 {
00437   DEBUGF(("[%d] _initDone.\n", CkMyPe()));
00438   if (!_triggersSent) _sendTriggers();
00439   CkNumberHandler(_triggerHandlerIdx, (CmiHandler)_discardHandler);
00440   CmiNodeBarrier();
00441   if(CkMyRank() == 0) {
00442     _processBufferedNodeBocInits();
00443   }
00444   CmiNodeBarrier(); // wait for all nodegroups to be created
00445   _processBufferedBocInits();
00446   DEBUGF(("Reached CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
00447   CmiNodeBarrier();
00448   DEBUGF(("Crossed CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
00449   _processBufferedMsgs();
00450   CkpvAccess(_charmEpoch)=1;
00451 }
00452 
00453 static void _triggerHandler(envelope *env)
00454 {
00455   if (_numExpectInitMsgs && CkpvAccess(_numInitsRecd) + CksvAccess(_numInitNodeMsgs) == _numExpectInitMsgs)
00456   {
00457     DEBUGF(("Calling Init Done from _triggerHandler\n"));
00458     _initDone();
00459   }
00460   CmiFree(env);
00461 }
00462 
00463 static inline void _processROMsgMsg(envelope *env)
00464 {
00465   *((char **)(_readonlyMsgs[env->getRoIdx()]->pMsg))=(char *)EnvToUsr(env);
00466 }
00467 
00468 static inline void _processRODataMsg(envelope *env)
00469 {
00470   //Unpack each readonly:
00471   PUP::fromMem pu((char *)EnvToUsr(env));
00472   for(int i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pu);
00473   CmiFree(env);
00474 }
00475 
00476 static void _roRestartHandler(void *msg)
00477 {
00478   CkAssert(CkMyPe()!=0);
00479   register envelope *env = (envelope *) msg;
00480   CkpvAccess(_numInitsRecd)+=2;  /*++;*/
00481   _numExpectInitMsgs = env->getCount();
00482   _processRODataMsg(env);
00483 }
00484 
00485 static void _roHandler(void *msg)
00486 {
00487   CpvAccess(_qd)->process();
00488   _roRestartHandler(msg);
00489 }
00490 
00491 static void _initHandler(void *msg)
00492 {
00493   CkAssert(CkMyPe()!=0);
00494   register envelope *env = (envelope *) msg;
00495   switch (env->getMsgtype()) {
00496     case BocInitMsg:
00497       if (env->getGroupEpoch()==0)
00498         CkpvAccess(_numInitsRecd)++;
00499       CpvAccess(_qd)->process();
00500       CkpvAccess(_bocInitVec)->insert(env->getGroupNum().idx, env);
00501       break;
00502     case NodeBocInitMsg:
00503       CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
00504       if (env->getGroupEpoch()==0)
00505         CksvAccess(_numInitNodeMsgs)++;
00506       CksvAccess(_nodeBocInitVec)->insert(env->getGroupNum().idx, env);
00507       CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
00508       CpvAccess(_qd)->process();
00509       break;
00510     case ROMsgMsg:
00511       CkpvAccess(_numInitsRecd)++;
00512       CpvAccess(_qd)->process();
00513       if(env->isPacked()) CkUnpackMessage(&env);
00514       _processROMsgMsg(env);
00515       break;
00516     case RODataMsg:
00517       CkpvAccess(_numInitsRecd)+=2;  /*++;*/
00518       CpvAccess(_qd)->process();
00519       _numExpectInitMsgs = env->getCount();
00520       _processRODataMsg(env);
00521       break;
00522     default:
00523       CmiAbort("Internal Error: Unknown-msg-type. Contact Developers.\n");
00524   }
00525         DEBUGF(("[%d,%.6lf] _numExpectInitMsgs %d CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs) %d\n",CmiMyPe(),CmiWallTimer(),_numExpectInitMsgs,CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs)));
00526   if(_numExpectInitMsgs&&(CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs)>=_numExpectInitMsgs)) {
00527     _initDone();
00528   }
00529 }
00530 
00531 // CkExit: start the termination process, but
00532 //   then drop into the scheduler so the user's
00533 //   method never returns (which would be confusing).
00534 extern "C"
00535 void _CkExit(void) 
00536 {
00537   // Shuts down Converse handlers for the upper layers on this processor
00538   //
00539   CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00540   CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00541   CkNumberHandler(_nodeBocHandlerIdx, (CmiHandler)_discardHandler);
00542   DEBUGF(("[%d] CkExit - _exitStarted:%d %d\n", CkMyPe(), _exitStarted, _exitHandlerIdx));
00543 
00544   if(CkMyPe()==0) {
00545     if(_exitStarted)
00546       CsdScheduler(-1);
00547     envelope *env = _allocEnv(ReqStatMsg);
00548     env->setSrcPe(CkMyPe());
00549     CmiSetHandler(env, _exitHandlerIdx);
00550                 /*FAULT_EVAC*/
00551     CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
00552   } else {
00553     envelope *env = _allocEnv(ExitMsg);
00554     env->setSrcPe(CkMyPe());
00555     CmiSetHandler(env, _exitHandlerIdx);
00556     CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
00557   }
00558 #if ! CMK_BLUEGENE_THREAD
00559   _TRACE_END_EXECUTE();
00560   //Wait for stats, which will call ConverseExit when finished:
00561   CsdScheduler(-1);
00562 #endif
00563 }
00564 
00565 CkQ<CkExitFn> _CkExitFnVec;
00566 
00567 // wrapper of CkExit
00568 // traverse _CkExitFnVec to call registered user exit functions
00569 // CkExitFn will call CkExit() when finished to make sure other
00570 // registered functions get called.
00571 extern "C"
00572 void CkExit(void)
00573 {
00574         /*FAULT_EVAC*/
00575         DEBUGF(("[%d] CkExit called \n",CkMyPe()));
00576   if (!_CkExitFnVec.isEmpty()) {
00577     CkExitFn fn = _CkExitFnVec.deq();
00578     fn();
00579   }
00580   else
00581     _CkExit();
00582 }
00583 
00584 static void _nullFn(void *, void *)
00585 {
00586   CmiAbort("Null-Method Called. Program may have Unregistered Module!!\n");
00587 }
00588 
00589 extern void _registerLBDatabase(void);
00590 extern void _registerExternalModules(char **argv);
00591 extern void _ckModuleInit(void);
00592 extern void _loadbalancerInit();
00593 extern "C" void initCharmProjections();
00594 
00595 void _registerInitCall(CkInitCallFn fn, int isNodeCall)
00596 {
00597   if (isNodeCall) _initCallTable.initNodeCalls.enq(fn);
00598   else _initCallTable.initProcCalls.enq(fn);
00599 }
00600 
00601 void InitCallTable::enumerateInitCalls()
00602 {
00603   int i;
00604 #ifdef __BLUEGENE__
00605   if(BgNodeRank()==0) 
00606 #else
00607   if(CkMyRank()==0) 
00608 #endif
00609   {
00610     for (i=0; i<initNodeCalls.length(); i++) initNodeCalls[i]();
00611   }
00612   // initproc may depend on initnode calls.
00613   CmiNodeAllBarrier();
00614   for (i=0; i<initProcCalls.length(); i++) initProcCalls[i]();
00615 }
00616 
00617 CpvCExtern(int, cmiArgDebugFlag);
00618 extern "C" void CpdFreeze(void);
00619 
00620 extern "C" void initQd()
00621 {
00622         CpvInitialize(QdState*, _qd);
00623         CpvAccess(_qd) = new QdState();
00624         if (CmiMyRank() == 0) {
00625 #if !defined(CMK_CPV_IS_SMP) && !CMK_SHARED_VARS_UNIPROCESSOR
00626         CpvAccessOther(_qd, 1) = new QdState(); // for i/o interrupt
00627 #endif
00628         }
00629         _qdHandlerIdx = CmiRegisterHandler((CmiHandler)_qdHandler);
00630 }
00631 
00641 void _initCharm(int unused_argc, char **argv)
00642 { 
00643         int inCommThread = (CmiMyRank() == CmiMyNodeSize());
00644 
00645         DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer()));
00646 
00647         CkpvInitialize(PtrQ*,_buffQ);
00648         CkpvInitialize(PtrVec*,_bocInitVec);
00649         CkpvInitialize(void*, _currentChare);
00650         CkpvInitialize(int,   _currentChareType);
00651         CkpvInitialize(CkGroupID, _currentGroup);
00652         CkpvInitialize(void *, _currentNodeGroupObj);
00653         CkpvInitialize(CkGroupID, _currentGroupRednMgr);
00654         CkpvInitialize(GroupTable*, _groupTable);
00655         CkpvInitialize(GroupIDTable*, _groupIDTable);
00656         CkpvInitialize(CmiImmediateLockType, _groupTableImmLock);
00657         CkpvInitialize(UInt, _numGroups);
00658         CkpvInitialize(int, _numInitsRecd);
00659         CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
00660         CkpvInitialize(MsgPool*, _msgPool);
00661         CkpvInitialize(CkCoreState *, _coreState);
00662         /*
00663                 Added for evacuation-sayantan
00664         */
00665 #ifndef __BLUEGENE__
00666         CpvInitialize(char *,_validProcessors);
00667         CpvInitialize(char ,startedEvac);
00668         CpvInitialize(int,serializer);
00669 #endif
00670 
00671         CksvInitialize(UInt, _numNodeGroups);
00672         CksvInitialize(GroupTable*, _nodeGroupTable);
00673         CksvInitialize(GroupIDTable, _nodeGroupIDTable);
00674         CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock);
00675         CksvInitialize(CmiNodeLock, _nodeLock);
00676         CksvInitialize(PtrVec*,_nodeBocInitVec);
00677         CksvInitialize(UInt,_numInitNodeMsgs);
00678         CkpvInitialize(int,_charmEpoch);
00679         CkpvAccess(_charmEpoch)=0;
00680 
00681         CkpvInitialize(_CkOutStream*, _ckout);
00682         CkpvInitialize(_CkErrStream*, _ckerr);
00683         CkpvInitialize(Stats*, _myStats);
00684 
00685         CkpvAccess(_groupIDTable) = new GroupIDTable(0);
00686         CkpvAccess(_groupTable) = new GroupTable;
00687         CkpvAccess(_groupTable)->init();
00688         CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock();
00689         CkpvAccess(_numGroups) = 1; // make 0 an invalid group number
00690         CkpvAccess(_buffQ) = new PtrQ();
00691         CkpvAccess(_bocInitVec) = new PtrVec();
00692 
00693         CkpvAccess(_currentNodeGroupObj) = NULL;
00694 
00695         if(CkMyRank()==0)
00696         {
00697                 CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number
00698                 CksvAccess(_numInitNodeMsgs) = 0;
00699                 CksvAccess(_nodeLock) = CmiCreateLock();
00700                 CksvAccess(_nodeGroupTable) = new GroupTable();
00701                 CksvAccess(_nodeGroupTable)->init();
00702                 CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock();
00703                 CksvAccess(_nodeBocInitVec) = new PtrVec();
00704         }
00705 
00706         CmiNodeAllBarrier();
00707 
00708 #if ! CMK_BLUEGENE_CHARM
00709         initQd();
00710 #endif
00711 
00712         CkpvAccess(_coreState)=new CkCoreState();
00713 
00714         CkpvAccess(_numInitsRecd) = -1;  /*0;*/
00715 
00716         CkpvAccess(_ckout) = new _CkOutStream();
00717         CkpvAccess(_ckerr) = new _CkErrStream();
00718 
00719         _charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler);
00720         _initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
00721         _roHandlerIdx = CkRegisterHandler((CmiHandler)_roHandler);
00722         _roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
00723         _exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
00724         _bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
00725         _nodeBocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
00726         _infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn);
00727         _triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler);
00728         _ckModuleInit();
00729 
00730         CldRegisterEstimator((CldEstimator)_charmLoadEstimator);
00731 
00732         _futuresModuleInit(); // part of futures implementation is a converse module
00733         _loadbalancerInit();
00734         
00735         initCharmProjections();
00736 #if CMK_TRACE_IN_CHARM
00737         // initialize trace module in ck
00738         traceCharmInit(argv);
00739 #endif
00740         
00741         CkMessageWatcherInit(argv,CkpvAccess(_coreState));
00742         
00756 #ifdef __BLUEGENE__
00757         if(BgNodeRank()==0) 
00758 #else
00759         if(CkMyRank()==0)
00760 #endif
00761         {
00762                 CmiArgGroup("Charm++",NULL);
00763                 _parseCommandLineOpts(argv);
00764                 _registerInit();
00765                 CkRegisterMsg("System", 0, 0, sizeof(int));
00766                 CkRegisterChareInCharm(CkRegisterChare("null", 0));
00767                 CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare));
00768                 CkRegisterChareInCharm(CkIndex_Chare::__idx);
00769                 CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group));
00770         CkRegisterChareInCharm(CkIndex_Group::__idx);
00771                 CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC);
00772                 
00780                 _registerCkFutures();
00781                 _registerCkArray();
00782                 _registerLBDatabase();
00783                 _registerCkCallback();
00784                 _registertempo();
00785                 _registerwaitqd();
00786                 _registercharisma();
00787                 _registerCkCheckpoint();
00788 #if CMK_MEM_CHECKPOINT
00789                 _registerCkMemCheckpoint();
00790 #endif
00791                 
00802                 _registerExternalModules(argv);
00803                 
00809                 CkRegisterMainModule();
00810                 _registerDone();
00811         }
00812         CmiNodeAllBarrier();
00813 
00814         // Execute the initcalls registered in modules
00815         _initCallTable.enumerateInitCalls();
00816         
00817         //CmiNodeAllBarrier();
00818 
00819         CkpvAccess(_myStats) = new Stats();
00820         CkpvAccess(_msgPool) = new MsgPool();
00821 
00822         CmiNodeAllBarrier();
00823 
00824         if (!inCommThread) {
00825           _TRACE_BEGIN_COMPUTATION();
00826         }
00827 #ifndef __BLUEGENE__
00828         /*
00829                 FAULT_EVAC
00830         */
00831         CpvAccess(_validProcessors) = new char[CkNumPes()];
00832         for(int vProc=0;vProc<CkNumPes();vProc++){
00833                 CpvAccess(_validProcessors)[vProc]=1;
00834         }
00835         CpvAccess(startedEvac) = 0;
00836         _ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
00837         _ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
00838 #endif
00839         CpvAccess(serializer) = 0;
00840 
00841         evacuate = 0;
00842         CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
00843         if(_raiseEvac){
00844                 processRaiseEvacFile(_raiseEvacFile);
00845                 /*
00846                 if(CkMyPe() == 2){
00847                 //      CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
00848                         CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
00849                 }
00850                 if(CkMyPe() == 3){
00851                         CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
00852                 }*/
00853         }       
00854         
00855         if (faultFunc) {
00856                 if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
00857                 if (!inCommThread) {
00858                   CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
00859                   msg->argc = CmiGetArgc(argv);
00860                   msg->argv = argv;
00861                   faultFunc(_restartDir, msg);
00862                   CkFreeMsg(msg);
00863                 }
00864         }else if(CkMyPe()==0){
00865                 _allStats = new Stats*[CkNumPes()];
00866                 register int i, nMains=_mainTable.size();
00867                 for(i=0;i<nMains;i++)  /* Create all mainchares */
00868                 {
00869                         register int size = _chareTable[_mainTable[i]->chareIdx]->size;
00870                         register void *obj = malloc(size);
00871                         _MEMCHECK(obj);
00872                         _mainTable[i]->setObj(obj);
00873                         CkpvAccess(_currentChare) = obj;
00874                         CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx;
00875                         register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
00876                         msg->argc = CmiGetArgc(argv);
00877                         msg->argv = argv;
00878                         _entryTable[_mainTable[i]->entryIdx]->call(msg, obj);
00879                 }
00880                 _mainDone = 1;
00881 
00882                 _STATS_RECORD_CREATE_CHARE_N(nMains);
00883                 _STATS_RECORD_PROCESS_CHARE_N(nMains);
00884 
00885 
00886 
00887 
00888                 for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */
00889                 {
00890                         register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg));
00891                         if(roMsg==0)
00892                                 continue;
00893                         //Pack the message and send it to all other processors
00894                         register envelope *env = UsrToEnv(roMsg);
00895                         env->setSrcPe(CkMyPe());
00896                         env->setMsgtype(ROMsgMsg);
00897                         env->setRoIdx(i);
00898                         CmiSetHandler(env, _initHandlerIdx);
00899                         CkPackMessage(&env);
00900                         CmiSyncBroadcast(env->getTotalsize(), (char *)env);
00901                         CpvAccess(_qd)->create(CkNumPes()-1);
00902 
00903                         //For processor 0, unpack and re-set the global
00904                         CkUnpackMessage(&env);
00905                         _processROMsgMsg(env);
00906                         _numInitMsgs++;
00907                 }
00908 
00909                 //Determine the size of the RODataMessage
00910                 PUP::sizer ps;
00911                 for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
00912 
00913                 //Allocate and fill out the RODataMessage
00914                 envelope *env = _allocEnv(RODataMsg, ps.size());
00915                 PUP::toMem pp((char *)EnvToUsr(env));
00916                 for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
00917 
00918                 env->setCount(++_numInitMsgs);
00919                 env->setSrcPe(CkMyPe());
00920                 CmiSetHandler(env, _initHandlerIdx);
00921                 DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize()));
00922                 CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
00923                 CpvAccess(_qd)->create(CkNumPes()-1);
00924                 _initDone();
00925         }
00926 
00927         DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread));
00928         // when I am a communication thread, I don't participate initDone.
00929         if (inCommThread) {
00930                 CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
00931                                         CkpvAccess(_coreState));
00932                 CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler
00933 ,
00934                                         CkpvAccess(_coreState));
00935         }
00936 
00937 #if CMK_CCS_AVAILABLE
00938        if (CpvAccess(cmiArgDebugFlag))
00939        { 
00940           //CmiPrintf("In Parallel Debugging mode .....\n");
00941           CpdFreeze();
00942        }
00943 #endif
00944 
00945 }
00946 
00947 // this is needed because on o2k, f90 programs have to have main in
00948 // fortran90.
00949 extern "C" void fmain_(int *argc,char _argv[][80],int length[])
00950 {
00951   int i;
00952   char **argv = new char*[*argc+2];
00953 
00954   for(i=0;i <= *argc;i++) {
00955     if (length[i] < 100) {