Charm++: Charm Source Code Documentation

00001 
00065 
00066 #include "ck.h"
00067 #include "trace.h"
00068 
00069 void CkRestartMain(const char* dirname);
00070 
00071 #define  DEBUGF(x)     //CmiPrintf x;
00072 
00073 #ifdef CALCULATE_HOPS
00074 
00077 #include "TopoManager.h"
00078 
00079 TopoManager *tmgr = NULL;
00080 double hops = 0;
00081 
00082 extern "C" void calculateTotalHops(int pe1, int pe2, int size) {
00083   if(tmgr == NULL)
00084     tmgr = new TopoManager();
00085   hops += (tmgr->getHopsBetweenRanks(pe1, pe2) * size);
00086 }
00087 
00088 extern "C" void printTotalHops() {
00089   CmiPrintf("TOTAL HOPS %lf\n", hops/(1024*1024) );
00090 }
00091 #endif
00092 
00093 UChar _defaultQueueing = CK_QUEUEING_FIFO;
00094 
00095 UInt  _printCS = 0;
00096 UInt  _printSS = 0;
00097 
00105 UInt  _numExpectInitMsgs = 0;
00111 UInt  _numInitMsgs = 0;
00117 CksvDeclare(UInt,_numInitNodeMsgs);
00118 int   _infoIdx;
00119 int   _charmHandlerIdx;
00120 int   _initHandlerIdx;
00121 int   _roRestartHandlerIdx;
00122 int   _bocHandlerIdx;
00123 int   _qdHandlerIdx;
00124 int   _qdCommHandlerIdx;
00125 int   _triggerHandlerIdx;
00126 int   _mainDone = 0;
00127 CksvDeclare(int, _triggersSent);
00128 
00129 CkOutStream ckout;
00130 CkErrStream ckerr;
00131 CkInStream  ckin;
00132 
00133 CkpvDeclare(void*,       _currentChare);
00134 CkpvDeclare(int,         _currentChareType);
00135 CkpvDeclare(CkGroupID,   _currentGroup);
00136 CkpvDeclare(void*,       _currentNodeGroupObj);
00137 CkpvDeclare(CkGroupID,   _currentGroupRednMgr);
00138 CkpvDeclare(GroupTable*, _groupTable);
00139 CkpvDeclare(GroupIDTable*, _groupIDTable);
00140 CkpvDeclare(CmiImmediateLockType, _groupTableImmLock);
00141 CkpvDeclare(UInt, _numGroups);
00142 
00143 CkpvDeclare(CkCoreState *, _coreState);
00144 
00145 CksvDeclare(UInt, _numNodeGroups);
00146 CksvDeclare(GroupTable*, _nodeGroupTable);
00147 CksvDeclare(GroupIDTable, _nodeGroupIDTable);
00148 CksvDeclare(CmiImmediateLockType, _nodeGroupTableImmLock);
00149 CksvDeclare(CmiNodeLock, _nodeLock);
00150 CksvStaticDeclare(PtrVec*,_nodeBocInitVec);
00151 CkpvDeclare(int, _charmEpoch);
00152 
00153 CkpvDeclare(Stats*, _myStats);
00154 CkpvDeclare(MsgPool*, _msgPool);
00155 
00156 CkpvDeclare(_CkOutStream*, _ckout);
00157 CkpvDeclare(_CkErrStream*, _ckerr);
00158 
00159 CkpvStaticDeclare(int,  _numInitsRecd);
00160 CkpvStaticDeclare(PtrQ*, _buffQ);
00161 CkpvStaticDeclare(PtrVec*, _bocInitVec);
00162 
00163 //for interoperability
00164 extern void _libExitHandler(envelope *env);
00165 extern int _libExitHandlerIdx;
00166 
00167 /*
00168         FAULT_EVAC
00169 */
00170 CpvCExtern(char *, _validProcessors);
00171 CkpvDeclare(char ,startedEvac);
00172 
00173 int    _exitHandlerIdx;
00174 
00175 #if CMK_WITH_STATS
00176 static Stats** _allStats = 0;
00177 #endif
00178 static int   _numStatsRecd = 0;
00179 static int   _exitStarted = 0;
00180 
00181 static InitCallTable _initCallTable;
00182 
00183 #if CMK_WITH_STATS
00184 #define _STATS_ON(x) (x) = 1
00185 #else
00186 #define _STATS_ON(x) \
00187           if (CkMyPe()==0) CmiPrintf("stats unavailable in optimized version. ignoring...\n");
00188 #endif
00189 
00190 // fault tolerance
00191 typedef void (*CkFtFn)(const char *, CkArgMsg *);
00192 static CkFtFn  faultFunc = NULL;
00193 static char* _restartDir;
00194 
00195 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
00196 int teamSize=1;
00197 int chkptPeriod=1000;
00198 bool fastRecovery = false;
00199 int parallelRecovery = 1;
00200 extern int BUFFER_TIME; //time spent waiting for buffered messages
00201 #endif
00202 
00203 // flag for killing processes 
00204 extern int killFlag;
00205 // file specifying the processes to be killed
00206 extern char *killFile;
00207 // function for reading the kill file
00208 void readKillFile();
00209 
00210 
00211 int _defaultObjectQ = 0;            // for obejct queue
00212 int _ringexit = 0;                  // for charm exit
00213 int _ringtoken = 8;
00214 
00215 
00216 /*
00217         FAULT_EVAC
00218 
00219         flag which marks whether or not to trigger the processor shutdowns
00220 */
00221 static int _raiseEvac=0;
00222 static char *_raiseEvacFile;
00223 void processRaiseEvacFile(char *raiseEvacFile);
00224 
00225 extern bool useNodeBlkMapping;
00226 
00227 static inline void _parseCommandLineOpts(char **argv)
00228 {
00229   if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
00230       _STATS_ON(_printCS);
00231   if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown"))
00232       _STATS_ON(_printSS);
00233   if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing"))
00234       _defaultQueueing = CK_QUEUEING_FIFO;
00235   if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing"))
00236       _defaultQueueing = CK_QUEUEING_LIFO;
00237   if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing"))
00238       _defaultQueueing = CK_QUEUEING_IFIFO;
00239   if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing"))
00240       _defaultQueueing = CK_QUEUEING_ILIFO;
00241   if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing"))
00242       _defaultQueueing = CK_QUEUEING_BFIFO;
00243   if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing"))
00244       _defaultQueueing = CK_QUEUEING_BLIFO;
00245   if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct"))
00246   {
00247 #if CMK_OBJECT_QUEUE_AVAILABLE
00248       _defaultObjectQ = 1;
00249       if (CkMyPe()==0)
00250         CmiPrintf("Charm++> Create object queue for every Charm object.\n");
00251 #else
00252       CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1.");
00253 #endif
00254   }
00255   if(CmiGetArgString(argv,"+restart",&_restartDir))
00256       faultFunc = CkRestartMain;
00257 #if __FAULT__
00258   if (CmiGetArgIntDesc(argv,"+restartaftercrash",&CpvAccess(_curRestartPhase),"restarting this processor after a crash")){      
00259 # if CMK_MEM_CHECKPOINT
00260       faultFunc = CkMemRestart;
00261 # endif
00262 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
00263       faultFunc = CkMlogRestart;
00264 #endif
00265       CmiPrintf("[%d] Restarting after crash \n",CmiMyPe());
00266   }
00267   // reading the killFile
00268   if(CmiGetArgStringDesc(argv,"+killFile", &killFile,"Generates SIGKILL on specified processors")){
00269     if(faultFunc == NULL){
00270       //do not read the killfile if this is a restarting processor
00271       killFlag = 1;
00272       if(CmiMyPe() == 0){
00273         printf("[%d] killFlag set to 1 for file %s\n",CkMyPe(),killFile);
00274       }
00275     }
00276   }
00277 #endif
00278 
00279   // shut down program in ring fashion to allow projections output w/o IO error
00280   if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion")) 
00281   {
00282     _ringexit = 1;
00283     if (CkMyPe()==0)
00284       CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken);
00285     if (_ringtoken > CkNumPes())  _ringtoken = CkNumPes();
00286   }
00287         /*
00288                 FAULT_EVAC
00289 
00290                 if the argument +raiseevac is present then cause faults
00291         */
00292         if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){
00293                 _raiseEvac = 1;
00294         }
00295 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
00296         if(!CmiGetArgIntDesc(argv,"+teamSize",&teamSize,"Set the team size for message logging")){
00297         teamSize = 1;
00298     }
00299     if(!CmiGetArgIntDesc(argv,"+chkptPeriod",&chkptPeriod,"Set the checkpoint period for the message logging fault tolerance algorithm in seconds")){
00300         chkptPeriod = 100;
00301     }
00302         if(CmiGetArgIntDesc(argv,"+fastRecovery", &parallelRecovery, "Parallel recovery with message logging protocol")){
00303         fastRecovery = true;
00304     }
00305 #endif
00306 
00307 #if defined(_FAULT_MLOG_)
00308     if(!CmiGetArgIntDesc(argv,"+mlog_local_buffer",&_maxBufferedMessages,"# of local messages buffered in the message logging protoocl")){
00309         _maxBufferedMessages = 2;
00310     }
00311     if(!CmiGetArgIntDesc(argv,"+mlog_remote_buffer",&_maxBufferedTicketRequests,"# of remote ticket requests buffered in the message logging protoocl")){
00312         _maxBufferedTicketRequests = 2;
00313     }
00314     if(!CmiGetArgIntDesc(argv,"+mlog_buffer_time",&BUFFER_TIME,"# Time spent waiting for messages to be buffered in the message logging protoocl")){
00315         BUFFER_TIME = 2;
00316     }
00317 #endif  
00318         /* Anytime migration flag */
00319         _isAnytimeMigration = true;
00320         if (CmiGetArgFlagDesc(argv,"+noAnytimeMigration","The program does not require support for anytime migration")) {
00321           _isAnytimeMigration = false;
00322         }
00323         
00324         _isNotifyChildInRed = true;
00325         if (CmiGetArgFlagDesc(argv,"+noNotifyChildInReduction","The program has at least one element per processor for each charm array created")) {
00326           _isNotifyChildInRed = false;
00327         }
00328 
00329         _isStaticInsertion = false;
00330         if (CmiGetArgFlagDesc(argv,"+staticInsertion","Array elements are only inserted at construction")) {
00331           _isStaticInsertion = true;
00332         }
00333 
00334         useNodeBlkMapping = false;
00335         if (CmiGetArgFlagDesc(argv,"+useNodeBlkMapping","Array elements are block-mapped in SMP-node level")) {
00336           useNodeBlkMapping = true;
00337         }
00338 
00339 #if ! CMK_WITH_CONTROLPOINT
00340         // Display a warning if charm++ wasn't compiled with control point support but user is expecting it
00341         if( CmiGetArgFlag(argv,"+CPSamplePeriod") || 
00342             CmiGetArgFlag(argv,"+CPSamplePeriodMs") || 
00343             CmiGetArgFlag(argv,"+CPSchemeRandom") || 
00344             CmiGetArgFlag(argv,"+CPExhaustiveSearch") || 
00345             CmiGetArgFlag(argv,"+CPAlwaysUseDefaults") || 
00346             CmiGetArgFlag(argv,"+CPSimulAnneal") || 
00347             CmiGetArgFlag(argv,"+CPCriticalPathPrio") || 
00348             CmiGetArgFlag(argv,"+CPBestKnown") || 
00349             CmiGetArgFlag(argv,"+CPSteering") || 
00350             CmiGetArgFlag(argv,"+CPMemoryAware") || 
00351             CmiGetArgFlag(argv,"+CPSimplex") || 
00352             CmiGetArgFlag(argv,"+CPDivideConquer") || 
00353             CmiGetArgFlag(argv,"+CPLDBPeriod") || 
00354             CmiGetArgFlag(argv,"+CPLDBPeriodLinear") || 
00355             CmiGetArgFlag(argv,"+CPLDBPeriodQuadratic") || 
00356             CmiGetArgFlag(argv,"+CPLDBPeriodOptimal") || 
00357             CmiGetArgFlag(argv,"+CPDefaultValues") || 
00358             CmiGetArgFlag(argv,"+CPGatherAll") || 
00359             CmiGetArgFlag(argv,"+CPGatherMemoryUsage") || 
00360             CmiGetArgFlag(argv,"+CPGatherUtilization") || 
00361             CmiGetArgFlag(argv,"+CPSaveData") || 
00362             CmiGetArgFlag(argv,"+CPNoFilterData") || 
00363             CmiGetArgFlag(argv,"+CPLoadData") || 
00364             CmiGetArgFlag(argv,"+CPDataFilename")    )
00365           {         
00366             CkAbort("You specified a control point command line argument, but compiled charm++ without control point support.\n");
00367           }
00368 #endif
00369        
00370 }
00371 
00372 static void _bufferHandler(void *msg)
00373 {
00374   DEBUGF(("[%d] _bufferHandler called.\n", CkMyPe()));
00375   CkpvAccess(_buffQ)->enq(msg);
00376 }
00377 
00378 static void _discardHandler(envelope *env)
00379 {
00380 //  MESSAGE_PHASE_CHECK(env);
00381 
00382   DEBUGF(("[%d] _discardHandler called.\n", CkMyPe()));
00383 #if CMK_MEM_CHECKPOINT
00384   CkPrintf("[%d] _discardHandler called!\n", CkMyPe());
00385   if (CkInRestarting()) CpvAccess(_qd)->process();
00386 #endif
00387   CmiFree(env);
00388 }
00389 
00390 #if CMK_WITH_STATS
00391 static inline void _printStats(void)
00392 {
00393   DEBUGF(("[%d] _printStats\n", CkMyPe()));
00394   int i;
00395   if(_printSS || _printCS) {
00396     Stats *total = new Stats();
00397     _MEMCHECK(total);
00398     for(i=0;i<CkNumPes();i++)
00399       total->combine(_allStats[i]);
00400     CkPrintf("Charm Kernel Summary Statistics:\n");
00401     for(i=0;i<CkNumPes();i++) {
00402       CkPrintf("Proc %d: [%d created, %d processed]\n", i,
00403                _allStats[i]->getCharesCreated(),
00404                _allStats[i]->getCharesProcessed());
00405     }
00406     CkPrintf("Total Chares: [%d created, %d processed]\n",
00407              total->getCharesCreated(), total->getCharesProcessed());
00408   }
00409   if(_printCS) {
00410     CkPrintf("Charm Kernel Detailed Statistics (R=requested P=processed):\n\n");
00411 
00412     CkPrintf("         Create    Mesgs     Create    Mesgs     Create    Mesgs\n");
00413     CkPrintf("         Chare     for       Group     for       Nodegroup for\n");
00414     CkPrintf("PE   R/P Mesgs     Chares    Mesgs     Groups    Mesgs     Nodegroups\n");
00415     CkPrintf("---- --- --------- --------- --------- --------- --------- ----------\n");
00416 
00417     for(i=0;i<CkNumPes();i++) {
00418       CkPrintf("%4d  R  %9d %9d %9d %9d %9d %9d\n      P  %9d %9d %9d %9d %9d %9d\n",i,
00419                _allStats[i]->getCharesCreated(),
00420                _allStats[i]->getForCharesCreated(),
00421                _allStats[i]->getGroupsCreated(),
00422                _allStats[i]->getGroupMsgsCreated(),
00423                _allStats[i]->getNodeGroupsCreated(),
00424                _allStats[i]->getNodeGroupMsgsCreated(),
00425                _allStats[i]->getCharesProcessed(),
00426                _allStats[i]->getForCharesProcessed(),
00427                _allStats[i]->getGroupsProcessed(),
00428                _allStats[i]->getGroupMsgsProcessed(),
00429                _allStats[i]->getNodeGroupsProcessed(),
00430                _allStats[i]->getNodeGroupMsgsProcessed());
00431     }
00432   }
00433 }
00434 #else
00435 static inline void _printStats(void) {}
00436 #endif
00437 
00438 static inline void _sendStats(void)
00439 {
00440   DEBUGF(("[%d] _sendStats\n", CkMyPe()));
00441 #if CMK_WITH_STATS
00442   envelope *env = UsrToEnv(CkpvAccess(_myStats));
00443 #else
00444   envelope *env = _allocEnv(StatMsg);
00445 #endif
00446   env->setSrcPe(CkMyPe());
00447   CmiSetHandler(env, _exitHandlerIdx);
00448   CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
00449 }
00450 
00451 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
00452 extern void _messageLoggingExit();
00453 #endif
00454 
00455 #if __FAULT__
00456 //CpvExtern(int, CldHandlerIndex);
00457 //extern "C" void CldHandler(char *);
00458 extern int index_skipCldHandler;
00459 extern void _skipCldHandler(void *converseMsg);
00460 
00461 void _discard_charm_message()
00462 {
00463   CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00464 //  CkNumberHandler(CpvAccess(CldHandlerIndex), (CmiHandler)_discardHandler);
00465   CkNumberHandler(index_skipCldHandler, (CmiHandler)_discardHandler);
00466 }
00467 
00468 void _resume_charm_message()
00469 {
00470   CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler,
00471         CkpvAccess(_coreState));
00472 //  CkNumberHandler(CpvAccess(CldHandlerIndex), (CmiHandler)CldHandler);
00473   CkNumberHandler(index_skipCldHandler, (CmiHandler)_skipCldHandler);
00474 }
00475 #endif
00476 
00477 static void _exitHandler(envelope *env)
00478 {
00479   DEBUGF(("exitHandler called on %d msgtype: %d\n", CkMyPe(), env->getMsgtype()));
00480   switch(env->getMsgtype()) {
00481     case StartExitMsg:
00482       CkAssert(CkMyPe()==0);
00483       if (!_CkExitFnVec.isEmpty()) {
00484         CkExitFn fn = _CkExitFnVec.deq();
00485         fn();
00486         break;
00487       }
00488       // else goto next
00489     case ExitMsg:
00490       CkAssert(CkMyPe()==0);
00491       if(_exitStarted) {
00492         CmiFree(env);
00493         return;
00494       }
00495       _exitStarted = 1;
00496       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00497       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00498       env->setMsgtype(ReqStatMsg);
00499       env->setSrcPe(CkMyPe());
00500       // if exit in ring, instead of broadcasting, send in ring
00501       if (_ringexit){
00502         DEBUGF(("[%d] Ring Exit \n",CkMyPe()));
00503         const int stride = CkNumPes()/_ringtoken;
00504         int pe = 0;
00505         while (pe<CkNumPes()) {
00506           CmiSyncSend(pe, env->getTotalsize(), (char *)env);
00507           pe += stride;
00508         }
00509         CmiFree(env);
00510       }else{
00511         CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
00512       } 
00513       break;
00514     case ReqStatMsg:
00515 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
00516       _messageLoggingExit();
00517 #endif
00518       DEBUGF(("ReqStatMsg on %d\n", CkMyPe()));
00519       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00520       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00521       /*FAULT_EVAC*/
00522       if(CmiNodeAlive(CkMyPe())){
00523          _sendStats();
00524       _mainDone = 1; // This is needed because the destructors for
00525                      // readonly variables will be called when the program
00526                      // exits. If the destructor is called while _mainDone
00527                      // is 0, it will assume that the readonly variable was
00528                      // declared locally. On all processors other than 0, 
00529                      // _mainDone is never set to 1 before the program exits.
00530 #if CMK_TRACE_ENABLED
00531       if (_ringexit) traceClose();
00532 #endif
00533     }
00534       if (_ringexit) {
00535         int stride = CkNumPes()/_ringtoken;
00536         int pe = CkMyPe()+1;
00537         if (pe < CkNumPes() && pe % stride != 0)
00538           CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
00539         else
00540           CmiFree(env);
00541       }
00542       else
00543         CmiFree(env);
00544       //everyone exits here - there may be issues with leftover messages in the queue
00545       if(CkMyPe()){
00546         DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
00547         ConverseExit();
00548                                 if(CharmLibInterOperate)
00549                                         CpvAccess(charmLibExitFlag) = 1;
00550       }
00551       break;
00552     case StatMsg:
00553       CkAssert(CkMyPe()==0);
00554 #if CMK_WITH_STATS
00555       _allStats[env->getSrcPe()] = (Stats*) EnvToUsr(env);
00556 #endif
00557       _numStatsRecd++;
00558       DEBUGF(("StatMsg on %d with %d\n", CkMyPe(), _numStatsRecd));
00559                         /*FAULT_EVAC*/
00560       if(_numStatsRecd==CkNumValidPes()) {
00561         _printStats();
00562         DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
00563         ConverseExit();
00564                                 if(CharmLibInterOperate)
00565                                         CpvAccess(charmLibExitFlag) = 1;
00566       }
00567       break;
00568     default:
00569       CmiAbort("Internal Error(_exitHandler): Unknown-msg-type. Contact Developers.\n");
00570   }
00571 }
00572 
00578 static inline void _processBufferedBocInits(void)
00579 {
00580   CkCoreState *ck = CkpvAccess(_coreState);
00581   CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler, ck);
00582   register int i = 0;
00583   PtrVec &inits=*CkpvAccess(_bocInitVec);
00584   register int len = inits.size();
00585   for(i=1; i<len; i++) {
00586     envelope *env = inits[i];
00587     if(env==0) CkAbort("_processBufferedBocInits: empty message");
00588     if(env->isPacked())
00589       CkUnpackMessage(&env);
00590     _processBocInitMsg(ck,env);
00591   }
00592   delete &inits;
00593 }
00594 
00600 static inline void _processBufferedNodeBocInits(void)
00601 {
00602   CkCoreState *ck = CkpvAccess(_coreState);
00603   register int i = 0;
00604   PtrVec &inits=*CksvAccess(_nodeBocInitVec);
00605   register int len = inits.size();
00606   for(i=1; i<len; i++) {
00607     envelope *env = inits[i];
00608     if(env==0) CkAbort("_processBufferedNodeBocInits: empty message");
00609     if(env->isPacked())
00610       CkUnpackMessage(&env);
00611     _processNodeBocInitMsg(ck,env);
00612   }
00613   delete &inits;
00614 }
00615 
00616 static inline void _processBufferedMsgs(void)
00617 {
00618   CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler,
00619         CkpvAccess(_coreState));
00620   envelope *env;
00621   while(NULL!=(env=(envelope*)CkpvAccess(_buffQ)->deq())) {
00622     if(env->getMsgtype()==NewChareMsg || env->getMsgtype()==NewVChareMsg) {
00623       if(env->isForAnyPE())
00624         _CldEnqueue(CLD_ANYWHERE, env, _infoIdx);
00625       else
00626         _processHandler((void *)env, CkpvAccess(_coreState));
00627     } else {
00628       _processHandler((void *)env, CkpvAccess(_coreState));
00629     }
00630   }
00631 }
00632 
00633 static int _charmLoadEstimator(void)
00634 {
00635   return CkpvAccess(_buffQ)->length();
00636 }
00637 
00646 static void _sendTriggers(void)
00647 {
00648   int i, num, first;
00649   CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
00650   if (CksvAccess(_triggersSent) == 0)
00651   {
00652     CksvAccess(_triggersSent)++;
00653     num = CmiMyNodeSize();
00654     register envelope *env = _allocEnv(RODataMsg); // Notice that the type here is irrelevant
00655     env->setSrcPe(CkMyPe());
00656     CmiSetHandler(env, _triggerHandlerIdx);
00657     first = CmiNodeFirst(CmiMyNode());
00658     for (i=0; i < num; i++)
00659       if(first+i != CkMyPe())
00660         CmiSyncSend(first+i, env->getTotalsize(), (char *)env);
00661     CmiFree(env);
00662   }
00663   CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
00664 }
00665 
00675 void _initDone(void)
00676 {
00677   DEBUGF(("[%d] _initDone.\n", CkMyPe()));
00678   if (!CksvAccess(_triggersSent)) _sendTriggers();
00679   CkNumberHandler(_triggerHandlerIdx, (CmiHandler)_discardHandler);
00680   CmiNodeBarrier();
00681   if(CkMyRank() == 0) {
00682     _processBufferedNodeBocInits();
00683   }
00684   CmiNodeBarrier(); // wait for all nodegroups to be created
00685   _processBufferedBocInits();
00686   DEBUGF(("Reached CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
00687   CmiNodeBarrier();
00688   DEBUGF(("Crossed CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
00689   _processBufferedMsgs();
00690   CkpvAccess(_charmEpoch)=1;
00691 }
00692 
00700 static void _triggerHandler(envelope *env)
00701 {
00702   if (_numExpectInitMsgs && CkpvAccess(_numInitsRecd) + CksvAccess(_numInitNodeMsgs) == _numExpectInitMsgs)
00703   {
00704     DEBUGF(("Calling Init Done from _triggerHandler\n"));
00705     _initDone();
00706   }
00707   CmiFree(env);
00708 }
00709 
00710 static inline void _processROMsgMsg(envelope *env)
00711 {
00712   *((char **)(_readonlyMsgs[env->getRoIdx()]->pMsg))=(char *)EnvToUsr(env);
00713 }
00714 
00715 static inline void _processRODataMsg(envelope *env)
00716 {
00717   //Unpack each readonly:
00718   PUP::fromMem pu((char *)EnvToUsr(env));
00719   for(size_t i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pu);
00720   CmiFree(env);
00721 }
00722 
00729 static void _roRestartHandler(void *msg)
00730 {
00731   CkAssert(CkMyPe()!=0);
00732   register envelope *env = (envelope *) msg;
00733   CkpvAccess(_numInitsRecd)++;
00734   _numExpectInitMsgs = env->getCount();
00735   _processRODataMsg(env);
00736 }
00737 
00749 static void _initHandler(void *msg, CkCoreState *ck)
00750 {
00751   CkAssert(CkMyPe()!=0);
00752   register envelope *env = (envelope *) msg;
00753   
00754   if (ck->watcher!=NULL) {
00755     if (!ck->watcher->processMessage(&env,ck)) return;
00756   }
00757   
00758   switch (env->getMsgtype()) {
00759     case BocInitMsg:
00760       if (env->getGroupEpoch()==0) {
00761         CkpvAccess(_numInitsRecd)++;
00762         // _processBocInitMsg already handles QD
00763         //CpvAccess(_qd)->process();
00764         CkpvAccess(_bocInitVec)->insert(env->getGroupNum().idx, env);
00765       } else _bufferHandler(msg);
00766       break;
00767     case NodeBocInitMsg:
00768       if (env->getGroupEpoch()==0) {
00769         CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
00770         CksvAccess(_numInitNodeMsgs)++;
00771         CksvAccess(_nodeBocInitVec)->insert(env->getGroupNum().idx, env);
00772         CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
00773         CpvAccess(_qd)->process();
00774       } else _bufferHandler(msg);
00775       break;
00776     case ROMsgMsg:
00777       CkpvAccess(_numInitsRecd)++;
00778       CpvAccess(_qd)->process();
00779       if(env->isPacked()) CkUnpackMessage(&env);
00780       _processROMsgMsg(env);
00781       break;
00782     case RODataMsg:
00783       CkpvAccess(_numInitsRecd)++;
00784       CpvAccess(_qd)->process();
00785       _numExpectInitMsgs = env->getCount();
00786       _processRODataMsg(env);
00787       break;
00788     default:
00789       CmiAbort("Internal Error: Unknown-msg-type. Contact Developers.\n");
00790   }
00791         DEBUGF(("[%d,%.6lf] _numExpectInitMsgs %d CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs) %d+%d\n",CmiMyPe(),CmiWallTimer(),_numExpectInitMsgs,CkpvAccess(_numInitsRecd),CksvAccess(_numInitNodeMsgs)));
00792   if(_numExpectInitMsgs&&(CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs)==_numExpectInitMsgs)) {
00793     _initDone();
00794   }
00795 }
00796 
00797 #if 0
00798 /*****************************************
00799  *          no longer needed
00800  * ***************************************/
00801 extern "C"
00802 void _CkExit(void) 
00803 {
00804   CmiAssert(CkMyPe() == 0);
00805   // Shuts down Converse handlers for the upper layers on this processor
00806   //
00807   CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
00808   CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
00809   DEBUGF(("[%d] CkExit - _exitStarted:%d %d\n", CkMyPe(), _exitStarted, _exitHandlerIdx));
00810 
00811   if(CkMyPe()==0) {
00812     if(_exitStarted)
00813       CsdScheduler(-1);
00814     envelope *env = _allocEnv(ReqStatMsg);
00815     env->setSrcPe(CkMyPe());
00816     CmiSetHandler(env, _exitHandlerIdx);
00817                 /*FAULT_EVAC*/
00818     CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
00819   } else {
00820     envelope *env = _allocEnv(ExitMsg);
00821     env->setSrcPe(CkMyPe());
00822     CmiSetHandler(env, _exitHandlerIdx);
00823     CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
00824   }
00825 #if ! CMK_BIGSIM_THREAD
00826   _TRACE_END_EXECUTE();
00827   //Wait for stats, which will call ConverseExit when finished:
00828   CsdScheduler(-1);
00829 #endif
00830 }
00831 #endif
00832 
00833 CkQ<CkExitFn> _CkExitFnVec;
00834 
00835 // triger exit on PE 0,
00836 // which traverses _CkExitFnVec to call every registered user exit functions.
00837 // Every user exit functions should end with CkExit() to continue the chain
00838 extern "C"
00839 void CkExit(void)
00840 {
00841         /*FAULT_EVAC*/
00842   DEBUGF(("[%d] CkExit called \n",CkMyPe()));
00843     // always send to PE 0
00844   envelope *env = _allocEnv(StartExitMsg);
00845   env->setSrcPe(CkMyPe());
00846   CmiSetHandler(env, _exitHandlerIdx);
00847   CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
00848 
00849 #if ! CMK_BIGSIM_THREAD
00850   _TRACE_END_EXECUTE();
00851   //Wait for stats, which will call ConverseExit when finished:
00852         if(!CharmLibInterOperate)
00853   CsdScheduler(-1);
00854 #endif
00855 }
00856 
00857 /* This is a routine called in case the application is closing due to a signal.
00858    Tear down structures that must be cleaned up even when unclean exit happens.
00859    It is called by the machine layer whenever some problem occurs (it is thus up
00860    to the machine layer to call this function). */
00861 extern "C"
00862 void EmergencyExit(void) {
00863 #ifndef __BIGSIM__
00864   /* Delete _coreState to force any CkMessageWatcher to close down. */
00865   if (CkpvAccess(_coreState) != NULL) {
00866     delete CkpvAccess(_coreState);
00867     CkpvAccess(_coreState) = NULL;
00868   }
00869 #endif
00870 }
00871 
00872 static void _nullFn(void *, void *)
00873 {
00874   CmiAbort("Null-Method Called. Program may have Unregistered Module!!\n");
00875 }
00876 
00877 extern void _registerLBDatabase(void);
00878 extern void _registerPathHistory(void);
00879 #if CMK_WITH_CONTROLPOINT
00880 extern void _registerControlPoints(void);
00881 #endif
00882 extern void _registerTraceControlPoints();
00883 extern void _registerExternalModules(char **argv);
00884 extern void _ckModuleInit(void);
00885 extern void _loadbalancerInit();
00886 extern void _initChareTables();
00887 #if CMK_MEM_CHECKPOINT
00888 extern void init_memcheckpt(char **argv);
00889 #endif
00890 extern "C" void initCharmProjections();
00891 extern "C" void CmiInitCPUTopology(char **argv);
00892 extern "C" void CmiInitCPUAffinity(char **argv);
00893 extern "C" void CmiInitMemAffinity(char **argv);
00894 extern "C" void CmiInitPxshm(char **argv);
00895 
00896 extern void TopoManager_init();
00897 
00898 //extern "C" void CldCallback();
00899 
00900 void _registerInitCall(CkInitCallFn fn, int isNodeCall)
00901 {
00902   if (isNodeCall) _initCallTable.initNodeCalls.enq(fn);
00903   else _initCallTable.initProcCalls.enq(fn);
00904 }
00905 
00906 void InitCallTable::enumerateInitCalls()
00907 {
00908   int i;
00909 #ifdef __BIGSIM__
00910   if(BgNodeRank()==0)        // called only once on an emulating node
00911 #else
00912   if(CkMyRank()==0) 
00913 #endif
00914   {
00915     for (i=0; i<initNodeCalls.length(); i++) initNodeCalls[i]();
00916   }
00917   // initproc may depend on initnode calls.
00918   CmiNodeAllBarrier();
00919   for (i=0; i<initProcCalls.length(); i++) initProcCalls[i]();
00920 }
00921 
00922 CpvCExtern(int, cpdSuspendStartup);
00923 extern "C" void CpdFreeze(void);
00924 
00925 extern int _dummy_dq;
00926 
00927 extern "C" void initQd(char **argv)
00928 {
00929         CpvInitialize(QdState*, _qd);
00930         CpvAccess(_qd) = new QdState();
00931         if (CmiMyRank() == 0) {
00932 #if !defined(CMK_CPV_IS_SMP) && !CMK_SHARED_VARS_UNIPROCESSOR
00933         CpvAccessOther(_qd, 1) = new QdState(); // for i/o interrupt
00934 #endif
00935         }
00936         _qdHandlerIdx = CmiRegisterHandler((CmiHandler)_qdHandler);
00937         _qdCommHandlerIdx = CmiRegisterHandler((CmiHandler)_qdCommHandler);
00938         if (CmiGetArgIntDesc(argv,"+qd",&_dummy_dq, "QD time in seconds")) {
00939           if (CmiMyPe()==0)
00940             CmiPrintf("Charm++> Fake QD using %d seconds.\n", _dummy_dq);
00941         }
00942 }
00943 
00944 #if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG
00945 void CpdBgInit();
00946 #endif
00947 void CpdBreakPointInit();
00948 
00958 void _initCharm(int unused_argc, char **argv)
00959 { 
00960         int inCommThread = (CmiMyRank() == CmiMyNodeSize());
00961 
00962         DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer()));
00963 
00964         CkpvInitialize(size_t *, _offsets);
00965         CkpvAccess(_offsets) = new size_t[32];
00966         CkpvInitialize(PtrQ*,_buffQ);
00967         CkpvInitialize(PtrVec*,_bocInitVec);
00968         CkpvInitialize(void*, _currentChare);
00969         CkpvInitialize(int,   _currentChareType);
00970         CkpvInitialize(CkGroupID, _currentGroup);
00971         CkpvInitialize(void *, _currentNodeGroupObj);
00972         CkpvInitialize(CkGroupID, _currentGroupRednMgr);
00973         CkpvInitialize(GroupTable*, _groupTable);
00974         CkpvInitialize(GroupIDTable*, _groupIDTable);
00975         CkpvInitialize(CmiImmediateLockType, _groupTableImmLock);
00976         CkpvInitialize(UInt, _numGroups);
00977         CkpvInitialize(int, _numInitsRecd);
00978         CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
00979         CkpvInitialize(MsgPool*, _msgPool);
00980         CkpvInitialize(CkCoreState *, _coreState);
00981         /*
00982                 Added for evacuation-sayantan
00983         */
00984 #ifndef __BIGSIM__
00985         CpvInitialize(char *,_validProcessors);
00986 #endif
00987         CkpvInitialize(char ,startedEvac);
00988         CpvInitialize(int,serializer);
00989 
00990         _initChareTables();            // for checkpointable plain chares
00991 
00992         CksvInitialize(UInt, _numNodeGroups);
00993         CksvInitialize(GroupTable*, _nodeGroupTable);
00994         CksvInitialize(GroupIDTable, _nodeGroupIDTable);
00995         CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock);
00996         CksvInitialize(CmiNodeLock, _nodeLock);
00997         CksvInitialize(PtrVec*,_nodeBocInitVec);
00998         CksvInitialize(UInt,_numInitNodeMsgs);
00999         CkpvInitialize(int,_charmEpoch);
01000         CkpvAccess(_charmEpoch)=0;
01001         CksvInitialize(int, _triggersSent);
01002         CksvAccess(_triggersSent) = 0;
01003 
01004         CkpvInitialize(_CkOutStream*, _ckout);
01005         CkpvInitialize(_CkErrStream*, _ckerr);
01006         CkpvInitialize(Stats*, _myStats);
01007 
01008         CkpvAccess(_groupIDTable) = new GroupIDTable(0);
01009         CkpvAccess(_groupTable) = new GroupTable;
01010         CkpvAccess(_groupTable)->init();
01011         CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock();
01012         CkpvAccess(_numGroups) = 1; // make 0 an invalid group number
01013         CkpvAccess(_buffQ) = new PtrQ();
01014         CkpvAccess(_bocInitVec) = new PtrVec();
01015 
01016         CkpvAccess(_currentNodeGroupObj) = NULL;
01017 
01018         if(CkMyRank()==0)
01019         {
01020                 CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number
01021                 CksvAccess(_numInitNodeMsgs) = 0;
01022                 CksvAccess(_nodeLock) = CmiCreateLock();
01023                 CksvAccess(_nodeGroupTable) = new GroupTable();
01024                 CksvAccess(_nodeGroupTable)->init();
01025                 CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock();
01026                 CksvAccess(_nodeBocInitVec) = new PtrVec();
01027         }
01028 
01029         CkCallbackInit();
01030         
01031         CmiNodeAllBarrier();
01032 
01033 #if ! CMK_BIGSIM_CHARM
01034         initQd(argv);         // bigsim calls it in ConverseCommonInit
01035 #endif
01036 
01037         CkpvAccess(_coreState)=new CkCoreState();
01038 
01039         CkpvAccess(_numInitsRecd) = 0;
01040 
01041         CkpvAccess(_ckout) = new _CkOutStream();
01042         CkpvAccess(_ckerr) = new _CkErrStream();
01043 
01044         _charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler);
01045         _initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
01046         CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
01047         _roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
01048         _exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
01049         //added for interoperabilitY
01050         _libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler);
01051         _bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
01052         CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
01053 
01054 #ifdef __BIGSIM__
01055         if(BgNodeRank()==0) 
01056 #endif
01057         _infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn);
01058 
01059         _triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler);
01060         _ckModuleInit();
01061 
01062         CldRegisterEstimator((CldEstimator)_charmLoadEstimator);
01063 
01064         _futuresModuleInit(); // part of futures implementation is a converse module
01065         _loadbalancerInit();
01066         
01067 #if CMK_MEM_CHECKPOINT
01068         init_memcheckpt(argv);
01069 #endif
01070 
01071         initCharmProjections();
01072 #if CMK_TRACE_IN_CHARM
01073         // initialize trace module in ck
01074         traceCharmInit(argv);
01075 #endif
01076         
01077     CkpvInitialize(int, envelopeEventID);
01078     CkpvAccess(envelopeEventID) = 0;
01079         CkMessageWatcherInit(argv,CkpvAccess(_coreState));
01080         
01094 #ifdef __BIGSIM__
01095         if(BgNodeRank()==0) 
01096 #else
01097         if(CkMyRank()==0)
01098 #endif
01099         {
01100                 CmiArgGroup("Charm++",NULL);
01101                 _parseCommandLineOpts(argv);
01102                 _registerInit();
01103                 CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int));
01104                 CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare));
01105                 CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare);
01106                 CkRegisterChareInCharm(CkIndex_Chare::__idx);
01107                 CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup);
01108                 CkRegisterChareInCharm(CkIndex_Group::__idx);
01109                 CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC);
01110                 
01118                 _registerCkFutures();
01119                 _registerCkArray();
01120                 _registerLBDatabase();
01121                 _registerCkCallback();
01122                 _registertempo();
01123                 _registerwaitqd();
01124                 _registercharisma();
01125                 _registerCkCheckpoint();
01126 #if CMK_MEM_CHECKPOINT
01127                 _registerCkMemCheckpoint();
01128 #endif
01129 
01130 
01131                 /*
01132                   Setup Control Point Automatic Tuning Framework.
01133 
01134                   By default it is enabled as a part of charm, 
01135                   however it won't enable its tracing module 
01136                   unless a +CPEnableMeasurements command line argument
01137                   is specified. See trace-common.C for more info
01138 
01139                   Thus there should be no noticable overhead to 
01140                   always having the control point framework linked
01141                   in.
01142                   
01143                 */
01144 #if CMK_WITH_CONTROLPOINT
01145                 _registerPathHistory();
01146                 _registerControlPoints();
01147                 _registerTraceControlPoints();
01148 #endif
01149 
01150 
01156                 CkRegisterMainModule();
01157 
01168                 _registerExternalModules(argv);
01169                 
01170                 _registerDone();
01171         }
01172         /* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */
01173         if (CkMyRank() == 0) {
01174           CpdBreakPointInit();
01175         }
01176         CmiNodeAllBarrier();
01177 
01178         // Execute the initcalls registered in modules
01179         _initCallTable.enumerateInitCalls();
01180 
01181 #if CMK_CHARMDEBUG
01182         CpdFinishInitialization();
01183 #endif
01184 
01185         //CmiNodeAllBarrier();
01186 
01187         CkpvAccess(_myStats) = new Stats();
01188         CkpvAccess(_msgPool) = new MsgPool();
01189 
01190         CmiNodeAllBarrier();
01191 
01192 #if ! CMK_MEM_CHECKPOINT && !_FAULT_MLOG_ && !_FAULT_CAUSAL_
01193         CmiBarrier();
01194         CmiBarrier();
01195         CmiBarrier();
01196 #endif
01197 #if CMK_SMP_TRACE_COMMTHREAD
01198         _TRACE_BEGIN_COMPUTATION();     
01199 #else
01200         if (!inCommThread) {
01201           _TRACE_BEGIN_COMPUTATION();
01202         }
01203 #endif
01204 
01205 #ifdef ADAPT_SCHED_MEM
01206     if(CkMyRank()==0){
01207         memCriticalEntries = new int[numMemCriticalEntries];
01208         int memcnt=0;
01209         for(int i=0; i<_entryTable.size(); i++){
01210             if(_entryTable[i]->isMemCritical){
01211                 memCriticalEntries[memcnt++] = i;
01212             }
01213         }
01214     }
01215 #endif
01216 
01217 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
01218     _messageLoggingInit();
01219 #endif
01220 
01221 #ifndef __BIGSIM__
01222         /*
01223                 FAULT_EVAC
01224         */
01225         CpvAccess(_validProcessors) = new char[CkNumPes()];
01226         for(int vProc=0;vProc<CkNumPes();vProc++){
01227                 CpvAccess(_validProcessors)[vProc]=1;
01228         }
01229         _ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
01230         _ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
01231 #endif
01232         CkpvAccess(startedEvac) = 0;
01233         CpvAccess(serializer) = 0;
01234 
01235         evacuate = 0;
01236         CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
01237 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) 
01238     CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0);
01239 #endif
01240 
01241         if(_raiseEvac){
01242                 processRaiseEvacFile(_raiseEvacFile);
01243                 /*
01244                 if(CkMyPe() == 2){
01245                 //      CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
01246                         CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
01247                 }
01248                 if(CkMyPe() == 3){
01249                         CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
01250                 }*/
01251         }       
01252 
01253     TopoManager_init();
01254     CmiNodeAllBarrier();
01255 
01256     if (!_replaySystem) {
01257         if (faultFunc == NULL) {         // this is not restart
01258             // these two are blocking calls for non-bigsim
01259 #if ! CMK_BIGSIM_CHARM
01260           CmiInitCPUAffinity(argv);
01261           CmiInitMemAffinity(argv);
01262 #endif
01263         }
01264         CmiInitCPUTopology(argv);
01265 #if CMK_SHARED_VARS_POSIX_THREADS_SMP
01266         if (CmiCpuTopologyEnabled()) {
01267             int *pelist;
01268             int num;
01269             CmiGetPesOnPhysicalNode(0, &pelist, &num);
01270             if (CkMyPe()==0 && !_Cmi_noprocforcommthread && num+num/CmiMyNodeSize() > CmiNumCores()) {
01271                 CkPrintf("\nCharm++> Warning: the number of SMP threads is greater than the number of physical cores, use +CmiNoProcForComThread runtime option.\n\n");
01272             }
01273         }
01274 #endif
01275     }
01276 
01277     if(CmiMyPe() == 0) {
01278         char *topoFilename;
01279         if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name")) 
01280         {
01281             TopoManager tmgr;
01282             FILE *fp;
01283             fp = fopen(topoFilename, "w");
01284             if (fp == NULL) {
01285               CkPrintf("Error opening topology.Info.txt file, writing to stdout\n");
01286               fp = stdout;
01287             }
01288             tmgr.printAllocation(fp);
01289             fclose(fp);
01290         }
01291     }
01292 
01293 #if CMK_USE_PXSHM && CMK_CRAYXE && CMK_SMP
01294       // for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized
01295       // again after cpuaffinity is done
01296     if (CkMyRank() == 0) {
01297       CmiInitPxshm(argv);
01298     }
01299     CmiNodeAllBarrier();
01300 #endif
01301 
01302     //CldCallback();
01303 #if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG
01304       // Register the BG handler for CCS. Notice that this is put into a variable shared by
01305       // the whole real processor. This because converse needs to find it. We check that all
01306       // virtual processors register the same index for this handler.
01307     CpdBgInit();
01308 #endif
01309 
01310         if (faultFunc) {
01311 #if CMK_WITH_STATS
01312                 if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
01313 #endif
01314                 if (!inCommThread) {
01315                   CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
01316                   msg->argc = CmiGetArgc(argv);
01317                   msg->argv = argv;
01318                   faultFunc(_restartDir, msg);
01319                   CkFreeMsg(msg);
01320                 }
01321         }else if(CkMyPe()==0){
01322 #if CMK_WITH_STATS
01323                 _allStats = new Stats*[CkNumPes()];
01324 #endif
01325                 register size_t i, nMains=_mainTable.size();
01326                 for(i=0;i<nMains;i++)  /* Create all mainchares */
01327                 {
01328                         register int size = _chareTable[_mainTable[i]->chareIdx]->size;
01329                         register void *obj = malloc(size);
01330                         _MEMCHECK(obj);
01331                         _mainTable[i]->setObj(obj);
01332                         CkpvAccess(_currentChare) = obj;
01333                         CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx;
01334                         register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
01335                         msg->argc = CmiGetArgc(argv);
01336                         msg->argv = argv;
01337                         _entryTable[_mainTable[i]->entryIdx]->call(msg, obj);
01338 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
01339             CpvAccess(_currentObj) = (Chare *)obj;
01340 #endif
01341                 }
01342                 _mainDone = 1;
01343 
01344                 _STATS_RECORD_CREATE_CHARE_N(nMains);
01345                 _STATS_RECORD_PROCESS_CHARE_N(nMains);
01346 
01347 
01348 
01349 
01350                 for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */
01351                 {
01352                         register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg));
01353                         if(roMsg==0)
01354                                 continue;
01355                         //Pack the message and send it to all other processors
01356                         register envelope *env = UsrToEnv(roMsg);
01357                         env->setSrcPe(CkMyPe());
01358                         env->setMsgtype(ROMsgMsg);
01359                         env->setRoIdx(i);
01360                         CmiSetHandler(env, _initHandlerIdx);
01361                         CkPackMessage(&env);
01362                         CmiSyncBroadcast(env->getTotalsize(), (char *)env);
01363                         CpvAccess(_qd)->create(CkNumPes()-1);
01364 
01365                         //For processor 0, unpack and re-set the global
01366                         CkUnpackMessage(&env);
01367                         _processROMsgMsg(env);
01368                         _numInitMsgs++;
01369                 }
01370 
01371                 //Determine the size of the RODataMessage
01372                 PUP::sizer ps;
01373                 for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
01374 
01375                 //Allocate and fill out the RODataMessage
01376                 envelope *env = _allocEnv(RODataMsg, ps.size());
01377                 PUP::toMem pp((char *)EnvToUsr(env));
01378                 for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
01379 
01380                 env->setCount(++_numInitMsgs);
01381                 env->setSrcPe(CkMyPe());
01382                 CmiSetHandler(env, _initHandlerIdx);
01383                 DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize()));
01384                 CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
01385                 CpvAccess(_qd)->create(CkNumPes()-1);
01386                 _initDone();
01387         }
01388 
01389         DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread));
01390         // when I am a communication thread, I don't participate initDone.
01391         if (inCommThread) {
01392                 CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
01393                                         CkpvAccess(_coreState));
01394                 CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler
01395 ,
01396                                         CkpvAccess(_coreState));
01397         }
01398 
01399 #if CMK_CHARMDEBUG
01400         // Should not use CpdFreeze inside a thread (since this processor is really a user-level thread)
01401        if (CpvAccess(cpdSuspendStartup))
01402        { 
01403           //CmiPrintf("In Parallel Debugging mode .....\n");
01404           CpdFreeze();
01405        }
01406 #endif
01407 
01408 
01409 #if __FAULT__
01410         if(killFlag){                                                  
01411                 readKillFile();                                        
01412         }
01413 #endif
01414 
01415 }
01416 
01417 // this is needed because on o2k, f90 programs have to have main in
01418 // fortran90.
01419 extern "C" void fmain_(int *argc,char _argv[][80],int length[])
01420 {
01421   int i;
01422   char **argv = new char*[*argc+2];
01423 
01424   for(i=0;i <= *argc;i++) {
01425     if (length[i] < 100) {
01426       _argv[i][length[i]]='\0';
01427       argv[i] = &(_argv[i][0]);
01428     } else {
01429       argv[i][0] = '\0';
01430     }
01431   }
01432   argv[*argc+1]=0;
01433 
01434   ConverseInit(*argc, argv, (CmiStartFn) _initCharm, 0, 0);
01435 }
01436 
01437 // user callable function to register an exit function, this function
01438 // will perform task of collecting of info from all pes to pe0, and call
01439 // CkExit() on pe0 again to recursively traverse the registered exitFn.
01440 // see trace-summary for an example.
01441 void registerExitFn(CkExitFn fn)
01442 {
01443   _CkExitFnVec.enq(fn);
01444 }
01445
ck-core/init.C
