Charm++: Charm Source Code Documentation

00001 
00005 
00006 #include <sys/stat.h>
00007 #include <sys/types.h>
00008 
00009 #include "charm.h"
00010 #include "middle.h"
00011 #include "cklists.h"
00012 #include "ckliststring.h"
00013 
00014 #include "trace.h"
00015 #include "trace-common.h"
00016 #include "allEvents.h"          //projector
00017 #include "register.h" // for _entryTable
00018 
00019 CpvExtern(int, _traceCoreOn);   // projector
00020 
00021 #if ! CMK_TRACE_ENABLED
00022 static int warned = 0;
00023 #define OPTIMIZE_WARNING if (!warned) { warned=1;  CmiPrintf("\n\n!!!! Warning: tracing not available without CMK_TRACE_ENABLED!\n");  return;  }
00024 #else
00025 #define OPTIMIZE_WARNING /*empty*/
00026 #endif
00027 
00028 #define DEBUGF(x)          // CmiPrintf x
00029 
00030 CkpvDeclare(TraceArray*, _traces);      // lists of all trace modules
00031 
00032 /* trace for bluegene */
00033 class TraceBluegene;
00034 CkpvDeclare(TraceBluegene*, _tracebg);
00035 int traceBluegeneLinked=0;          // if trace-bluegene is linked
00036 
00037 CkpvDeclare(bool,   dumpData);
00038 CkpvDeclare(double, traceInitTime);
00039 CkpvDeclare(double, traceInitCpuTime);
00040 CpvDeclare(int, traceOn);
00041 CkpvDeclare(int, traceOnPe);
00042 CkpvDeclare(char*, traceRoot);
00043 CkpvDeclare(char*, partitionRoot);
00044 CkpvDeclare(int, traceRootBaseLength);
00045 CkpvDeclare(char*, selective);
00046 CkpvDeclare(bool, verbose);
00047 bool outlierAutomatic;
00048 bool findOutliers;
00049 int numKSeeds;
00050 int peNumKeep;
00051 bool outlierUsePhases;
00052 double entryThreshold;
00053 
00054 typedef void (*mTFP)();                   // function pointer for
00055 CpvStaticDeclare(mTFP, machineTraceFuncPtr);    // machine user event
00056                                           // registration
00057 
00058 int _threadMsg, _threadChare, _threadEP;
00059 int _packMsg, _packChare, _packEP;
00060 int _unpackMsg, _unpackChare, _unpackEP;
00061 int _sdagMsg, _sdagChare, _sdagEP;
00062 
00063 CtvDeclare(int, curThreadEvent);
00064 CpvDeclare(int, curPeEvent);
00065 
00066 #if CMK_BIGSIM_CHARM
00067 double TraceTimerCommon(){return TRACE_TIMER();}
00068 #else
00069 double TraceTimerCommon(){return TRACE_TIMER() - CkpvAccess(traceInitTime);}
00070 #endif
00071 #if CMK_TRACE_ENABLED
00072 void CthSetEventInfo(CthThread t, int event, int srcPE);
00073 #endif
00075 static void traceCommonInit(char **argv)
00076 {
00077   CmiArgGroup("Charm++","Tracing");
00078   DEBUGF(("[%d] in traceCommonInit.\n", CkMyPe()));
00079   CkpvInitialize(double, traceInitTime);
00080   CkpvAccess(traceInitTime) = CmiStartTimer();
00081   CkpvInitialize(bool, dumpData);
00082   CkpvAccess(dumpData) = true;
00083   CkpvInitialize(double, traceInitCpuTime);
00084   CkpvAccess(traceInitCpuTime) = TRACE_CPUTIMER();
00085   CpvInitialize(int, traceOn);
00086   CpvAccess(traceOn) = 0;
00087   CpvInitialize(int, _traceCoreOn); //projector
00088   CpvAccess(_traceCoreOn)=0; //projector
00089   CpvInitialize(mTFP, machineTraceFuncPtr);
00090   CpvAccess(machineTraceFuncPtr) = NULL;
00091   CkpvInitialize(int, traceOnPe);
00092   CkpvAccess(traceOnPe) = 1;
00093   CkpvInitialize(bool, verbose);
00094   if (CmiGetArgFlag(argv, "+traceWarn")) {
00095     CkpvAccess(verbose) = true;
00096   } else {
00097     CkpvAccess(verbose) = false;
00098   }
00099 
00100   char *root=NULL;
00101   char *temproot;
00102   char *temproot2;
00103   CkpvInitialize(char*, traceRoot);
00104   CkpvInitialize(char*, partitionRoot);
00105   CkpvInitialize(int, traceRootBaseLength);
00106   /* Ctv variable to store Cthread Local event ID for Cthread tracing */
00107   CtvInitialize(int,curThreadEvent);
00108   CtvAccess(curThreadEvent)=0;
00109   /* Cpv variable to store current PE event ID for [local] and [inline] method tracing */
00110   CpvInitialize(int, curPeEvent);
00111   CpvAccess(curPeEvent)=0;
00112 
00113   char subdir[20];
00114   if(CmiNumPartitions() > 1) {
00115     sprintf(subdir, "prj.part%d%s", CmiMyPartition(), PATHSEPSTR);
00116   } else {
00117     subdir[0]='\0';
00118   }
00119 
00120   if (CmiGetArgStringDesc(argv, "+traceroot", &temproot, "Directory to write trace files to")) {
00121     int i;
00122     // Trying to decide if the traceroot path is absolute or not. If it is not
00123     // then create an absolute pathname for it.
00124     if (temproot[0] != PATHSEP) {
00125       temproot2 = GETCWD(NULL,0);
00126       root = (char *)malloc(strlen(temproot2)+1+strlen(temproot)+1);
00127       strcpy(root, temproot2);
00128       strcat(root, PATHSEPSTR);
00129       strcat(root, temproot);
00130     } else {
00131       root = (char *)malloc(strlen(temproot)+1);
00132       strcpy(root,temproot);
00133     }
00134     for (i=strlen(argv[0])-1; i>=0; i--) if (argv[0][i] == PATHSEP) break;
00135     i++;
00136     CkpvAccess(traceRootBaseLength) = strlen(root)+1;
00137     CkpvAccess(traceRoot) = (char *)malloc(strlen(argv[0]+i) + strlen(root) + 2 +strlen(subdir));    _MEMCHECK(CkpvAccess(traceRoot));
00138     CkpvAccess(partitionRoot) = (char *)malloc(strlen(argv[0]+i) + strlen(root) + 2 +strlen(subdir));    _MEMCHECK(CkpvAccess(partitionRoot));
00139     strcpy(CkpvAccess(traceRoot), root);
00140     strcat(CkpvAccess(traceRoot), PATHSEPSTR);
00141     strcat(CkpvAccess(traceRoot), subdir);
00142     strcpy(CkpvAccess(partitionRoot),CkpvAccess(traceRoot));
00143     strcat(CkpvAccess(traceRoot), argv[0]+i);
00144   }
00145   else {
00146     CkpvAccess(traceRoot) = (char *) malloc(strlen(argv[0])+1 +strlen(subdir));
00147     _MEMCHECK(CkpvAccess(traceRoot));
00148     CkpvAccess(partitionRoot) = (char *) malloc(strlen(argv[0])+1 +strlen(subdir));
00149     _MEMCHECK(CkpvAccess(partitionRoot));
00150     strcpy(CkpvAccess(traceRoot), subdir);
00151     strcpy(CkpvAccess(partitionRoot),CkpvAccess(traceRoot));
00152     strcat(CkpvAccess(traceRoot), argv[0]);
00153   }
00154   CkpvAccess(traceRootBaseLength)  +=  strlen(subdir);
00155     /* added for TAU trace module. */
00156   char *cwd;
00157   CkpvInitialize(char*, selective);
00158   if (CmiGetArgStringDesc(argv, "+selective", &temproot, "TAU's selective instrumentation file")) {
00159     // Trying to decide if the traceroot path is absolute or not. If it is not
00160     // then create an absolute pathname for it.
00161     if (temproot[0] != PATHSEP) {
00162       cwd = GETCWD(NULL,0);
00163       root = (char *)malloc(strlen(cwd)+strlen(temproot)+2);
00164       strcpy(root, cwd);
00165       strcat(root, PATHSEPSTR);
00166       strcat(root, temproot);
00167     } else {
00168       root = (char *)malloc(strlen(temproot)+1);
00169       strcpy(root,temproot);
00170     }
00171     CkpvAccess(selective) = (char *) malloc(strlen(root)+1);
00172     _MEMCHECK(CkpvAccess(selective));
00173     strcpy(CkpvAccess(selective), root);
00174     if (CkMyPe() == 0) 
00175       CmiPrintf("Trace: selective: %s\n", CkpvAccess(selective));
00176   }
00177   else {
00178     CkpvAccess(selective) = (char *) malloc(3);
00179     _MEMCHECK(CkpvAccess(selective));
00180     strcpy(CkpvAccess(selective), "");
00181   }
00182 
00183   outlierAutomatic = true;
00184   findOutliers = false;
00185   numKSeeds = 10;
00186   peNumKeep = CkNumPes();
00187   outlierUsePhases = false;
00188   entryThreshold = 0.0;
00189   //For KMeans
00190   if (outlierAutomatic) {
00191     CmiGetArgIntDesc(argv, "+outlierNumSeeds", &numKSeeds,
00192              "Number of cluster seeds to apply at outlier analysis.");
00193     CmiGetArgIntDesc(argv, "+outlierPeNumKeep", 
00194              &peNumKeep, "Number of Processors to retain data");
00195     CmiGetArgDoubleDesc(argv, "+outlierEpThresh", &entryThreshold,
00196             "Minimum significance of entry points to be considered for clustering (%).");
00197     findOutliers =
00198       CmiGetArgFlagDesc(argv,"+outlier", "Find Outliers.");
00199     outlierUsePhases = 
00200       CmiGetArgFlagDesc(argv,"+outlierUsePhases",
00201             "Apply automatic outlier analysis to any available phases.");
00202     if (outlierUsePhases) {
00203       // if the user wants to use an outlier feature, it is assumed outlier
00204       //    analysis is desired.
00205       findOutliers = true;
00206     }
00207     if(root)
00208         free(root);
00209   }
00210 
00211   
00212   
00213 #ifdef __BIGSIM__
00214   if(BgNodeRank()==0) {
00215 #else
00216   if(CkMyRank()==0) {
00217 #endif
00218     _threadMsg = CkRegisterMsg("dummy_thread_msg", 0, 0, 0, 0);
00219     _threadChare = CkRegisterChare("dummy_thread_chare", 0, TypeInvalid);
00220     CkRegisterChareInCharm(_threadChare);
00221     _threadEP = CkRegisterEp("dummy_thread_ep", 0, _threadMsg,_threadChare, 0+CK_EP_INTRINSIC);
00222 
00223     _packMsg = CkRegisterMsg("dummy_pack_msg", 0, 0, 0, 0);
00224     _packChare = CkRegisterChare("dummy_pack_chare", 0, TypeInvalid);
00225     CkRegisterChareInCharm(_packChare);
00226     _packEP = CkRegisterEp("dummy_pack_ep", 0, _packMsg,_packChare, 0+CK_EP_INTRINSIC);
00227 
00228     _unpackMsg = CkRegisterMsg("dummy_unpack_msg", 0, 0, 0, 0);
00229     _unpackChare = CkRegisterChare("dummy_unpack_chare", 0, TypeInvalid);
00230     CkRegisterChareInCharm(_unpackChare);
00231     _unpackEP = CkRegisterEp("dummy_unpack_ep", 0, _unpackMsg,_unpackChare, 0+CK_EP_INTRINSIC);
00232 
00233     _sdagMsg = CkRegisterMsg("sdag_msg", 0, 0, 0, 0);
00234     _sdagChare = CkRegisterChare("SDAG", 0, TypeInvalid);
00235     CkRegisterChareInCharm(_sdagChare);
00236     _sdagEP = CkRegisterEp("SDAG_RTS", 0, _sdagMsg, _sdagChare, 0+CK_EP_INTRINSIC);
00237   }
00238 }
00239 
00241 void traceWriteSTS(FILE *stsfp,int nUserEvents) {
00242   fprintf(stsfp, "MACHINE \"%s\"\n",CMK_MACHINE_NAME);
00243 #if CMK_SMP_TRACE_COMMTHREAD
00244   //Assuming there's only 1 comm thread now! --Chao Mei
00245   //considering the extra comm thread per node
00246   fprintf(stsfp, "PROCESSORS %d\n", CkNumPes()+CkNumNodes());  
00247   fprintf(stsfp, "SMPMODE %d %d\n", CkMyNodeSize(), CkNumNodes());
00248 #else   
00249   fprintf(stsfp, "PROCESSORS %d\n", CkNumPes());
00250 #endif  
00251   fprintf(stsfp, "TOTAL_CHARES %d\n", (int)_chareTable.size());
00252   fprintf(stsfp, "TOTAL_EPS %d\n", (int)_entryTable.size());
00253   fprintf(stsfp, "TOTAL_MSGS %d\n", (int)_msgTable.size());
00254   fprintf(stsfp, "TOTAL_PSEUDOS %d\n", (int)0);
00255   fprintf(stsfp, "TOTAL_EVENTS %d\n", (int)nUserEvents);
00256   size_t i;
00257   for(i=0;i<_chareTable.size();i++)
00258     fprintf(stsfp, "CHARE %d \"%s\" %d\n", (int)i, _chareTable[i]->name, _chareTable[i]->ndims);
00259   for(i=0;i<_entryTable.size();i++)
00260     fprintf(stsfp, "ENTRY CHARE %d \"%s\" %d %d\n", (int)i, _entryTable[i]->name,
00261                  (int)_entryTable[i]->chareIdx, (int)_entryTable[i]->msgIdx);
00262   for(i=0;i<_msgTable.size();i++)
00263     fprintf(stsfp, "MESSAGE %d %u\n", (int)i, (int)_msgTable[i]->size);
00264 }
00265 
00266 void traceCommonBeginIdle(void *proj,double curWallTime)
00267 {
00268   ((TraceArray *)proj)->beginIdle(curWallTime);
00269 }
00270  
00271 void traceCommonEndIdle(void *proj,double curWallTime)
00272 {
00273   ((TraceArray *)proj)->endIdle(curWallTime);
00274 }
00275 
00276 void TraceArray::traceBegin() {
00277   if (n==0) return; // No tracing modules registered.
00278 #if ! CMK_TRACE_IN_CHARM
00279   cancel_beginIdle = CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,(CcdVoidFn)traceCommonBeginIdle,this);
00280   cancel_endIdle = CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_BUSY,(CcdVoidFn)traceCommonEndIdle,this);
00281 #endif
00282   ALLDO(traceBegin());
00283 }
00284 
00285 void TraceArray::traceBeginOnCommThread() {
00286 #if CMK_SMP_TRACE_COMMTHREAD
00287   if (n==0) return; // No tracing modules registered.
00288 /*#if ! CMK_TRACE_IN_CHARM  
00289   cancel_beginIdle = CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,(CcdVoidFn)traceCommonBeginIdle,this);
00290   cancel_endIdle = CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_BUSY,(CcdVoidFn)traceCommonEndIdle,this);
00291 #endif*/
00292   ALLDO(traceBeginOnCommThread());
00293 #endif
00294 }
00295 
00296 void TraceArray::traceEnd() {
00297   if (n==0) return; // No tracing modules registered.
00298   ALLDO(traceEnd());
00299 #if ! CMK_TRACE_IN_CHARM
00300   CcdCancelCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE, cancel_beginIdle);
00301   CcdCancelCallOnConditionKeep(CcdPROCESSOR_BEGIN_BUSY, cancel_endIdle);
00302 #endif
00303 }
00304 
00305 void TraceArray::traceEndOnCommThread() {
00306 #if CMK_SMP_TRACE_COMMTHREAD
00307   if (n==0) return; // No tracing modules registered.
00308   ALLDO(traceEndOnCommThread());
00309 /*#if ! CMK_TRACE_IN_CHARM
00310   CcdCancelCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE, cancel_beginIdle);
00311   CcdCancelCallOnConditionKeep(CcdPROCESSOR_BEGIN_BUSY, cancel_endIdle);
00312 #endif*/
00313 #endif
00314 }
00315 
00316 #if CMK_MULTICORE
00317 extern int Cmi_commthread;
00318 #endif
00319 
00320 /*Install the beginIdle/endIdle condition handlers.*/
00321 void traceBegin(void) {
00322 #if CMK_TRACE_ENABLED
00323   DEBUGF(("[%d] traceBegin called with %d at %f\n", CkMyPe(), CpvAccess(traceOn), TraceTimer()));
00324   
00325 #if CMK_SMP_TRACE_COMMTHREAD
00326   //the first core of this node controls the condition of comm thread
00327 #if CMK_MULTICORE
00328   if (Cmi_commthread)
00329 #endif
00330   if(CmiMyRank()==0){
00331     if(CpvAccessOther(traceOn, CmiMyNodeSize())!=1){
00332         CkpvAccessOther(_traces, CmiMyNodeSize())->traceBeginOnCommThread();        
00333         CpvAccessOther(traceOn, CmiMyNodeSize()) = 1;
00334     }
00335   }
00336 #endif
00337   if (CpvAccess(traceOn)==1) return;
00338   CkpvAccess(_traces)->traceBegin();
00339   CpvAccess(traceOn) = 1;
00340 #endif
00341 }
00342 
00343 /*Cancel the beginIdle/endIdle condition handlers.*/
00344 void traceEnd(void) {
00345 #if CMK_TRACE_ENABLED
00346   DEBUGF(("[%d] traceEnd called with %d at %f\n", CkMyPe(), CpvAccess(traceOn), TraceTimer()));
00347 
00348 #if CMK_SMP_TRACE_COMMTHREAD
00349 //the first core of this node controls the condition of comm thread
00350 #if CMK_MULTICORE
00351   if (Cmi_commthread)
00352 #endif
00353   if(CmiMyRank()==0){
00354     if(CkpvAccessOther(traceOn, CmiMyNodeSize())!=0){
00355         CkpvAccessOther(_traces, CmiMyNodeSize())->traceEndOnCommThread();
00356         CkpvAccessOther(traceOn, CmiMyNodeSize()) = 0;
00357     }
00358 }
00359 #endif
00360     
00361     
00362   if (CpvAccess(traceOn)==0) return;
00363   if (CkpvAccess(_traces) == NULL) {
00364     CmiPrintf("Warning: did you mix compilation with and without -DCMK_TRACE_ENABLED? \n");
00365   }
00366   CkpvAccess(_traces)->traceEnd();
00367   CpvAccess(traceOn) = 0;
00368 #endif
00369 }
00370 
00371 void traceBeginComm(void) {
00372 #if CMK_TRACE_ENABLED && CMK_SMP_TRACE_COMMTHREAD
00373 #if CMK_MULTICORE
00374   if (Cmi_commthread)
00375 #endif
00376     if (CmiMyRank() == 0) {
00377       if (CkpvAccessOther(traceOn, CmiMyNodeSize()) != 1) {
00378         CkpvAccessOther(_traces, CmiMyNodeSize())->traceBeginOnCommThread();
00379         CkpvAccessOther(traceOn, CmiMyNodeSize()) = 1;
00380       }
00381     }
00382 #endif
00383 }
00384 
00385 void traceEndComm(void) {
00386 #if CMK_TRACE_ENABLED && CMK_SMP_TRACE_COMMTHREAD
00387 #if CMK_MULTICORE
00388   if (Cmi_commthread)
00389 #endif
00390     if (CmiMyRank() == 0) {
00391       if (CkpvAccessOther(traceOn, CmiMyNodeSize()) != 0) {
00392         CkpvAccessOther(_traces, CmiMyNodeSize())->traceEndOnCommThread();
00393         CkpvAccessOther(traceOn, CmiMyNodeSize()) = 0;
00394       }
00395     }
00396 #endif
00397 }
00398 
00399 static int checkTraceOnPe(char **argv)
00400 {
00401   int traceOnPE = 1;
00402   char *procs = NULL;
00403 #if CMK_BIGSIM_CHARM
00404   // check bgconfig file for settings
00405   traceOnPE=0;
00406   if (BgTraceProjectionOn(CkMyPe())) traceOnPE = 1;
00407 #endif
00408   if (CmiGetArgStringDesc(argv, "+traceprocessors", &procs, "A list of processors to trace, e.g. 0,10,20-30"))
00409   {
00410     CkListString procList(procs);
00411     traceOnPE = procList.includes(CkMyPe());
00412   }
00413 
00414   if (CmiGetArgFlagDesc(argv, "+traceselective", " Whether only dump data for PEs based on perfReport"))
00415   {
00416       if(CkMyPe() !=0)
00417           CkpvAccess(dumpData) = false;
00418   }
00419 
00420   // must include pe 0, otherwise sts file is not generated
00421   if (CkMyPe()==0) traceOnPE = 1;
00422 #if !CMK_TRACE_IN_CHARM
00423 #if !CMK_SMP_TRACE_COMMTHREAD
00424   /* skip communication thread */
00425   traceOnPE = traceOnPE && (CkMyRank() != CkMyNodeSize());
00426 #endif
00427 #endif
00428   return traceOnPE;
00429 }
00430 
00432 void _createTraces(char **argv);
00433 
00434 
00435 bool enableCPTracing; // A global variable specifying whether or not the control point tracing module should be active in the run
00436 extern void _registerTraceControlPoints();
00437 extern void _createTracecontrolPoints(char **argv);
00438 
00439 
00444 
00445 static inline void _traceInit(char **argv) 
00446 {
00447   CkpvInitialize(TraceArray *, _traces);
00448   CkpvAccess(_traces) = new TraceArray;
00449 
00450   // common init
00451   traceCommonInit(argv);
00452 
00453   // check if trace is turned on/off for this pe
00454   CkpvAccess(traceOnPe) = checkTraceOnPe(argv);
00455 
00456 #if !CMK_CHARMPY
00457   // defined in moduleInit.C
00458   _createTraces(argv);
00459 #endif
00460 
00461   // Now setup the control point tracing module if desired. It is always compiled/linked in, but is not always enabled
00462   // FIXME: make sure it is safe to use argv in SMP version 
00463   // because CmiGetArgFlagDesc is destructive and this is called on all PEs.
00464   if( CmiGetArgFlagDesc(argv,"+CPEnableMeasurements","Enable recording of measurements for Control Points") ){
00465     enableCPTracing = true;
00466     _createTracecontrolPoints(argv);   
00467   } else {
00468     enableCPTracing = false;
00469   }
00470   
00471 
00472   // set trace on/off
00473   CkpvAccess(_traces)->setTraceOnPE(CkpvAccess(traceOnPe));
00474 
00475 #if CMK_SMP_TRACE_COMMTHREAD
00476 
00485    CmiBarrier();
00486 #endif
00487 
00488   if (CkpvAccess(_traces)->length()) {
00489     if (CkMyPe() == 0) 
00490       CmiPrintf("Trace: traceroot: %s\n", CkpvAccess(traceRoot));
00491     if (!CmiGetArgFlagDesc(argv,"+traceoff","Disable tracing"))
00492       traceBegin();
00493   }
00494 }
00495 
00497 void traceInit(char **argv) 
00498 {
00499 #if ! CMK_TRACE_IN_CHARM
00500   _traceInit(argv);
00501   initTraceCore(argv);
00502 #endif
00503 }
00504 
00506 void traceCharmInit(char **argv) 
00507 {
00508 #if CMK_TRACE_IN_CHARM
00509   _traceInit(argv);
00510 #endif
00511 }
00512 
00513 // CMK_TRACE_ENABLED is already guarded in convcore.C
00514 void traceMessageRecv(char *msg, int pe)
00515 {
00516 #if ! CMK_TRACE_IN_CHARM
00517   CkpvAccessOther(_traces, CmiRankOf(pe))->messageRecv(msg, pe);
00518 #endif
00519 }
00520 
00521 
00522 void traceBeginIdle()
00523 {
00524     _TRACE_ONLY(CkpvAccess(_traces)->beginIdle(CmiWallTimer()));
00525 }
00526 
00527 
00528 void traceEndIdle()
00529 {
00530     _TRACE_ONLY(CkpvAccess(_traces)->endIdle(CmiWallTimer()));
00531 }
00532 
00533 // CMK_TRACE_ENABLED is already guarded in convcore.C
00534 // converse thread tracing is not supported in blue gene simulator
00535 // in BigSim, threads need to be traced manually (because virtual processors
00536 // themselves are implemented as threads and we don't want them to be traced
00537 // In BigSim, so far, only AMPI threads are traced.
00538 void traceResume(int eventID, int srcPE, CmiObjId *tid)
00539 {
00540     _TRACE_BEGIN_EXECUTE_DETAILED(eventID, ForChareMsg, _threadEP, srcPE, 0, NULL, tid);
00541     if(CpvAccess(_traceCoreOn))
00542         resumeTraceCore();
00543 }
00544 
00545 void traceSuspend(void)
00546 {
00547   _TRACE_ONLY(CkpvAccess(_traces)->endExecute());
00548 }
00549 
00550 void traceAwaken(CthThread t)
00551 {
00552   CkpvAccess(_traces)->creation(0, _threadEP);
00553 #if CMK_TRACE_ENABLED
00554   CthSetEventInfo(t, CtvAccess(curThreadEvent), CkMyPe());
00555 #endif
00556 }
00557 
00558 void traceUserEvent(int e)
00559 {
00560 #if CMK_TRACE_ENABLED
00561   if (CpvAccess(traceOn))
00562     CkpvAccess(_traces)->userEvent(e);
00563 #endif
00564 }
00565 
00566 
00567 void beginAppWork()
00568 {
00569 #if CMK_TRACE_ENABLED
00570     if (CpvAccess(traceOn) && CkpvAccess(_traces))
00571     {
00572         CkpvAccess(_traces)->beginAppWork();
00573     }
00574 #endif
00575 }
00576 
00577 
00578 void endAppWork()
00579 {
00580 #if CMK_TRACE_ENABLED
00581     if (CpvAccess(traceOn) && CkpvAccess(_traces))
00582     {
00583         CkpvAccess(_traces)->endAppWork();
00584     }
00585 #endif
00586 }
00587 
00588 void countNewChare()
00589 {
00590 #if CMK_TRACE_ENABLED
00591     if (CpvAccess(traceOn) && CkpvAccess(_traces))
00592     {
00593         CkpvAccess(_traces)->countNewChare();
00594     }
00595 #endif
00596 }
00597 
00598 
00599 void beginTuneOverhead()
00600 {
00601 #if CMK_TRACE_ENABLED
00602     if (CpvAccess(traceOn) && CkpvAccess(_traces))
00603     {
00604         CkpvAccess(_traces)->beginTuneOverhead();
00605     }
00606 #endif
00607 }
00608 
00609 void endTuneOverhead()
00610 {
00611 #if CMK_TRACE_ENABLED
00612     if (CpvAccess(traceOn) && CkpvAccess(_traces))
00613     {
00614         CkpvAccess(_traces)->endTuneOverhead();
00615     }
00616 #endif
00617 }
00618 
00619 void traceUserBracketEvent(int e, double beginT, double endT)
00620 {
00621 #if CMK_TRACE_ENABLED
00622   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00623     CkpvAccess(_traces)->userBracketEvent(e, beginT, endT);
00624 #endif
00625 }
00626 
00627 // trace a UserBracketEvent that is coming from a "nested" thread, e.g. a virtual AMPI rank
00628 void traceUserBracketEventNestedID(int e, double beginT, double endT, int nestedID)
00629 {
00630 #if CMK_TRACE_ENABLED
00631   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00632     CkpvAccess(_traces)->userBracketEvent(e, beginT, endT, nestedID);
00633 #endif
00634 }
00635 
00636 void traceBeginUserBracketEvent(int e)
00637 {
00638 #if CMK_TRACE_ENABLED
00639   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00640     CkpvAccess(_traces)->beginUserBracketEvent(e);
00641 #endif
00642 }
00643 
00644 void traceBeginUserBracketEventNestedID(int e, int nestedID)
00645 {
00646 #if CMK_TRACE_ENABLED
00647   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00648     CkpvAccess(_traces)->beginUserBracketEvent(e, nestedID);
00649 #endif
00650 }
00651 
00652 void traceEndUserBracketEvent(int e)
00653 {
00654 #if CMK_TRACE_ENABLED
00655   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00656     CkpvAccess(_traces)->endUserBracketEvent(e);
00657 #endif
00658 }
00659 
00660 void traceEndUserBracketEventNestedID(int e, int nestedID)
00661 {
00662 #if CMK_TRACE_ENABLED
00663   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00664     CkpvAccess(_traces)->endUserBracketEvent(e, nestedID);
00665 #endif
00666 }
00667 
00668 //common version of User Stat Functions
00669 int traceRegisterUserStat(const char*x, int e)
00670 {
00671 #if CMK_TRACE_ENABLED
00672   return CkpvAccess(_traces)->traceRegisterUserStat(x, e);
00673 #else
00674   return 0;
00675 #endif
00676 }
00677 
00678 void updateStatPair(int e, double stat, double time)
00679 {
00680 #if CMK_TRACE_ENABLED
00681   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00682     CkpvAccess(_traces)->updateStatPair(e, stat, time);
00683 #endif
00684 }
00685 
00686 void updateStat(int e, double stat)
00687 {
00688 #if CMK_TRACE_ENABLED
00689   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00690     CkpvAccess(_traces)->updateStat(e, stat);
00691 #endif
00692 }
00693 
00694 void traceUserSuppliedData(int d)
00695 {
00696 #if CMK_TRACE_ENABLED
00697   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00698     CkpvAccess(_traces)->userSuppliedData(d);
00699 #endif
00700 }
00701 
00702 void traceUserSuppliedNote(const char * note)
00703 {
00704 #if CMK_TRACE_ENABLED
00705   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00706     CkpvAccess(_traces)->userSuppliedNote(note);
00707 #endif
00708 }
00709 
00710 
00711 void traceUserSuppliedBracketedNote(const char *note, int eventID, double bt, double et)
00712 {
00713   //CkPrintf("traceUserSuppliedBracketedNote(const char *note, int eventID, double bt, double et)\n");
00714 #if CMK_TRACE_ENABLED
00715   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00716     CkpvAccess(_traces)->userSuppliedBracketedNote(note, eventID, bt, et);
00717 #endif
00718 }
00719 
00720 
00721 void traceMemoryUsage()
00722 {
00723 #if CMK_TRACE_ENABLED
00724   double d = CmiMemoryUsage()*1.0;
00725 
00726   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00727     CkpvAccess(_traces)->memoryUsage(d);
00728 #endif
00729 }
00730 
00731 void tracePhaseEnd()
00732 {
00733   _TRACE_ONLY(CkpvAccess(_traces)->endPhase());
00734 }
00735 
00736 void registerMachineUserEventsFunction(void (*eventRegistrationFunc)()) {
00737   CmiAssert(CpvInitialized(machineTraceFuncPtr));
00738   CpvAccess(machineTraceFuncPtr) = eventRegistrationFunc;
00739 }
00740 
00741 void (*registerMachineUserEvents())() {
00742   CmiAssert(CpvInitialized(machineTraceFuncPtr));
00743   if (CpvAccess(machineTraceFuncPtr) != NULL) {
00744     return CpvAccess(machineTraceFuncPtr);
00745   } else {
00746     return NULL;
00747   }
00748 }
00749 
00750 int traceRegisterUserEvent(const char*x, int e)
00751 {
00752 #if CMK_TRACE_ENABLED
00753   return CkpvAccess(_traces)->traceRegisterUserEvent(x, e);
00754 #else
00755   return 0;
00756 #endif
00757 }
00758 
00759 void traceClearEps(void)
00760 {
00761   OPTIMIZE_WARNING
00762   CkpvAccess(_traces)->traceClearEps();
00763 }
00764 
00765 void traceWriteSts(void)
00766 {
00767   OPTIMIZE_WARNING
00768   CkpvAccess(_traces)->traceWriteSts();
00769 }
00770 
00771 void traceFlushLog(void)
00772 {
00773   OPTIMIZE_WARNING
00774   CkpvAccess(_traces)->traceFlushLog();
00775 }
00776 
00781 void traceClose(void)
00782 {
00783 #if ! CMK_BIGSIM_CHARM
00784   OPTIMIZE_WARNING
00785   CkpvAccess(_traces)->traceClose();
00786 #endif   
00787 }
00788 
00789 void traceCharmClose(void)
00790 {
00791 #if CMK_BIGSIM_CHARM
00792   OPTIMIZE_WARNING
00793   CkpvAccess(_traces)->traceClose();
00794 #endif
00795 }
00796 
00797 /* **CW** This is the API called from user code to support CCS operations 
00798    if supported by the underlying trace module.
00799  */
00800 void traceEnableCCS(void)
00801 {
00802   OPTIMIZE_WARNING
00803   CkpvAccess(_traces)->traceEnableCCS();  
00804 }
00805 
00806 /* **CW** Support for thread listeners. This makes a call to each
00807    trace module which must support the call.
00808 */
00809 void traceAddThreadListeners(CthThread tid, envelope *e) {
00810   _TRACE_ONLY(CkpvAccess(_traces)->traceAddThreadListeners(tid, e));
00811 }
00812 
00813 #if 1
00814 // helper functions
00815 extern int _charmHandlerIdx;
00816 class CkCoreState;
00817 extern void _processHandler(void *, CkCoreState*);
00818 int isCharmEnvelope(void *msg);
00819 int CkIsCharmMessage(char *msg)
00820 {
00821 //CmiPrintf("[%d] CkIsCharmMessage: %d %p %d %p\n", CkMyPe(),CmiGetHandler(msg), CmiGetHandlerFunction(msg), _charmHandlerIdx, _processHandler);
00822   if ((CmiGetHandler(msg) == _charmHandlerIdx) &&
00823          (CmiGetHandlerFunction(msg) == (CmiHandlerEx)_processHandler))
00824     return 1;
00825   if (CmiGetXHandler(msg) == _charmHandlerIdx) return isCharmEnvelope(msg);
00826   return 0;
00827 }
00828 #endif
00829 
00830 // return 1 if any one of tracing modules is linked.
00831 int  traceAvailable()
00832 {
00833 #if ! CMK_TRACE_ENABLED
00834   return 0;
00835 #else
00836   return CkpvAccess(_traces)->length()>0;
00837 #endif
00838 }
00839 
00840 double CmiTraceTimer()
00841 {
00842   return TraceTimer();
00843 }
00844 
00845 void TraceArray::creation(envelope *env, int ep, int num)
00846 { 
00847     if (_entryTable[ep]->traceEnabled)
00848         ALLDO(creation(env, ep, num));
00849 }
00850 
00851 void TraceArray::creationMulticast(envelope *env, int ep, int num,
00852                    const int *pelist)
00853 {
00854   if (_entryTable[ep]->traceEnabled)
00855     ALLDO(creationMulticast(env, ep, num, pelist));
00856 }
00857 
00858 #if CMK_SMP_TRACE_COMMTHREAD
00859 int traceBeginCommOp(char *msg){
00860 #if CMK_TRACE_ENABLED
00861   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg)) {
00862     CkpvAccess(_traces)->beginExecute(msg);
00863     return 1;
00864   }
00865 #endif
00866   return 0;
00867 }
00868 
00869 void traceEndCommOp(char *msg){
00870 #if CMK_TRACE_ENABLED
00871   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
00872     CkpvAccess(_traces)->endExecute(msg);
00873 #endif
00874 }
00875 
00876 void traceSendMsgComm(char *msg){
00877 #if CMK_TRACE_ENABLED
00878   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
00879     CkpvAccess(_traces)->creation(msg);
00880 #endif
00881 }
00882 
00883 void traceCommSetMsgID(char *msg){
00884 #if CMK_TRACE_ENABLED
00885   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
00886     CkpvAccess(_traces)->traceCommSetMsgID(msg);
00887 #endif
00888 }
00889 
00890 #endif
00891 
00892 void traceGetMsgID(char *msg, int *pe, int *event)
00893 {
00894 #if CMK_TRACE_ENABLED
00895   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
00896     CkpvAccess(_traces)->traceGetMsgID(msg, pe, event);
00897 #endif
00898 }
00899 
00900 void traceSetMsgID(char *msg, int pe, int event)
00901 {
00902 #if CMK_TRACE_ENABLED
00903   if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
00904     CkpvAccess(_traces)->traceSetMsgID(msg, pe, event);
00905 #endif
00906 }
00907 
00908 
00909 void traceChangeLastTimestamp(double ts){
00910 #if CMK_TRACE_ENABLED
00911   if (CpvAccess(traceOn) && CkpvAccess(_traces))
00912     CkpvAccess(_traces)->changeLastEntryTimestamp(ts);
00913 #endif
00914 }
00915 
00916 #if CMK_HAS_COUNTER_PAPI
00917 CkpvDeclare(int, papiEventSet);
00918 CkpvDeclare(LONG_LONG_PAPI*, papiValues);
00919 CkpvDeclare(int, papiStarted);
00920 CkpvDeclare(int, papiStopped);
00921 #ifdef USE_SPP_PAPI
00922 int papiEvents[NUMPAPIEVENTS];
00923 #else
00924 int papiEvents[NUMPAPIEVENTS] = { PAPI_L1_TCM, PAPI_L1_TCA, PAPI_L2_TCM, PAPI_L2_TCA};
00925 #endif
00926 #endif // CMK_HAS_COUNTER_PAPI
00927 
00928 #if CMK_HAS_COUNTER_PAPI
00929 void initPAPI() {
00930 #if CMK_HAS_COUNTER_PAPI
00931   // We initialize and create the event sets for use with PAPI here.
00932   int papiRetValue;
00933   if(CkMyRank()==0){
00934       papiRetValue = PAPI_is_initialized();
00935       if(papiRetValue != PAPI_NOT_INITED)
00936           return;
00937     papiRetValue = PAPI_library_init(PAPI_VER_CURRENT);
00938     if (papiRetValue != PAPI_VER_CURRENT) {
00939       CmiAbort("PAPI Library initialization failure!\n");
00940     }
00941 #if CMK_SMP
00942     if(PAPI_thread_init(pthread_self) != PAPI_OK){
00943       CmiAbort("PAPI could not be initialized in SMP mode!\n");
00944     }
00945 #endif
00946   }
00947   CkpvInitialize(int, papiStarted);
00948   CkpvAccess(papiStarted) = 0;
00949   CkpvInitialize(int, papiStopped);
00950   CkpvAccess(papiStopped) = 0;
00951 
00952 #if CMK_SMP
00953   //PAPI_thread_init has to finish before calling PAPI_create_eventset
00954   #if CMK_SMP_TRACE_COMMTHREAD
00955       CmiNodeAllBarrier();
00956   #else
00957       CmiNodeBarrier();
00958   #endif
00959 #endif
00960   // PAPI 3 mandates the initialization of the set to PAPI_NULL
00961   CkpvInitialize(int, papiEventSet); 
00962   CkpvAccess(papiEventSet) = PAPI_NULL; 
00963   if (PAPI_create_eventset(&CkpvAccess(papiEventSet)) != PAPI_OK) {
00964     CmiAbort("PAPI failed to create event set!\n");
00965   }
00966 #ifdef USE_SPP_PAPI
00967   //  CmiPrintf("Using SPP counters for PAPI\n");
00968   if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00969     papiEvents[0] = PAPI_FP_OPS;
00970   }else{
00971     if(CmiMyPe()==0){
00972       CmiAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00973     }
00974   }
00975   if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00976     papiEvents[1] = PAPI_TOT_INS;
00977   }else{
00978     CmiAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00979   }
00980   int EventCode;
00981   int ret;
00982   ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00983   if(PAPI_query_event(EventCode)==PAPI_OK) {
00984     papiEvents[2] = EventCode;
00985   }else{
00986     CmiAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00987   }
00988   ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00989   if(PAPI_query_event(EventCode)==PAPI_OK) {
00990     papiEvents[3] = EventCode;
00991   }else{
00992     CmiAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00993   }
00994   if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00995     papiEvents[4] = PAPI_L1_DCA;
00996   }else{
00997     CmiAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00998   }
00999   if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
01000     papiEvents[5] = PAPI_TOT_CYC;
01001   }else{
01002     CmiAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
01003   }
01004 #else
01005   // just uses { PAPI_L2_DCM, PAPI_FP_OPS } the 2 initialized PAPI_EVENTS
01006 #endif
01007   papiRetValue = PAPI_add_events(CkpvAccess(papiEventSet), papiEvents, NUMPAPIEVENTS);
01008   if (papiRetValue < 0) {
01009     if (papiRetValue == PAPI_ECNFLCT) {
01010       CmiAbort("PAPI events conflict! Please re-assign event types!\n");
01011     } else {
01012       char error_str[PAPI_MAX_STR_LEN];
01013       //PAPI_perror(error_str);
01014       //PAPI_perror(papiRetValue,error_str,PAPI_MAX_STR_LEN);
01015       CmiAbort("PAPI failed to add designated events!\n");
01016     }
01017   }
01018   if(CkMyPe()==0)
01019     {
01020       CmiPrintf("Registered %d PAPI counters:",NUMPAPIEVENTS);
01021       char nameBuf[PAPI_MAX_STR_LEN];
01022       for(int i=0;i<NUMPAPIEVENTS;i++)
01023     {
01024       PAPI_event_code_to_name(papiEvents[i], nameBuf);
01025       CmiPrintf("%s ",nameBuf);
01026     }
01027       CmiPrintf("\n");
01028     }
01029   CkpvInitialize(LONG_LONG_PAPI*, papiValues);
01030   CkpvAccess(papiValues) = (LONG_LONG_PAPI*)malloc(NUMPAPIEVENTS*sizeof(LONG_LONG_PAPI));
01031   memset(CkpvAccess(papiValues), 0, NUMPAPIEVENTS*sizeof(LONG_LONG_PAPI));
01032 #endif
01033 }
01034 #endif
01035 
01036 void traceSend(void *env, int pe, int size)
01037 {
01038 #if CMK_TRACE_ENABLED
01039   if (CpvAccess(traceOn) && CkpvAccess(_traces))
01040       CkpvAccess(_traces)->messageSend(env, pe, size);
01041 #endif
01042 }
01043 
01044 void traceRecv(void *env , int size)
01045 {
01046 #if CMK_TRACE_ENABLED
01047   if (CpvAccess(traceOn) && CkpvAccess(_traces))
01048       CkpvAccess(_traces)->messageRecv(env, size);
01049 #endif
01050 }
01051
ck-perf/trace-common.C
