Charm++: Charm Source Code Documentation

00001 
00005 
00006 #ifndef LBDBH_H
00007 #define LBDBH_H
00008 
00009 #include "converse.h"
00010 #include "charm.h"
00011 #include "middle.h"
00012 
00013 #ifndef __STDC_FORMAT_MACROS
00014 # define __STDC_FORMAT_MACROS
00015 #endif
00016 #ifndef __STDC_LIMIT_MACROS
00017 # define __STDC_LIMIT_MACROS
00018 #endif
00019 #include <inttypes.h>
00020 #include <list>
00021 
00022 class LBDatabase;//Forward declaration
00023 
00024 //typedef float floatType;
00025 // type defined by build option
00026 #ifndef CMK_LBTIME_TYPE
00027 #define CMK_LBTIME_TYPE double
00028 #endif
00029 typedef CMK_LBTIME_TYPE LBRealType;
00030 
00031 #define COMPRESS_LDB    1
00032 
00033 extern int _lb_version;
00034 
00035 #ifdef __cplusplus
00036 extern "C" {
00037 #endif
00038 
00039 typedef void* cvoid; /* To eliminate warnings, because a C void* is not
00040             the same as a C++ void* */
00041 
00042   /*  User-defined object ID is 4 ints long (as defined in converse.h) */
00043   /*  as OBJ_ID_SZ */
00044 
00045 #if CMK_LBDB_ON
00046 typedef struct {
00047   void *handle;            // pointer to LBDB
00048 } LDHandle;
00049 #else
00050 typedef int LDHandle;
00051 #endif
00052 
00053 typedef struct _LDOMid {
00054   CkGroupID id;
00055   bool operator==(const struct _LDOMid& omId) const {
00056     return id == omId.id?true:false;
00057   }
00058   bool operator<(const struct _LDOMid& omId) const {
00059     return id < omId.id?true:false;
00060   }
00061   bool operator!=(const struct _LDOMid& omId) const {
00062     return id == omId.id?false:true;
00063   }
00064   inline void pup(PUP::er &p);
00065 } LDOMid;
00066 
00067 typedef struct {
00068   LDHandle ldb;
00069 //  void *user_ptr;
00070   LDOMid id;
00071   int handle;       // index to LBOM
00072   inline void pup(PUP::er &p);
00073 } LDOMHandle;
00074 
00075 /* LDObjKey uniquely identify one object */
00076 typedef struct _LDObjKey {
00078   LDOMid omId;
00079   CmiUInt8 objId;
00080 public:
00081   bool operator==(const _LDObjKey& obj) const {
00082     return (bool)(omId == obj.omId && objId == obj.objId);
00083   }
00084   bool operator<(const _LDObjKey& obj) const {
00085     if (omId < obj.omId) return true;
00086     else if (omId == obj.omId) return objId < obj.objId;
00087     else return false;
00088   }
00089   inline LDOMid &omID() { return omId; }
00090   inline CmiUInt8 &objID() { return objId; }
00091   inline const LDOMid &omID() const { return omId; }
00092   inline const CmiUInt8 &objID() const { return objId; }
00093   inline void pup(PUP::er &p);
00094 } LDObjKey;
00095 
00096 typedef int LDObjIndex;
00097 typedef int LDOMIndex;
00098 
00099 typedef struct {
00100   LDOMHandle omhandle;
00101   CmiUInt8 id;
00102   LDObjIndex  handle;
00103   inline const LDOMid &omID() const { return omhandle.id; }
00104   inline const CmiUInt8 &objID() const { return id; }
00105   inline void pup(PUP::er &p);
00106 } LDObjHandle;
00107 
00108 /* defines user data layout  */
00109 class LBUserDataLayout {
00110 int length;
00111 int count;
00112 public:
00113   LBUserDataLayout(): length(0), count(0) {}
00114   int claim(int size) {
00115     count++;
00116     int oldlen = length;
00117     length+=size;
00118     return oldlen;
00119   }
00120   int size() { return length; }
00121 };
00122 
00123 CkpvExtern(LBUserDataLayout, lbobjdatalayout);
00124 
00125 class LBObjUserData {
00126   char *data;
00127 public:
00128   LBObjUserData() : data(NULL) {}
00129 
00130   LBObjUserData(const LBObjUserData &d) {
00131     if (d.data != NULL) {
00132       init();
00133       memcpy(data, d.data, CkpvAccess(lbobjdatalayout).size());
00134     }
00135   }
00136 
00137   ~LBObjUserData() { delete [] data; }
00138   LBObjUserData &operator = (const LBObjUserData &d) {
00139     if (d.data != NULL) {
00140       if (data==NULL) init();
00141       memcpy(data, d.data, CkpvAccess(lbobjdatalayout).size());
00142     }
00143     return *this;
00144   }
00145   inline void init() { data = new char[CkpvAccess(lbobjdatalayout).size()]; }
00146   inline void pup(PUP::er &p);
00147   void *getData(int idx) { if (data==NULL) init(); return (void*)(data+idx); }
00148 };
00149 
00150 typedef struct {
00151   LDObjHandle handle;
00152   LBRealType wallTime;
00153 #if CMK_LB_CPUTIMER
00154   LBRealType cpuTime;
00155 #endif
00156 #if ! COMPRESS_LDB
00157   LBRealType minWall, maxWall;
00158 #endif
00159   bool migratable;
00160   bool asyncArrival;
00161 #if CMK_LB_USER_DATA
00162   LBObjUserData   userData;
00163 #endif
00164   // An encoded approximation of the amount of data the object would pack;
00165   // call pup_decodeSize(pupSize) to get the actual approximate value
00166   CmiUInt2 pupSize;
00167   inline const LDOMHandle &omHandle() const { return handle.omhandle; }
00168   inline const LDOMid &omID() const { return handle.omhandle.id; }
00169   inline const CmiUInt8 &objID() const { return handle.id; }
00170   inline const CmiUInt8 &id() const { return handle.id; }
00171   inline void pup(PUP::er &p);
00172 #if CMK_LB_USER_DATA
00173   void* getUserData(int idx)  { return userData.getData(idx); }
00174 #endif
00175 } LDObjData;
00176 
00177 /* used by load balancer */
00178 typedef struct {
00179   int index;
00180   LDObjData data;
00181   int from_proc;
00182   int to_proc;
00183   inline void pup(PUP::er &p);
00184 } LDObjStats;
00185 
00186 #define LD_PROC_MSG      1
00187 #define LD_OBJ_MSG       2
00188 #define LD_OBJLIST_MSG   3
00189 
00190 typedef struct _LDCommDesc {
00191   char type;
00192   union {
00193     int destProc;       /* 1:   processor level message */
00194     struct{
00195       LDObjKey  destObj;        /* 2:   object based message    */
00196       int destObjProc;
00197     } destObj;
00198     struct {
00199       LDObjKey  *objs;
00200       int len;
00201     } destObjs;         /* 3:   one to many message     */
00202   } dest;
00203   char &get_type() { return type; }
00204   char get_type() const { return type; }
00205   int proc() const { return type==LD_PROC_MSG?dest.destProc:-1; }
00206   void setProc(int pe) { CmiAssert(type==LD_PROC_MSG); dest.destProc = pe; }
00207   int lastKnown() const { 
00208     if (type==LD_OBJ_MSG) return dest.destObj.destObjProc;
00209     if (type==LD_PROC_MSG) return dest.destProc;
00210     return -1;
00211   }
00212   LDObjKey &get_destObj() 
00213     { CmiAssert(type==LD_OBJ_MSG); return dest.destObj.destObj; }
00214   LDObjKey const &get_destObj() const 
00215     { CmiAssert(type==LD_OBJ_MSG); return dest.destObj.destObj; }
00216   LDObjKey * get_destObjs(int &len) 
00217     { CmiAssert(type==LD_OBJLIST_MSG); len=dest.destObjs.len; return dest.destObjs.objs; }
00218   void init_objmsg(LDOMid &omid, CmiUInt8 &objid, int destObjProc) {
00219     type=LD_OBJ_MSG; 
00220     dest.destObj.destObj.omID()=omid;
00221     dest.destObj.destObj.objID() =objid;
00222     dest.destObj.destObjProc = destObjProc;
00223   }
00224   void init_mcastmsg(LDOMid &omid, CmiUInt8 *objid, int len) {
00225     type=LD_OBJLIST_MSG; 
00226     dest.destObjs.len = len;
00227     dest.destObjs.objs = new LDObjKey[len];
00228         for (int i=0; i<len; i++) {
00229       dest.destObjs.objs[i].omID()=omid;
00230       dest.destObjs.objs[i].objID() =objid[i];
00231     }
00232   }
00233   inline bool operator==(const _LDCommDesc &obj) const;
00234   inline _LDCommDesc &operator=(const _LDCommDesc &c);
00235   inline void pup(PUP::er &p);
00236 } LDCommDesc;
00237 
00238 typedef struct _LDCommData {
00239   int src_proc;         // sender can either be a proc or an obj
00240   LDObjKey  sender;     // 
00241   LDCommDesc   receiver;
00242   int  sendHash, recvHash;
00243   int messages;
00244   int bytes;
00245   inline _LDCommData &operator=(const _LDCommData &o) {
00246     if (&o == this) return *this;
00247     src_proc = o.src_proc;
00248     sender = o.sender; receiver = o.receiver;
00249     sendHash = o.sendHash; recvHash = o.recvHash;
00250     messages = o.messages;
00251     bytes = o.bytes;
00252     return *this;
00253   }
00254   inline int from_proc() const { return (src_proc != -1); }
00255   inline int recv_type() const { return receiver.get_type(); }
00256   inline void pup(PUP::er &p);
00257   inline void clearHash() { sendHash = recvHash = -1; }
00258 } LDCommData;
00259 
00260 /*
00261  * Requests to load balancer
00262  *   FIXME: these routines don't seem to exist anywhere-- are they obsolete?
00263  *   Are the official versions now in LBDatabase.h?
00264  */
00265 void LBBalance(void *param);
00266 void LBCollectStatsOn(void);
00267 void LBCollectStatsOff(void);
00268 
00269 /*
00270  * Callbacks from database to object managers
00271  */
00272 typedef void (*LDMigrateFn)(LDObjHandle handle, int dest);
00273 typedef void (*LDStatsFn)(LDOMHandle h, int state);
00274 typedef void (*LDQueryEstLoadFn)(LDOMHandle h);
00275 typedef void (*LDMetaLBResumeWaitingCharesFn) (LDObjHandle handle, int lb_ideal_period);
00276 typedef void (*LDMetaLBCallLBOnCharesFn) (LDObjHandle handle);
00277 
00278 typedef struct {
00279   LDMigrateFn migrate;
00280   LDStatsFn setStats;
00281   LDQueryEstLoadFn queryEstLoad;
00282   LDMetaLBResumeWaitingCharesFn metaLBResumeWaitingChares;
00283   LDMetaLBCallLBOnCharesFn metaLBCallLBOnChares;
00284 } LDCallbacks;
00285 
00286 /*
00287  * Calls from object managers to load database
00288  */
00289 #if CMK_LBDB_ON
00290 LDHandle LDCreate(void);
00291 #else
00292 #define LDCreate() 0
00293 #endif
00294 
00295 LDOMHandle LDRegisterOM(LDHandle _lbdb, LDOMid userID, 
00296             void *userptr, LDCallbacks cb);
00297 void LDUnregisterOM(LDHandle _db, LDOMHandle handle);
00298 
00299 void LDOMMetaLBResumeWaitingChares(LDHandle _h, int lb_ideal_period);
00300 void LDOMMetaLBCallLBOnChares(LDHandle _h);
00301 void * LDOMUserData(LDOMHandle &_h);
00302 void LDRegisteringObjects(LDOMHandle _h);
00303 void LDDoneRegisteringObjects(LDOMHandle _h);
00304 
00305 LDObjHandle LDRegisterObj(LDOMHandle h, CmiUInt8 id, void *userptr,
00306               bool migratable);
00307 void LDUnregisterObj(LDObjHandle h);
00308 
00309 void *LDObjUserData(LDObjHandle &_h);
00310 #if CMK_LB_USER_DATA
00311 void *LDDBObjUserData(LDObjHandle &_h, int idx);
00312 #endif
00313 void LDObjTime(LDObjHandle &h, LBRealType walltime, LBRealType cputime);
00314 int  CLDRunningObject(LDHandle _h, LDObjHandle* _o );
00315 void LDObjectStart(const LDObjHandle &_h);
00316 void LDObjectStop(const LDObjHandle &_h);
00317 void LDSend(const LDOMHandle &destOM, const CmiUInt8 &destid, unsigned int bytes, int destObjProc, int force);
00318 void LDMulticastSend(const LDOMHandle &destOM, CmiUInt8 *destids, int ndests, unsigned int bytes, int nMsgs);
00319 
00320 void LDMessage(LDObjHandle from, 
00321            LDOMid toOM, CmiUInt8 *toID, int bytes);
00322 
00323 void LDEstObjLoad(LDObjHandle h, double load);
00324 void LDNonMigratable(const LDObjHandle &h);
00325 void LDMigratable(const LDObjHandle &h);
00326 void LDSetPupSize(const LDObjHandle &h, size_t);
00327 void LDAsyncMigrate(const LDObjHandle &h, bool);
00328 void LDDumpDatabase(LDHandle _lbdb);
00329 
00330 /*
00331  * Calls from load balancer to load database
00332  */  
00333 typedef void (*LDMigratedFn)(void* data, LDObjHandle handle, int waitBarrier);
00334 void LDNotifyMigrated(LDHandle _lbdb, LDMigratedFn fn, void* data);
00335 
00336 typedef void (*LDStartLBFn)(void *user_ptr);
00337 void LDAddStartLBFn(LDHandle _lbdb, LDStartLBFn fn, void* data);
00338 void LDRemoveStartLBFn(LDHandle _lbdb, LDStartLBFn fn);
00339 void LDStartLB(LDHandle _db);
00340 void LDTurnManualLBOn(LDHandle _lbdb);
00341 void LDTurnManualLBOff(LDHandle _lbdb);
00342 
00343 typedef void (*LDMigrationDoneFn)(void *user_ptr);
00344 int LDAddMigrationDoneFn(LDHandle _lbdb, LDMigrationDoneFn fn,  void* data);
00345 void  LDRemoveMigrationDoneFn(LDHandle _lbdb, LDMigrationDoneFn fn);
00346 void LDMigrationDone(LDHandle _lbdb);
00347 
00348 typedef void (*LDPredictFn)(void* user_ptr);
00349 typedef void (*LDPredictModelFn)(void* user_ptr, void* model);
00350 typedef void (*LDPredictWindowFn)(void* user_ptr, void* model, int wind);
00351 void LDTurnPredictorOn(LDHandle _lbdb, void *model);
00352 void LDTurnPredictorOnWin(LDHandle _lbdb, void *model, int wind);
00353 void LDTurnPredictorOff(LDHandle _lbdb);
00354 void LDChangePredictor(LDHandle _lbdb, void *model);
00355 void LDCollectStatsOn(LDHandle _lbdb);
00356 void LDCollectStatsOff(LDHandle _lbdb);
00357 int  CLDCollectingStats(LDHandle _lbdb);
00358 void LDQueryEstLoad(LDHandle bdb);
00359 void LDGetObjLoad(LDObjHandle &h, LBRealType *wallT, LBRealType *cpuT);
00360 void LDQueryKnownObjLoad(LDObjHandle &h, LBRealType *wallT, LBRealType *cpuT);
00361 
00362 int LDGetObjDataSz(LDHandle _lbdb);
00363 void LDGetObjData(LDHandle _lbdb, LDObjData *data);
00364 
00365 int LDGetCommDataSz(LDHandle _lbdb);
00366 void LDGetCommData(LDHandle _lbdb, LDCommData *data);
00367 void LDGetCommInfo(LDHandle _lbdb, int& bytes, int& msgs, int& withinbytes, int& outsidebytes, int& num_nghbors, int& hops, int& hopbytes);
00368 
00369 void LDBackgroundLoad(LDHandle _lbdb, LBRealType *walltime, LBRealType *cputime);
00370 void LDIdleTime(LDHandle _lbdb, LBRealType *walltime);
00371 void LDTotalTime(LDHandle _lbdb, LBRealType *walltime, LBRealType *cputime);
00372 void LDGetTime(LDHandle _db, LBRealType *total_walltime,LBRealType *total_cputime,
00373                    LBRealType *idletime, LBRealType *bg_walltime, LBRealType *bg_cputime);
00374 
00375 void LDClearLoads(LDHandle _lbdb);
00376 int  LDMigrate(LDObjHandle h, int dest);
00377 void LDMigrated(LDObjHandle h, int waitBarrier);
00378 
00379 /*
00380  * Local Barrier calls
00381  */
00382 typedef void (*LDBarrierFn)(void *user_ptr);
00383 typedef void (*LDResumeFn)(void *user_ptr);
00384 
00385 class client;
00386 struct LDBarrierClient {
00387   std::list<client *>::iterator i;
00388   LDBarrierClient() { }
00389   LDBarrierClient(std::list<client *>::iterator in)
00390   : i(in) { }
00391 };
00392 
00393 class receiver;
00394 struct LDBarrierReceiver {
00395   std::list<receiver *>::iterator i;
00396   LDBarrierReceiver() { }
00397   LDBarrierReceiver(std::list<receiver *>::iterator in)
00398   : i(in) { }
00399 };
00400 
00401 void LDAtLocalBarrier(LDHandle _lbdb, LDBarrierClient h);
00402 void LDDecreaseLocalBarrier(LDHandle _lbdb, LDBarrierClient h, int c);
00403 void LDLocalBarrierOn(LDHandle _db);
00404 void LDLocalBarrierOff(LDHandle _db);
00405 void LDResumeClients(LDHandle _lbdb);
00406 int LDProcessorSpeed();
00407 bool LDOMidEqual(const LDOMid &i1, const LDOMid &i2);
00408 
00409 /*
00410  *  LBDB Configuration calls
00411  */
00412 void LDSetLBPeriod(LDHandle _db, double s);
00413 double LDGetLBPeriod(LDHandle _db);
00414 
00415 int LDMemusage(LDHandle _db);
00416 
00417 #ifdef __cplusplus
00418 }
00419 #endif /* _cplusplus */
00420 
00421 const LDObjHandle &LDGetObjHandle(LDHandle h, int idx);
00422 LDBarrierClient LDAddLocalBarrierClient(LDHandle _lbdb,LDResumeFn fn,
00423                     void* data);
00424 void LDRemoveLocalBarrierClient(LDHandle _lbdb, LDBarrierClient h);
00425 LDBarrierReceiver LDAddLocalBarrierReceiver(LDHandle _lbdb,LDBarrierFn fn,
00426                         void* data);
00427 void LDRemoveLocalBarrierReceiver(LDHandle _lbdb,LDBarrierReceiver h);
00428 
00429 #if CMK_LBDB_ON
00430 PUPbytes(LDHandle)
00431 #endif
00432 
00433 inline void LDOMid::pup(PUP::er &p) {
00434   id.pup(p);
00435 }
00436 PUPmarshall(LDOMid)
00437 
00438 inline void LDObjKey::pup(PUP::er &p) {
00439   p|omId;
00440   p|objId;
00441 }
00442 PUPmarshall(LDObjKey)
00443 
00444 inline void LDObjStats::pup(PUP::er &p) {
00445   p|index;
00446   p|data;
00447   p|from_proc;
00448   p|to_proc;
00449 }
00450 PUPmarshall(LDObjStats)
00451 inline void LDOMHandle::pup(PUP::er &p) {
00452   // skip ldb since it is a pointer
00453   int ptrSize = sizeof(void *);
00454   p|ptrSize;
00455   // if pointer size is not expected, must be in simulation mode
00456   // ignore this field
00457   if (p.isUnpacking() && ptrSize != sizeof(void *)) {  
00458     char dummy;
00459     for (int i=0; i<ptrSize; i++) p|dummy;
00460   }
00461   else
00462     p|ldb;
00463   p|id;
00464   p|handle;
00465 }
00466 PUPmarshall(LDOMHandle)
00467 
00468 inline void LDObjHandle::pup(PUP::er &p) {
00469   p|omhandle;
00470   p|id;
00471   p|handle;
00472 }
00473 PUPmarshall(LDObjHandle)
00474 
00475 inline void LBObjUserData::pup(PUP::er &p) {
00476   int hasData;
00477   if (!p.isUnpacking()) hasData = data != NULL;
00478   p|hasData;
00479   if (p.isUnpacking()) {
00480     if (hasData)
00481       data = new char[CkpvAccess(lbobjdatalayout).size()];
00482     else
00483       data = NULL;
00484   }
00485   if (data) p(data, CkpvAccess(lbobjdatalayout).size());
00486 }
00487 PUPmarshall(LBObjUserData)
00488 
00489 inline void LDObjData::pup(PUP::er &p) {
00490   p|handle;
00491   p|wallTime;
00492 #if CMK_LB_CPUTIMER
00493   p|cpuTime;
00494 #endif
00495 #if ! COMPRESS_LDB
00496   p|minWall;
00497   p|maxWall;
00498 #endif
00499   p|migratable;
00500   if (_lb_version > -1) p|asyncArrival;
00501 #if CMK_LB_USER_DATA
00502   if (_lb_version > 2) {
00503     p|userData;
00504   }
00505 #endif
00506   p|pupSize;
00507 }
00508 PUPmarshall(LDObjData)
00509 
00510 inline bool LDCommDesc::operator==(const LDCommDesc &obj) const {
00511     if (type != obj.type) return false;
00512     switch (type) {
00513     case LD_PROC_MSG: return dest.destProc == obj.dest.destProc;
00514     case LD_OBJ_MSG:  return dest.destObj.destObj == obj.dest.destObj.destObj;
00515     case LD_OBJLIST_MSG: { if (dest.destObjs.len != obj.dest.destObjs.len) 
00516                                return false;
00517                            for (int i=0; i<dest.destObjs.len; i++)
00518                              if (!(dest.destObjs.objs[i] == obj.dest.destObjs.objs[i])) return false;
00519                            return true; }
00520     }
00521     return false;
00522 }
00523 inline LDCommDesc & LDCommDesc::operator=(const LDCommDesc &c) {
00524     type = c.type;
00525     switch (type) {
00526     case LD_PROC_MSG: dest.destProc = c.dest.destProc; break;
00527     case LD_OBJ_MSG:  dest.destObj = c.dest.destObj; break;
00528     case LD_OBJLIST_MSG: { dest.destObjs.len = c.dest.destObjs.len;
00529                            dest.destObjs.objs = new LDObjKey[dest.destObjs.len];
00530                            for (int i=0; i<dest.destObjs.len; i++)
00531                              dest.destObjs.objs[i] = c.dest.destObjs.objs[i]; 
00532                            break; }
00533     }
00534     return *this;
00535 }
00536 inline void LDCommDesc::pup(PUP::er &p) {
00537   p|type;
00538   switch (type) {
00539   case LD_PROC_MSG:  p|dest.destProc; break;
00540   case LD_OBJ_MSG:   p|dest.destObj.destObj; 
00541                      if (_lb_version == -1 && p.isUnpacking()) 
00542                dest.destObj.destObjProc = -1;
00543              else
00544                p|dest.destObj.destObjProc; 
00545              break;
00546   case LD_OBJLIST_MSG:  {  p|dest.destObjs.len; 
00547                            if (p.isUnpacking()) 
00548                                dest.destObjs.objs = new LDObjKey[dest.destObjs.len];
00549                            for (int i=0; i<dest.destObjs.len; i++) p|dest.destObjs.objs[i];
00550                            break; }
00551   }   // end of switch
00552 }
00553 PUPmarshall(LDCommDesc)
00554 
00555 inline void LDCommData::pup(PUP::er &p) {
00556     p|src_proc;
00557     p|sender;
00558     p|receiver;
00559     p|messages;
00560     p|bytes;
00561     if (p.isUnpacking()) {
00562       sendHash = recvHash = -1;
00563     }
00564 }
00565 PUPmarshall(LDCommData)
00566 
00567 #endif /* LBDBH_H */
00568
ck-ldb/lbdb.h
