Charm++: Charm Source Code Documentation

00001 
00007 #ifndef _BGP_TORUS_H_
00008 #define _BGP_TORUS_H_
00009 
00010 #include "converse.h"
00011 
00012 #if CMK_BLUEGENEP
00013 
00014 #include "dcmf.h"
00015 
00016 class BGPTorusManager {
00017   private:
00018     DCMF_Hardware_t bgp_hwt;
00019     int dimX;   // dimension of the allocation in X (no. of processors)
00020     int dimY;   // dimension of the allocation in Y (no. of processors)
00021     int dimZ;   // dimension of the allocation in Z (no. of processors)
00022     
00023     int hw_NX;  // dimension of the allocation in X (no. of nodes, booted partition)
00024     int hw_NY;  // dimension of the allocation in Y (no. of nodes, booted partition)
00025     int hw_NZ;  // dimension of the allocation in Z (no. of nodes, booted partition)
00026     int hw_NT;  // dimension of the allocation in T (no. of processors per node, booted partition)
00027     
00028     int rn_NX;  // acutal dimension of the allocation in X for the job
00029     int rn_NY;  // acutal dimension of the allocation in Y for the job
00030     int rn_NZ;  // acutal dimension of the allocation in Z for the job
00031     
00032     int thdsPerProc; //the number of threads per process (the value of +ppn)
00033     int procsPerNode; //the number of processes per node
00034     
00035     int torus[4];
00036     char *mapping;
00037 
00038   public:
00039     BGPTorusManager() {
00040       DCMF_Hardware(&bgp_hwt);
00041       hw_NX = bgp_hwt.xSize;
00042       hw_NY = bgp_hwt.ySize;
00043       hw_NZ = bgp_hwt.zSize;
00044     
00045       procsPerNode = bgp_hwt.tSize;
00046       int numPes = CmiNumPes();
00047       thdsPerProc = CmiMyNodeSize();
00048       
00049       hw_NT = procsPerNode*thdsPerProc;      
00050 
00051 
00052       //if(CmiMyPe()==0) printf("hw_NX/Y/Z/T=[%d, %d, %d, %d]\n", hw_NX, hw_NY, hw_NZ, hw_NT);
00053 
00054       //initialize the rn_N(*) to hw_N(*), then adjust them
00055       rn_NX = hw_NX;
00056       rn_NY = hw_NY;
00057       rn_NZ = hw_NZ;
00058       
00059       int max_t = 0;
00060       if(rn_NX * rn_NY * rn_NZ != numPes/hw_NT) {
00061         rn_NX = rn_NY = rn_NZ = 0;
00062         int rn_NT=0;
00063         int min_x, min_y, min_z, min_t;
00064         min_x = min_y = min_z = min_t = (~(-1));
00065         unsigned int tmp_t, tmp_x, tmp_y, tmp_z;
00066                      
00067         for(int c = 0; c < numPes/thdsPerProc; c++) {
00068 #if (DCMF_VERSION_MAJOR >= 3)
00069           DCMF_NetworkCoord_t  nc;
00070           DCMF_Messager_rank2network (c, DCMF_DEFAULT_NETWORK, &nc);
00071           tmp_x = nc.torus.x;
00072           tmp_y = nc.torus.y;
00073           tmp_z = nc.torus.z;
00074           tmp_t = nc.torus.t;
00075 #else
00076           DCMF_Messager_rank2torus(c, &tmp_x, &tmp_y, &tmp_z, &tmp_t);
00077 #endif
00078           //if(CmiMyPe()==0){
00079             //printf("Adjusting proc %d, runtime x/y/z/t=[%d, %d, %d, %d]\n", c, tmp_x, tmp_y, tmp_z, tmp_t);
00080           //}             
00081  
00082           if(tmp_x > rn_NX) rn_NX = tmp_x;
00083           if(tmp_x < min_x) min_x = tmp_x;
00084           if(tmp_y > rn_NY) rn_NY = tmp_y;
00085           if(tmp_y < min_y) min_y = tmp_y;
00086           if(tmp_z > rn_NZ) rn_NZ = tmp_z;
00087           if(tmp_z < min_z) min_z = tmp_z;
00088           if(tmp_t > rn_NT) rn_NT = tmp_t;
00089           if(tmp_t < min_t) min_t = tmp_t;
00090         }                
00091         rn_NX = rn_NX - min_x + 1;
00092         rn_NY = rn_NY - min_y + 1;
00093         rn_NZ = rn_NZ - min_z + 1;
00094 
00095         procsPerNode = rn_NT - min_t + 1;
00096         hw_NT = procsPerNode * thdsPerProc;
00097       }
00098       dimX = rn_NX;
00099       dimY = rn_NY;
00100       dimZ = rn_NZ;
00101       dimX = dimX * hw_NT;      // assuming TXYZ
00102 
00103       torus[0] = bgp_hwt.xTorus;
00104       torus[1] = bgp_hwt.yTorus;
00105       torus[2] = bgp_hwt.zTorus;
00106       torus[3] = bgp_hwt.tTorus;
00107       
00108       mapping = getenv("BG_MAPPING");      
00109       
00110       //printf("DimN[X,Y,Z,T]=[%d,%d,%d,%d], Dim[X,Y,Z]=[%d,%d,%d]\n", rn_NX,rn_NY,rn_NZ,hw_NT,dimX, dimY, dimZ);    
00111     }
00112 
00113     ~BGPTorusManager() {
00114      }
00115 
00116     inline int getDimX() { return dimX; }
00117     inline int getDimY() { return dimY; }
00118     inline int getDimZ() { return dimZ; }
00119 
00120     inline int getDimNX() { return rn_NX; }
00121     inline int getDimNY() { return rn_NY; }
00122     inline int getDimNZ() { return rn_NZ; }
00123     inline int getDimNT() { return hw_NT; }
00124 
00125     inline int getProcsPerNode() { return procsPerNode; }
00126 
00127     inline int* isTorus() { return torus; }
00128 
00129 
00130     inline void rankToCoordinates(int pe, int &x, int &y, int &z) {
00131       x = pe % dimX;
00132       y = (pe % (dimX*dimY)) / dimX;
00133       z = pe / (dimX*dimY);
00134     }
00135 
00136     inline int coordinatesToRank(int x, int y, int z) {
00137       return x + (y + z*dimY) * dimX;
00138     }
00139 
00140 #if 0
00141     inline void rankToCoordinates(int pe, int &x, int &y, int &z, int &t) {
00142       if(mapping==NULL || (mapping!=NULL && mapping[0]=='X')) {
00143         x = pe % rn_NX;
00144         y = (pe % (rn_NX*rn_NY)) / rn_NX;
00145         z = (pe % (rn_NX*rn_NY*rn_NZ)) / (rn_NX*rn_NY);
00146         t = pe / (rn_NX*rn_NY*rn_NZ);
00147       } else {
00148         t = pe % hw_NT;
00149         x = (pe % (hw_NT*rn_NX)) / hw_NT;
00150         y = (pe % (hw_NT*rn_NX*rn_NY)) / (hw_NT*rn_NX);
00151         z = pe / (hw_NT*rn_NX*rn_NY);
00152       }
00153     }
00154 
00155     inline int coordinatesToRank(int x, int y, int z, int t) {
00156       if(mapping==NULL || (mapping!=NULL && mapping[0]=='X'))
00157         return x + (y + (z + t*rn_NZ) * rn_NY) * rn_NX;
00158       else
00159         return t + (x + (y + z*rn_NY) * rn_NX) * hw_NT;
00160     }
00161 #else
00162     //Make it smp-aware that each node could have different numbers of processes
00163     //Basically the machine is viewed as NX*NY*NZ*(#cores/node), 
00164     //(#cores/node) can be viewed as (#processes/node * #threads/process)
00165     
00166     inline void rankToCoordinates(int pe, int &x, int &y, int &z, int &t) {
00167       if(mapping==NULL || (mapping!=NULL && mapping[0]=='X')) {
00168         int nthRank = pe % thdsPerProc;
00169         //tmpPE = x+y*rn_NX+z*rn_NX*rn_NY+nthProc*rn_NX*rn_NY*rn_NZ
00170         int tmpPE = pe / thdsPerProc;
00171         x = tmpPE%rn_NX;
00172         y = tmpPE / rn_NX % rn_NY;
00173         z = tmpPE / (rn_NX*rn_NY) % rn_NZ;
00174         int nthProc = tmpPE / (rn_NX*rn_NY*rn_NZ);
00175         t = nthProc*thdsPerProc + nthRank;        
00176       } else {
00177         t = pe % (thdsPerProc*procsPerNode);
00178         x = pe / (thdsPerProc*procsPerNode) % rn_NX;
00179         y = pe / (thdsPerProc*procsPerNode*rn_NX) % (rn_NY);
00180         z = pe / (thdsPerProc*procsPerNode*rn_NX*rn_NY);
00181       }
00182     }
00183 
00184     inline int coordinatesToRank(int x, int y, int z, int t) {
00185       if(mapping==NULL || (mapping!=NULL && mapping[0]=='X')){
00186         int pe;
00187         int nthProc = t/thdsPerProc;
00188         int nthRank = t%thdsPerProc;
00189         
00190         pe = (x+y*rn_NX+z*rn_NX*rn_NY)*thdsPerProc;
00191         
00192         pe += nthProc*rn_NX*rn_NY*rn_NZ*thdsPerProc + nthRank;
00193         
00194         return pe;
00195       }      
00196       else
00197         return t + (x + (y + z*rn_NY) * rn_NX) * thdsPerProc * procsPerNode;
00198     }    
00199      
00200 #endif        
00201 
00202         inline int getTotalPhyNodes(){
00203                 return rn_NX * rn_NY * rn_NZ;
00204         }
00205         inline int getMyPhyNodeID(int pe){
00206                 int x,y,z,t;
00207                 rankToCoordinates(pe, x, y, z, t);
00208                 return x+y*rn_NX+z*rn_NX*rn_NY;
00209         }
00210         
00211 };
00212 
00213 #endif // CMK_BLUEGENEP
00214 #endif //_BGP_TORUS_H_
util/BGPTorus.h
