00001
00011 #include "TopoManager.h"
00012
00013 TopoManager::TopoManager() {
00014 #if CMK_BLUEGENEL
00015 dimX = bgltm.getDimX();
00016 dimY = bgltm.getDimY();
00017 dimZ = bgltm.getDimZ();
00018
00019 dimNX = bgltm.getDimNX();
00020 dimNY = bgltm.getDimNY();
00021 dimNZ = bgltm.getDimNZ();
00022 dimNT = bgltm.getDimNT();
00023
00024 procsPerNode = bgltm.getProcsPerNode();
00025 int *torus;
00026 torus = bgltm.isTorus();
00027 torusX = torus[0];
00028 torusY = torus[1];
00029 torusZ = torus[2];
00030 torusT = torus[3];
00031
00032 #elif CMK_BLUEGENEP
00033 dimX = bgptm.getDimX();
00034 dimY = bgptm.getDimY();
00035 dimZ = bgptm.getDimZ();
00036
00037 dimNX = bgptm.getDimNX();
00038 dimNY = bgptm.getDimNY();
00039 dimNZ = bgptm.getDimNZ();
00040 dimNT = bgptm.getDimNT();
00041
00042 procsPerNode = bgptm.getProcsPerNode();
00043 int *torus;
00044 torus = bgptm.isTorus();
00045 torusX = torus[0];
00046 torusY = torus[1];
00047 torusZ = torus[2];
00048 torusT = torus[3];
00049
00050 #elif CMK_BLUEGENEQ
00051 dimX = bgqtm.getDimX();
00052 dimY = bgqtm.getDimY();
00053 dimZ = bgqtm.getDimZ();
00054
00055 dimNX = bgqtm.getDimNX();
00056 dimNY = bgqtm.getDimNY();
00057 dimNZ = bgqtm.getDimNZ();
00058 dimNT = bgqtm.getDimNT();
00059
00060 dimNA = bgqtm.getDimNA();
00061 dimNB = bgqtm.getDimNB();
00062 dimNC = bgqtm.getDimNC();
00063 dimND = bgqtm.getDimND();
00064 dimNE = bgqtm.getDimNE();
00065
00066 procsPerNode = bgqtm.getProcsPerNode();
00067 int *torus;
00068 torus = bgqtm.isTorus();
00069 torusA = torus[0];
00070 torusB = torus[1];
00071 torusC = torus[2];
00072 torusD = torus[3];
00073 torusE = torus[4];
00074
00075 #elif XT3_TOPOLOGY
00076 dimX = xt3tm.getDimX();
00077 dimY = xt3tm.getDimY();
00078 dimZ = xt3tm.getDimZ();
00079
00080 dimNX = xt3tm.getDimNX();
00081 dimNY = xt3tm.getDimNY();
00082 dimNZ = xt3tm.getDimNZ();
00083 dimNT = xt3tm.getDimNT();
00084
00085 procsPerNode = xt3tm.getProcsPerNode();
00086 int *torus;
00087 torus = xt3tm.isTorus();
00088 torusX = torus[0];
00089 torusY = torus[1];
00090 torusZ = torus[2];
00091 torusT = torus[3];
00092
00093 #elif XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00094 dimX = xttm.getDimX();
00095 dimY = xttm.getDimY();
00096 dimZ = xttm.getDimZ();
00097
00098 dimNX = xttm.getDimNX();
00099 dimNY = xttm.getDimNY();
00100 dimNZ = xttm.getDimNZ();
00101 dimNT = xttm.getDimNT();
00102
00103 procsPerNode = xttm.getProcsPerNode();
00104 int *torus;
00105 torus = xttm.isTorus();
00106 torusX = torus[0];
00107 torusY = torus[1];
00108 torusZ = torus[2];
00109 torusT = torus[3];
00110
00111 #else
00112 dimX = CmiNumPes();
00113 dimY = 1;
00114 dimZ = 1;
00115
00116 dimNX = dimX;
00117 dimNY = 1;
00118 dimNZ = 1;
00119
00120 dimNT = procsPerNode = 1;
00121 torusX = true;
00122 torusY = true;
00123 torusZ = true;
00124 torusT = false;
00125 #endif
00126
00127 #if CMK_BIGSIM_CHARM
00128 BgGetSize(&dimNX, &dimNY, &dimNZ);
00129
00130 dimNT = procsPerNode = BgGetNumWorkThread();
00131 dimX = dimNX * procsPerNode;
00132 dimY = dimNY;
00133 dimZ = dimNZ;
00134
00135 torusX = true;
00136 torusY = true;
00137 torusZ = true;
00138 torusT = false;
00139 #endif
00140
00141 numPes = CmiNumPes();
00142 }
00143
00144 TopoManager::TopoManager(int NX, int NY, int NZ, int NT) : dimNX(NX), dimNY(NY), dimNZ(NZ), dimNT(NT) {
00145
00146 procsPerNode = dimNT;
00147 dimX = dimNX * dimNT;
00148 dimY = dimNY;
00149 dimZ = dimNZ;
00150 torusX = true;
00151 torusY = true;
00152 torusZ = true;
00153 #if CMK_BLUEGENEQ
00154 torusA = true;
00155 torusB = true;
00156 torusC = true;
00157 torusD = true;
00158 torusE = true;
00159 #endif
00160 numPes = dimNX * dimNY * dimNZ * dimNT;
00161 }
00162
00163 int TopoManager::hasMultipleProcsPerNode() const {
00164 if(procsPerNode == 1)
00165 return 0;
00166 else
00167 return 1;
00168 }
00169
00170 void TopoManager::rankToCoordinates(int pe, int &x, int &y, int &z) {
00171 CmiAssert( pe >= 0 && pe < numPes );
00172 #if CMK_BLUEGENEL
00173 bgltm.rankToCoordinates(pe, x, y, z);
00174 #elif CMK_BLUEGENEP
00175 bgptm.rankToCoordinates(pe, x, y, z);
00176 #elif XT3_TOPOLOGY || XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00177 int t;
00178 xttm.rankToCoordinates(pe, x, y, z, t);
00179 #else
00180 if(dimY > 1){
00181
00182 x = pe % dimX;
00183 y = (pe % (dimX * dimY)) / dimX;
00184 z = pe / (dimX * dimY);
00185 }
00186 else {
00187 x = pe;
00188 y = 0;
00189 z = 0;
00190 }
00191 #endif
00192
00193 #if CMK_BIGSIM_CHARM
00194 if(dimY > 1){
00195
00196 x = pe % dimX;
00197 y = (pe % (dimX * dimY)) / dimX;
00198 z = pe / (dimX * dimY);
00199 }
00200 else {
00201 x = pe;
00202 y = 0;
00203 z = 0;
00204 }
00205 #endif
00206 }
00207
00208 void TopoManager::rankToCoordinates(int pe, int &x, int &y, int &z, int &t) {
00209 CmiAssert( pe >= 0 && pe < numPes );
00210 #if CMK_BLUEGENEL
00211 bgltm.rankToCoordinates(pe, x, y, z, t);
00212 #elif CMK_BLUEGENEP
00213 bgptm.rankToCoordinates(pe, x, y, z, t);
00214 #elif CMK_BLUEGENEQ
00215 bgqtm.rankToCoordinates(pe, x, y, z, t);
00216 #elif XT3_TOPOLOGY
00217 xt3tm.rankToCoordinates(pe, x, y, z, t);
00218 #elif XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00219 xttm.rankToCoordinates(pe, x, y, z, t);
00220 #else
00221 if(dimNY > 1) {
00222 t = pe % dimNT;
00223 x = (pe % (dimNT*dimNX)) / dimNT;
00224 y = (pe % (dimNT*dimNX*dimNY)) / (dimNT*dimNX);
00225 z = pe / (dimNT*dimNX*dimNY);
00226 } else {
00227 t = pe % dimNT;
00228 x = (pe % (dimNT*dimNX)) / dimNT;
00229 y = 0;
00230 z = 0;
00231 }
00232 #endif
00233
00234 #if CMK_BIGSIM_CHARM
00235 if(dimNY > 1) {
00236 t = pe % dimNT;
00237 x = (pe % (dimNT*dimNX)) / dimNT;
00238 y = (pe % (dimNT*dimNX*dimNY)) / (dimNT*dimNX);
00239 z = pe / (dimNT*dimNX*dimNY);
00240 } else {
00241 t = pe % dimNT;
00242 x = (pe % (dimNT*dimNX)) / dimNT;
00243 y = 0;
00244 z = 0;
00245 }
00246 #endif
00247 }
00248
00249 #if CMK_BLUEGENEQ
00250 void TopoManager::rankToCoordinates(int pe, int &a, int &b, int &c, int &d, int &e, int &t) {
00251 CmiAssert( pe >= 0 && pe < numPes );
00252 bgqtm.rankToCoordinates(pe, a, b, c, d, e, t);
00253 }
00254 #endif
00255
00256 int TopoManager::coordinatesToRank(int x, int y, int z) {
00257 CmiAssert( x>=0 && x<dimX && y>=0 && y<dimY && z>=0 && z<dimZ );
00258 #if CMK_BIGSIM_CHARM
00259 if(dimY > 1)
00260 return x + y*dimX + z*dimX*dimY;
00261 else
00262 return x;
00263 #endif
00264
00265 #if CMK_BLUEGENEL
00266 return bgltm.coordinatesToRank(x, y, z);
00267 #elif CMK_BLUEGENEP
00268 return bgptm.coordinatesToRank(x, y, z);
00269 #elif XT3_TOPOLOGY || XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00270 return xttm.coordinatesToRank(x, y, z, 0);
00271 #else
00272 if(dimY > 1)
00273 return x + y*dimX + z*dimX*dimY;
00274 else
00275 return x;
00276 #endif
00277 }
00278
00279 int TopoManager::coordinatesToRank(int x, int y, int z, int t) {
00280 CmiAssert( x>=0 && x<dimNX && y>=0 && y<dimNY && z>=0 && z<dimNZ && t>=0 && t<dimNT );
00281 #if CMK_BIGSIM_CHARM
00282 if(dimNY > 1)
00283 return t + (x + (y + z*dimNY) * dimNX) * dimNT;
00284 else
00285 return t + x * dimNT;
00286 #endif
00287
00288 #if CMK_BLUEGENEL
00289 return bgltm.coordinatesToRank(x, y, z, t);
00290 #elif CMK_BLUEGENEP
00291 return bgptm.coordinatesToRank(x, y, z, t);
00292 #elif CMK_BLUEGENEQ
00293 return bgqtm.coordinatesToRank(x, y, z, t);
00294 #elif XT3_TOPOLOGY
00295 return xt3tm.coordinatesToRank(x, y, z, t);
00296 #elif XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00297 return xttm.coordinatesToRank(x, y, z, t);
00298 #else
00299 if(dimNY > 1)
00300 return t + (x + (y + z*dimNY) * dimNX) * dimNT;
00301 else
00302 return t + x * dimNT;
00303 #endif
00304 }
00305
00306 #if CMK_BLUEGENEQ
00307 int TopoManager::coordinatesToRank(int a, int b, int c, int d, int e, int t) {
00308 CmiAssert( a>=0 && a<dimNA && b>=0 && b<dimNB && c>=0 && c<dimNC && d>=0 && d<dimND && e>=0 && e<dimNE && t>=0 && t<dimNT );
00309 return bgqtm.coordinatesToRank(a, b, c, d, e, t);
00310 }
00311 #endif
00312
00313 int TopoManager::getHopsBetweenRanks(int pe1, int pe2) {
00314 CmiAssert( pe1 >= 0 && pe1 < numPes );
00315 CmiAssert( pe2 >= 0 && pe2 < numPes );
00316 #if CMK_BLUEGENEQ
00317 int a1, b1, c1, d1, e1, t1, a2, b2, c2, d2, e2, t2;
00318 rankToCoordinates(pe1, a1, b1, c1, d1, e1, t1);
00319 rankToCoordinates(pe2, a2, b2, c2, d2, e2, t2);
00320 return (absA(a2-a1)+absB(b2-b1)+absC(c2-c1)+absD(d2-d1)+absE(e2-e1));
00321 #else
00322 int x1, y1, z1, x2, y2, z2, t1, t2;
00323 rankToCoordinates(pe1, x1, y1, z1, t1);
00324 rankToCoordinates(pe2, x2, y2, z2, t2);
00325 return (absX(x2-x1)+absY(y2-y1)+absZ(z2-z1));
00326 #endif
00327 }
00328
00329 void TopoManager::sortRanksByHops(int pe, int *pes, int *idx, int n) {
00330
00331
00332
00333
00334
00335 for(int i=0;i<n;i++)
00336 idx[i] = i;
00337 quicksort(pe, pes, idx, 0, n-1);
00338 }
00339
00340
00341
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351 int TopoManager::pickClosestRank(int mype, int *pes, int n){
00352 int minHops = getHopsBetweenRanks(mype, pes[0]);
00353 int minIdx=0;
00354 int nowHops;
00355 for(int i=1; i<n; i++) {
00356 nowHops = getHopsBetweenRanks(mype, pes[i]);
00357 if(nowHops < minHops) {
00358 minHops = nowHops;
00359 minIdx=i;
00360 }
00361 }
00362 return minIdx;
00363 }
00364
00365 int TopoManager::areNeighbors(int pe1, int pe2, int pe3, int distance) {
00366 #if CMK_BLUEGENEQ
00367 int pe1_a, pe1_b, pe1_c, pe1_d, pe1_e, pe1_t;
00368 int pe2_a, pe2_b, pe2_c, pe2_d, pe2_e, pe2_t;
00369 int pe3_a, pe3_b, pe3_c, pe3_d, pe3_e, pe3_t;
00370 rankToCoordinates(pe1, pe1_a, pe1_b, pe1_c, pe1_d, pe1_e, pe1_t);
00371 rankToCoordinates(pe2, pe2_a, pe2_b, pe2_c, pe2_d, pe2_e, pe2_t);
00372 rankToCoordinates(pe3, pe3_a, pe3_b, pe3_c, pe3_d, pe3_e, pe3_t);
00373
00374 if ( (absA(pe1_a - (pe2_a+pe3_a)/2) + absB(pe1_b - (pe2_b+pe3_b)/2)+absC(pe1_c - (pe2_c+pe3_c)/2)+absD(pe1_d - (pe2_d+pe3_d)/2)+absE(pe1_e - (pe2_e+pe3_e)/2)) <= distance )
00375 return 1;
00376 else
00377 return 0;
00378 #else
00379 int pe1_x, pe1_y, pe1_z, pe1_t;
00380 int pe2_x, pe2_y, pe2_z, pe2_t;
00381 int pe3_x, pe3_y, pe3_z, pe3_t;
00382
00383 rankToCoordinates(pe1, pe1_x, pe1_y, pe1_z, pe1_t);
00384 rankToCoordinates(pe2, pe2_x, pe2_y, pe2_z, pe2_t);
00385 rankToCoordinates(pe3, pe3_x, pe3_y, pe3_z, pe3_t);
00386
00387 if ( (absX(pe1_x - (pe2_x+pe3_x)/2) + absY(pe1_y - (pe2_y+pe3_y)/2) + absZ(pe1_z - (pe2_z+pe3_z)/2)) <= distance )
00388 return 1;
00389 else
00390 return 0;
00391 #endif
00392 }
00393
00394 void TopoManager::quicksort(int pe, int *pes, int *arr, int left, int right) {
00395 if(left<right) {
00396 int split = partition(pe, pes, arr, left, right);
00397 quicksort(pe, pes, arr, left, split);
00398 quicksort(pe, pes, arr, split+1, right);
00399 }
00400 }
00401
00402 int TopoManager::partition(int pe, int *pes, int *idx, int left, int right) {
00403 int val = getHopsBetweenRanks(pe, pes[idx[(left+right)/2]]);
00404 int lm = left-1;
00405 int rm = right+1;
00406 for(;;) {
00407 do
00408 rm--;
00409 while(getHopsBetweenRanks(pe, pes[idx[rm]]) > val);
00410 do
00411 lm++;
00412 while(getHopsBetweenRanks(pe, pes[idx[lm]]) < val);
00413 if(lm < rm) {
00414 int tmp = idx[rm];
00415 idx[rm] = idx[lm];
00416 idx[lm] = tmp;
00417 }
00418 else
00419 return rm;
00420 }
00421 }
00422
00423 void TopoManager::printAllocation(FILE *fp)
00424 {
00425 int i,x,y,z,t;
00426 fprintf(fp, "Topology Info-\n");
00427 fprintf(fp, "NumPes - %d\n", numPes);
00428 fprintf(fp, "Dims - %d %d %d\n",dimNX,dimNY,dimNZ);
00429 fprintf(fp, "Rank - x y z t\n");
00430 for(i=0; i<numPes; i++) {
00431 rankToCoordinates(i,x,y,z,t);
00432 fprintf(fp, "%d - %d %d %d %d\n",i,x,y,z,t);
00433 }
00434 }
00435
00436 #if XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00437 extern "C" void craynid_init();
00438 #endif
00439
00440 void TopoManager_init()
00441 {
00442 #if XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
00443 craynid_init();
00444 #endif
00445 }
00446
00447 extern "C" int CmiGetHopsBetweenRanks(int pe1, int pe2)
00448 {
00449 TopoManager topomgr;
00450 return topomgr.getHopsBetweenRanks(pe1, pe2);
00451 }
00452
00453
1.5.5