00001 #include "BGQTorus.h"
00002
00003 #if CMK_BLUEGENEQ
00004
00005 #include "spi/include/kernel/process.h"
00006 #include "spi/include/kernel/location.h"
00007 #include <firmware/include/personality.h>
00008
00009 int *bgq_localNodes = NULL;
00010 CmiNodeLock bgq_lock;
00011 int bgq_isLocalSet = 0;
00012
00013 void bgq_topo_init() {
00014 static int init_done = 0;
00015 if (!init_done) {
00016 bgq_lock = CmiCreateLock();
00017 init_done = 1;
00018 }
00019 }
00020
00021 void bgq_topo_free() {
00022 CmiLock(bgq_lock);
00023 if(bgq_localNodes) free(bgq_localNodes);
00024 bgq_isLocalSet = 0;
00025 CmiUnlock(bgq_lock);
00026 }
00027
00028 void bgq_topo_reset() {
00029 CmiLock(bgq_lock);
00030 bgq_isLocalSet = 0;
00031 CmiUnlock(bgq_lock);
00032 }
00033
00034 BGQTorusManager::BGQTorusManager() {
00035 order[0] = 5;
00036 order[1] = 4;
00037 order[2] = 3;
00038 order[3] = 2;
00039 order[4] = 1;
00040 order[5] = 0;
00041
00042 int numPes = CmiNumPesGlobal();
00043 procsPerNode = Kernel_ProcessCount();
00044 thdsPerProc = CmiMyNodeSize();
00045 hw_NT = procsPerNode*thdsPerProc;
00046
00047 Personality_t pers;
00048 Kernel_GetPersonality(&pers, sizeof(pers));
00049
00050 hw_NA = pers.Network_Config.Anodes;
00051 hw_NB = pers.Network_Config.Bnodes;
00052 hw_NC = pers.Network_Config.Cnodes;
00053 hw_ND = pers.Network_Config.Dnodes;
00054 hw_NE = pers.Network_Config.Enodes;
00055
00056 unsigned int isFile = 0;
00057 Kernel_GetMapping(10, mapping, &isFile);
00058 if(!isFile) {
00059 for(int i = 0; i < 6 ; i++) {
00060 if(mapping[i] != 'T') {
00061 order[5 - i] = mapping[i] - 'A';
00062 } else {
00063 order[5 - i] = 5;
00064 }
00065 }
00066 }
00067
00068
00069 rn_NA = hw_NA;
00070 rn_NB = hw_NB;
00071 rn_NC = hw_NC;
00072 rn_ND = hw_ND;
00073 rn_NE = hw_NE;
00074
00075 int max_t = 0;
00076 if(rn_NA * rn_NB * rn_NC * rn_ND * rn_NE != numPes/hw_NT) {
00077 rn_NA = rn_NB = rn_NC = rn_ND =rn_NE =0;
00078 int rn_NT=0;
00079 int min_a, min_b, min_c, min_d, min_e, min_t;
00080 min_a = min_b = min_c = min_d = min_e = min_t = (~(-1));
00081 int tmp_t, tmp_a, tmp_b, tmp_c, tmp_d, tmp_e;
00082 uint64_t numentries;
00083 BG_CoordinateMapping_t *coord;
00084
00085 int nranks=numPes/thdsPerProc;
00086 coord = (BG_CoordinateMapping_t *) malloc(sizeof(BG_CoordinateMapping_t)*nranks);
00087 Kernel_RanksToCoords(sizeof(BG_CoordinateMapping_t)*nranks, coord, &numentries);
00088
00089 for(int c = 0; c < nranks; c++) {
00090 tmp_a = coord[c].a;
00091 tmp_b = coord[c].b;
00092 tmp_c = coord[c].c;
00093 tmp_d = coord[c].d;
00094 tmp_e = coord[c].e;
00095 tmp_t = coord[c].t;
00096
00097 if(tmp_a > rn_NA) rn_NA = tmp_a;
00098 if(tmp_a < min_a) min_a = tmp_a;
00099 if(tmp_b > rn_NB) rn_NB = tmp_b;
00100 if(tmp_b < min_b) min_b = tmp_b;
00101 if(tmp_c > rn_NC) rn_NC = tmp_c;
00102 if(tmp_c < min_c) min_c = tmp_c;
00103 if(tmp_d > rn_ND) rn_ND = tmp_d;
00104 if(tmp_d < min_d) min_d = tmp_d;
00105 if(tmp_e > rn_NE) rn_NE = tmp_e;
00106 if(tmp_e < min_e) min_e = tmp_e;
00107 if(tmp_t > rn_NT) rn_NT = tmp_t;
00108 if(tmp_t < min_t) min_t = tmp_t;
00109 }
00110 rn_NA = rn_NA - min_a + 1;
00111 rn_NB = rn_NB - min_b + 1;
00112 rn_NC = rn_NC - min_c + 1;
00113 rn_ND = rn_ND - min_d + 1;
00114 rn_NE = rn_NE - min_e + 1;
00115 procsPerNode = rn_NT - min_t + 1;
00116 hw_NT = procsPerNode * thdsPerProc;
00117 free(coord);
00118 }
00119
00120 dimA = rn_NA;
00121 dimB = rn_NB;
00122 dimC = rn_NC;
00123 dimD = rn_ND;
00124 dimE = rn_NE;
00125 dimA = dimA * hw_NT;
00126
00127 dims[0] = rn_NA;
00128 dims[1] = rn_NB;
00129 dims[2] = rn_NC;
00130 dims[3] = rn_ND;
00131 dims[4] = rn_NE;
00132 dims[5] = hw_NT;
00133
00134 torus[0] = ((rn_NA % 4) == 0)? true:false;
00135 torus[1] = ((rn_NB % 4) == 0)? true:false;
00136 torus[2] = ((rn_NC % 4) == 0)? true:false;
00137 torus[3] = ((rn_ND % 4) == 0)? true:false;
00138 torus[4] = true;
00139
00140 populateLocalNodes();
00141 }
00142
00143 void BGQTorusManager::populateLocalNodes() {
00144 if(CmiNumPartitions() == 1) return;
00145 #ifndef __TPM_STANDALONE__
00146 CmiLock(bgq_lock);
00147 if(bgq_isLocalSet) {
00148 CmiUnlock(bgq_lock);
00149 return;
00150 }
00151
00152 if(bgq_localNodes == NULL)
00153 bgq_localNodes = (int *)malloc(CmiNumNodesGlobal()*sizeof(int));
00154
00155 CmiAssert(bgq_localNodes != NULL);
00156
00157 for(int i = 0; i < CmiNumNodesGlobal(); i++)
00158 bgq_localNodes[i] = -1;
00159
00160 for(int i = 0; i < CmiNumNodes(); i++) {
00161 int a, b, c, d, e, t;
00162 int global;
00163
00164 rankToCoordinates(CmiNodeFirst(i), a, b, c, d, e, t);
00165 global = CmiNodeOf(coordinatesToRank(a, b, c, d, e, t));
00166
00167 bgq_localNodes[global] = i;
00168 }
00169
00170 bgq_isLocalSet = 1;
00171
00172 CmiUnlock(bgq_lock);
00173 #endif
00174 }
00175
00176 #endif
00177
1.5.5