00001
00013 #include "3dgridrouter.h"
00014
00015
00016 #define gmap(pe) {if (gpes) pe=gpes[pe];}
00017
00020 #if CMK_COMLIB_USE_VECTORIZE
00021 #define GRIDSENDFN(kid, u1, u2, knpe, kpelist, khndl, knextpe) \
00022 {int len;\
00023 PTvectorlist newmsg;\
00024 newmsg=PeGrid->ExtractAndVectorize(kid, u1, knpe, kpelist);\
00025 if (newmsg) {\
00026 CmiSetHandler(newmsg->msgs[0], khndl);\
00027 CmiSyncVectorSendAndFree(knextpe, -newmsg->count, newmsg->sizes, newmsg->msgs);\
00028 }\
00029 else {\
00030 SendDummyMsg(kid, knextpe, u2);\
00031 }\
00032 }
00033 #else
00034 #define GRIDSENDFN(kid, u1, u2, knpe, kpelist, khndl, knextpe) \
00035 {int len;\
00036 char * newmsg;\
00037 newmsg=PeGrid->ExtractAndPack(kid, u1, knpe, kpelist, &len);\
00038 if (newmsg) {\
00039 CmiSetHandler(newmsg, khndl);\
00040 CmiSyncSendAndFree(knextpe, len, (char *)newmsg);\
00041 }\
00042 else {\
00043 SendDummyMsg(kid, knextpe, u2);\
00044 }\
00045 }
00046 #endif
00047
00048 #define ROWLEN COLLEN
00049
00050 #define RowLen(pe) ColLen3D(pe)
00051 #define PELISTSIZE ((ROWLEN-1)/sizeof(int)+1)
00052
00053 inline int ColLen3D(int npes)
00054 {
00055 int len= (int)cubeRoot((double)npes);
00056
00057 if (npes > (len * len * len)) len++;
00058 return(len);
00059 }
00060
00061 inline int Expect1(int gpe, int gnpes)
00062 {
00063 int i, len=ColLen3D(gnpes);
00064 int pe = gpe % (len * len);
00065
00066 int npes = len * len;
00067 if((gnpes - 1)/(len * len) == gpe / (len * len))
00068 npes = ((gnpes - 1) % (len*len)) + 1;
00069
00070 for (i=len-1;i>=0;i--) {
00071 int myrow=pe/len;
00072 int toprep=i*len;
00073 int offset=pe-myrow*len;
00074 if ((toprep+offset) <= (npes-1)) return(i+1);
00075 }
00076 return 0;
00077
00078 }
00079
00080 inline int Expect2(int gpe, int gnpes) {
00081 int len=RowLen(gnpes);
00082 int myplane = gpe / (len * len);
00083 int lastplane = (gnpes - 1)/(len * len);
00084
00085 if(myplane < lastplane)
00086 return len;
00087
00088 int pe = gpe % (len * len);
00089 int myrow = pe / len;
00090 int lastrow = ((gnpes - 1) % (len * len)) / len;
00091
00092 if (myrow < lastrow)
00093 return len;
00094
00095 int ret = ((gnpes - 1) % (len * len)) - myrow * len + 1;
00096 if(ret < 0)
00097 ret = 0;
00098 return ret;
00099 }
00100
00101
00102 inline int LPMsgExpect(int gpe, int gnpes)
00103 {
00104 int i;
00105 int row = RowLen(gnpes);
00106 int col = ColLen3D(gnpes);
00107 int len = (int)ceil(((double)gnpes) / (row * col));
00108
00109 for (i=len-1;i>=0;i--) {
00110 int toprep=i*(row * col);
00111
00112 if ((toprep + (gpe % (row * col))) <= (gnpes-1)) return(i+1);
00113 }
00114 return(len);
00115 }
00116
00117
00118
00119
00120 D3GridRouter::D3GridRouter(int n, int me, Strategy *parent) : Router(parent)
00121 {
00122 ComlibPrintf("PE=%d me=%d NUMPES=%d\n", CkMyPe(), me, n);
00123
00124 NumPes=n;
00125 MyPe=me;
00126 gpes=NULL;
00127
00128 COLLEN=ColLen3D(NumPes);
00129
00130 recvExpected[0] = 0;
00131 recvExpected[1] = 0;
00132 routerStage = 0;
00133
00134 int myrow = (MyPe % (ROWLEN * COLLEN)) / COLLEN;
00135 int myrep = myrow * COLLEN + MyPe - (MyPe % (ROWLEN * COLLEN));
00136 int numunmappedpes=myrep+ROWLEN-NumPes;
00137 int nummappedpes=ROWLEN;
00138
00139 if (numunmappedpes >0) {
00140 nummappedpes=NumPes-myrep;
00141 int i=NumPes+MyPe-myrep;
00142 while (i<myrep+ROWLEN) {
00143 recvExpected[0] += Expect1(i, NumPes);
00144 i+=nummappedpes;
00145 }
00146 }
00147
00148 if((NumPes % (COLLEN * ROWLEN) != 0) && ((NumPes - 1)/(ROWLEN*COLLEN) - MyPe/(ROWLEN*COLLEN) == 1)){
00149 if(myrep + ROWLEN * COLLEN >= NumPes)
00150 recvExpected[0] += Expect1(MyPe + ROWLEN*COLLEN, NumPes);
00151
00152 if(MyPe + ROWLEN * COLLEN >= NumPes)
00153 recvExpected[1] += Expect2(MyPe + ROWLEN*COLLEN, NumPes);
00154 ComlibPrintf("%d: here\n");
00155 }
00156
00157 recvExpected[0] += Expect1(MyPe, NumPes);
00158 recvExpected[1] += Expect2(MyPe, NumPes);
00159
00160 LPMsgExpected = LPMsgExpect(MyPe, NumPes);
00161
00162
00163 PeGrid = new PeTable(NumPes);
00164
00165 nplanes = (int)ceil(((double)NumPes) / (ROWLEN * COLLEN));
00166
00167 oneplane = new int*[COLLEN];
00168 psize = new int[COLLEN];
00169 for(int count = 0; count < COLLEN; count ++) {
00170 oneplane[count] = new int[nplanes * ROWLEN];
00171
00172 int idx = 0;
00173
00174 for (int j=0;j< ROWLEN;j++)
00175 for(int k = 0; k < nplanes; k++) {
00176 int dest = count * ROWLEN + j + ROWLEN * COLLEN * k;
00177 if(dest < NumPes) {
00178 oneplane[count][idx++] = dest;
00179 }
00180 else break;
00181 }
00182 psize[count] = idx;
00183 }
00184
00185 zline = new int[nplanes];
00186
00187 InitVars();
00188 ComlibPrintf("%d:%d:COLLEN=%d, ROWLEN=%d, recvexpected=%d,%d\n", CkMyPe(), MyPe, COLLEN, ROWLEN, recvExpected[0], recvExpected[1]);
00189 }
00190
00191 D3GridRouter::~D3GridRouter()
00192 {
00193 delete PeGrid;
00194 delete[] zline;
00195 for(int count = 0; count < COLLEN; count ++)
00196 delete[] oneplane[count];
00197
00198 delete[] oneplane;
00199 }
00200
00201 void D3GridRouter :: InitVars()
00202 {
00203 recvCount[0]=0;
00204 recvCount[1]=0;
00205
00206 LPMsgCount=0;
00207 }
00208
00209 void D3GridRouter::NumDeposits(comID, int num)
00210 {
00211 }
00212
00213 void D3GridRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
00214 {
00215 int npe=NumPes;
00216 int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
00217 for (int i=0;i<npe;i++) destpes[i]=i;
00218 EachToManyMulticast(id, size, msg, npe, destpes, more);
00219 }
00220
00221 void D3GridRouter::EachToManyMulticast(comID id, int size, void *msg, int numpes, int *destpes, int more)
00222 {
00223 int i;
00224
00225
00226 if (size) {
00227 PeGrid->InsertMsgs(numpes, destpes, size, msg);
00228 }
00229
00230 if (more) return;
00231
00232 routerStage = 0;
00233 ComlibPrintf("All messages received %d %d\n", CkMyPe(), COLLEN);
00234
00235
00236 int firstproc = MyPe - (MyPe % (ROWLEN * COLLEN));
00237 for (i=0;i<COLLEN;i++) {
00238
00239 ComlibPrintf("ROWLEN = %d, COLLEN =%d first proc = %d\n",
00240 ROWLEN, COLLEN, firstproc);
00241
00242
00243 int nextrowrep = firstproc + i*ROWLEN;
00244 int nextpe = (MyPe % (ROWLEN * COLLEN)) % ROWLEN + nextrowrep;
00245 int nummappedpes=NumPes-nextrowrep;
00246
00247 if (nummappedpes <= 0) {
00248 nextpe -= ROWLEN * COLLEN;
00249 if(nextpe < 0)
00250 continue;
00251 }
00252
00253 if (nextpe >= NumPes) {
00254 int mm=(nextpe-NumPes) % nummappedpes;
00255 nextpe=nextrowrep+mm;
00256 }
00257
00258 if (nextpe == MyPe) {
00259 ComlibPrintf("%d calling recv directly\n", MyPe);
00260 recvCount[0]++;
00261 RecvManyMsg(id, NULL);
00262 continue;
00263 }
00264
00265
00266
00267 gmap(nextpe);
00268 ComlibPrintf("sending to column %d and dest %d in %d\n", i, nextpe, CkMyPe());
00269 GRIDSENDFN(id, 0, 0, psize[i], oneplane[i],CkpvAccess(RouterRecvHandle), nextpe);
00270 }
00271 }
00272
00273 void D3GridRouter::RecvManyMsg(comID id, char *msg)
00274 {
00275 ComlibPrintf("%d recvcount=%d,%d recvexpected = %d,%d\n", MyPe, recvCount[0], recvCount[1], recvExpected[0],recvExpected[1]);
00276 int stage = 0;
00277 if (msg) {
00278 stage = PeGrid->UnpackAndInsert(msg);
00279 recvCount[stage]++;
00280 }
00281
00282 if ((recvCount[0] == recvExpected[0]) && (routerStage == 0)){
00283 routerStage = 1;
00284 int myrow = (MyPe % (ROWLEN * COLLEN)) / COLLEN;
00285 int myrep = myrow*ROWLEN + MyPe - (MyPe % (ROWLEN * COLLEN));
00286 for (int i=0;i<ROWLEN;i++) {
00287 int nextpe = myrep + i;
00288
00289
00290
00291 if(nextpe == MyPe) {
00292 recvCount[1]++;
00293 RecvManyMsg(id, NULL);
00294 continue;
00295 }
00296
00297 if(nextpe >= NumPes) {
00298 nextpe -= ROWLEN * COLLEN;
00299 if(nextpe < 0)
00300 continue;
00301 }
00302
00303 int *pelist = zline;
00304 int k = 0;
00305
00306
00307
00308 for(k = 0; k < nplanes; k++) {
00309 int dest = myrow * ROWLEN + i + ROWLEN * COLLEN * k;
00310
00311 if(dest >= NumPes)
00312 break;
00313 zline[k] = dest;
00314 }
00315
00316
00317 ComlibPrintf("Before gmap %d\n", nextpe);
00318
00319 gmap(nextpe);
00320
00321
00322 ComlibPrintf("%d:sending recv message %d %d\n", MyPe, nextpe, myrep);
00323 GRIDSENDFN(id, 1, 1, k, pelist, CkpvAccess(RouterRecvHandle), nextpe);
00324 }
00325 }
00326
00327 if((recvCount[1] == recvExpected[1]) && (routerStage == 1)){
00328 routerStage = 2;
00329 for (int k=0; k < nplanes; k++) {
00330 int nextpe = (MyPe % (ROWLEN * COLLEN)) + k * ROWLEN * COLLEN;
00331
00332 if (nextpe >= NumPes || nextpe==MyPe) continue;
00333
00334 int gnextpe = nextpe;
00335 int *pelist = &gnextpe;
00336
00337 ComlibPrintf("Before gmap %d\n", nextpe);
00338
00339 gmap(nextpe);
00340
00341
00342
00343 ComlibPrintf("%d:sending proc message %d %d\n", MyPe, nextpe, nplanes);
00344 GRIDSENDFN(id, 2, 2, 1, pelist, CkpvAccess(RouterProcHandle), nextpe);
00345 }
00346 LocalProcMsg(id);
00347 }
00348 }
00349
00350 void D3GridRouter::DummyEP(comID id, int stage)
00351 {
00352 if (stage == 2) {
00353 ComlibPrintf("%d dummy calling lp\n", MyPe);
00354 LocalProcMsg(id);
00355 }
00356 else {
00357 recvCount[stage]++;
00358 ComlibPrintf("%d dummy calling recv\n", MyPe);
00359 RecvManyMsg(id, NULL);
00360 }
00361 }
00362
00363 void D3GridRouter:: ProcManyMsg(comID id, char *m)
00364 {
00365 PeGrid->UnpackAndInsert(m);
00366 ComlibPrintf("%d proc calling lp\n", MyPe);
00367 LocalProcMsg(id);
00368 }
00369
00370 void D3GridRouter:: LocalProcMsg(comID id)
00371 {
00372 ComlibPrintf("%d local procmsg called\n", MyPe);
00373
00374 LPMsgCount++;
00375 PeGrid->ExtractAndDeliverLocalMsgs(MyPe, container);
00376
00377 if (LPMsgCount==LPMsgExpected) {
00378 PeGrid->Purge();
00379 InitVars();
00380 routerStage = 0;
00381 ComlibPrintf("%d:Round Done\n", CkMyPe());
00382 Done(id);
00383 }
00384 }
00385
00386 void D3GridRouter :: SetMap(int *pes)
00387 {
00388 gpes=pes;
00389 }
00390
1.5.5