OpenAtom  Version1.5a
MapTable.C
Go to the documentation of this file.
1 /*****************************************************************************
2  * $Source$
3  * $Author$
4  * $Date$
5  * $Revision$
6  *****************************************************************************/
7 
8 /** \file MapTable.C
9  *
10  */
11 
12 #include "charm++.h"
13 #include "PeList.h"
14 #include "utility/MapFile.h"
15 #define USE_INT_MAP 1
16 #include "IntMap.h"
17 
18 #include "MapTable.h"
19 extern TopoManager *topoMgr;
20 extern Config config;
21 
22 //////////////////////////////////////////////////////////////////////////////
23 int MapType1::getCentroid(int torusMap) {
24  int points, bestPe;
25  if(torusMap==1) {
26  int sumX=0;
27  int sumY=0;
28  int sumZ=0;
29  int X=0,Y=0,Z=0,T=0;
30  points=0;
31  for(int i=0;i<getXmax();i++)
32  {
33  CkAssert(get(i)>=0);
34  topoMgr->rankToCoordinates(get(i), X, Y, Z, T);
35  sumX+=X;
36  sumY+=Y;
37  sumZ+=Z;
38  points++;
39  }
40  int avgX=sumX/points;
41  int avgY=sumY/points;
42  int avgZ=sumZ/points;
43  bestPe=topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
44  }
45  else {
46  int sum=0;
47  points=0;
48  for(int i=0;i<getXmax();i++)
49  {
50  CkAssert(get(i)>=0);
51  sum+=get(i);
52  points++;
53  }
54  bestPe=sum/points;
55  }
56  return(bestPe);
57 }
58 
59 int MapType2::getCentroid(int torusMap) {
60  int points, bestPe;
61  if(torusMap==1) {
62  int sumX=0;
63  int sumY=0;
64  int sumZ=0;
65  int X=0,Y=0,Z=0,T=0;
66  points=0;
67  for(int i=0;i<getXmax();i++)
68  for(int j=0;j<getYmax();j++) {
69  CkAssert(get(i,j)>=0);
70  topoMgr->rankToCoordinates(get(i, j), X, Y, Z, T);
71  sumX+=X;
72  sumY+=Y;
73  sumZ+=Z;
74  points++;
75  }
76  int avgX=sumX/points;
77  int avgY=sumY/points;
78  int avgZ=sumZ/points;
79  bestPe=topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
80  }
81  else {
82  int sum=0;
83  points=0;
84  for(int i=0;i<getXmax();i++)
85  for(int j=0;j<getYmax();j++) {
86  CkAssert(get(i,j)>=0);
87  sum+=get(i,j);
88  points++;
89  }
90  bestPe=sum/points;
91  }
92  return(bestPe);
93 }
94 
95 int MapType3::getCentroid(int torusMap) {
96  int points, bestPe;
97  if(torusMap==1) {
98  int sumX=0;
99  int sumY=0;
100  int sumZ=0;
101  int X=0,Y=0,Z=0,T=0;
102  points=0;
103  for(int i=0;i<getXmax();i++)
104  for(int j=0;j<getYmax();j++) {
105  for(int k=0;k<getZmax();k++) {
106  CkAssert(get(i,j,k)>=0);
107  topoMgr->rankToCoordinates(get(i, j, k), X, Y, Z, T);
108  sumX+=X;
109  sumY+=Y;
110  sumZ+=Z;
111  points++;
112  }
113  }
114  int avgX=sumX/points;
115  int avgY=sumY/points;
116  int avgZ=sumZ/points;
117  bestPe=topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
118  }
119  else {
120  int sum=0;
121  points=0;
122  for(int i=0;i<getXmax();i++)
123  for(int j=0;j<getYmax();j++){
124  for(int k=0;k<getZmax();k++){
125  CkAssert(get(i,j,k)>=0);
126  sum+=get(i,j,k);
127  points++;
128  }
129  }
130  bestPe=sum/points;
131  }
132  return(bestPe);
133 }
134 
135 void IntMap1::translate(IntMap1 *fromMap, int offsetX, int offsetY, int offsetZ, bool torus )
136 {
137 
138  keyXmax=fromMap->keyXmax;
139  CkAssert(keyXmax>0);
140  Map= new int[keyXmax];
141  if(torus)
142  {
143  int x, y, z, t, destpe;
144  for(int xind=0; xind<keyXmax; xind++)
145  {
146  topoMgr->rankToCoordinates(fromMap->get(xind), x, y, z, t);
147  int newx=(x+offsetX)%topoMgr->getDimNX();
148  int newy=(y+offsetY)%topoMgr->getDimNY();
149  int newz=(z+offsetZ)%topoMgr->getDimNZ();
150  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
151  set(xind, destpe);
152  }
153  }
154  else
155  {
156  for(int xind=0; xind<keyXmax; xind++)
157  set(xind,fromMap->get(xind)+offsetX);
158  }
159 };
160 
161 
162 void IntMap2on2::translate(IntMap2on2 *fromMap, int offsetX, int offsetY, int offsetZ, bool torus )
163 {
164  keyXmax=fromMap->keyXmax;
165  keyYmax=fromMap->keyYmax;
166  CkAssert(keyXmax>0);
167  CkAssert(keyYmax>0);
168  CkAssert(keyXmax<10000000);
169  CkAssert(keyYmax<10000000);
170  Map= new int*[keyXmax];
171 
172  int *mapbuf=new int[keyXmax*keyYmax];
173  for(int x=0;x<keyXmax;x++)
174  {
175  Map[x] = mapbuf + keyYmax * x;
176  memset(Map[x],-1,keyYmax*sizeof(int));
177  }
178 
179  if(torus)
180  {
181  int x, y, z, t, destpe;
182  for(int xind=0; xind<keyXmax; xind++)
183  for(int yind=0; yind<keyYmax; yind++) {
184  topoMgr->rankToCoordinates(fromMap->get(xind, yind), x, y, z, t);
185  int newx=(x+offsetX)%topoMgr->getDimNX();
186  int newy=(y+offsetY)%topoMgr->getDimNY();
187  int newz=(z+offsetZ)%topoMgr->getDimNZ();
188  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
189  set(xind, yind, destpe);
190  }
191  }
192  else
193  {
194  for(int xind=0; xind<keyXmax; xind++)
195  for(int yind=0; yind<keyYmax; yind++) {
196  set(xind, yind,fromMap->get(xind, yind)+offsetX);
197  }
198  }
199 };
200 
201 void IntMap3::translate(IntMap3 *fromMap, int offsetX, int offsetY, int offsetZ, bool torus )
202 {
203 
204  keyXmax=fromMap->keyXmax;
205  keyYmax=fromMap->keyYmax;
206  keyZmax=fromMap->keyZmax;
207  CkAssert(keyXmax>0);
208  CkAssert(keyYmax>0);
209  CkAssert(keyZmax>0);
210  Map=new int**[keyXmax];
211  int **mappointbuf = new int*[keyXmax*keyYmax];
212  int *mapbuf= new int[keyXmax*keyYmax*keyZmax];
213  for(int x=0;x<keyXmax;x++)
214  {
215  Map[x]= mappointbuf + (x*keyYmax);
216  for(int y=0;y<keyYmax;y++)
217  {
218  Map[x][y]= mapbuf + (x*keyYmax+y)*keyZmax;
219  memset(Map[x][y],-1,keyZmax*sizeof(int));
220  }
221  }
222  if(torus)
223  {
224  int x, y, z, t, destpe;
225  for(int xind=0; xind<keyXmax; xind++)
226  for(int yind=0; yind<keyYmax; yind++) {
227  for(int zind=0; zind<keyZmax; zind++) {
228  topoMgr->rankToCoordinates(fromMap->get(xind, yind,zind), x, y, z, t);
229  int newx=(x+offsetX)%topoMgr->getDimNX();
230  int newy=(y+offsetY)%topoMgr->getDimNY();
231  int newz=(z+offsetZ)%topoMgr->getDimNZ();
232  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
233  set(xind, yind, zind, destpe);
234  }
235  }
236  }
237  else
238  {
239  for(int xind=0; xind<keyXmax; xind++)
240  for(int yind=0; yind<keyYmax; yind++) {
241  for(int zind=0; zind<keyZmax; zind++) {
242  set(xind, yind, zind, fromMap->get(xind, yind,zind)+offsetX);
243  }
244  }
245  }
246 };
247 void IntMap4::translate(IntMap4 *fromMap, int offsetX, int offsetY, int offsetZ, bool torus )
248 {
249  keyWmax=fromMap->keyWmax;
250  keyXmax=fromMap->keyXmax;
251  keyYmax=fromMap->keyYmax;
252  keyZmax=fromMap->keyZmax;
253  keyStep=fromMap->keyStep;
254  CkAssert(keyWmax>0);
255  CkAssert(keyXmax>0);
256  CkAssert(keyYmax>0);
257  CkAssert(keyZmax>0);
258 
259  Map=new int***[keyWmax];
260  int ***mappointpointbuf = new int**[keyWmax*keyXmax];
261  int **mappointbuf = new int*[keyWmax*keyXmax*keyYmax];
262  int *mapbuf= new int[keyWmax*keyXmax*keyYmax*keyZmax];
263  for(int w=0;w<keyWmax;w++)
264  {
265  Map[w]= mappointpointbuf + (w*keyXmax);
266  for(int x=0;x<keyXmax;x++)
267  {
268  Map[w][x]= mappointbuf + (w*keyXmax+x)*keyYmax;
269  for(int y=0;y<keyYmax;y++)
270  Map[w][x][y]= mapbuf + ((w*keyXmax+x)*keyYmax+y)*keyZmax;
271  }
272  }
273  if(torus)
274  {
275  int x, y, z, t, destpe;
276  for(int wind=0; wind<keyWmax; wind++)
277  for(int xind=0; xind<keyXmax; xind++){
278  for(int yind=0; yind<keyYmax; yind++) {
279  for(int zind=0; zind<keyZmax; zind++) {
280  topoMgr->rankToCoordinates(fromMap->get(wind,xind*keyStep, yind*keyStep,zind), x, y, z, t);
281  int newx=(x+offsetX)%topoMgr->getDimNX();
282  int newy=(y+offsetY)%topoMgr->getDimNY();
283  int newz=(z+offsetZ)%topoMgr->getDimNZ();
284  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
285  set(wind, xind*keyStep, yind*keyStep, zind, destpe);
286  }
287  }
288  }
289  }
290  else
291  {
292  for(int wind=0; wind<keyWmax; wind++)
293  for(int xind=0; xind<keyXmax; xind++){
294  for(int yind=0; yind<keyYmax; yind++) {
295  for(int zind=0; zind<keyZmax; zind++) {
296  set(wind, xind*keyStep, yind*keyStep, zind, fromMap->get(wind, xind*keyStep, yind*keyStep,zind)+offsetX);
297  }
298  }
299  }
300  }
301  // The stepTable is translate-invariant. Simply generate the same values
302  if (keyXmax > 0 && keyStep > 0)
303  {
304  if (stepTable) delete [] stepTable;
305  stepTable= new int [keyXmax*keyStep];
306  for(int s=0; s<keyXmax*keyStep; s++)
307  stepTable[s] = s/keyStep;
308  }
309 };
310 
311 
312 AtomMapTable::AtomMapTable(MapType1 *_tomap, PeList *availprocs, int numInst,
313  int _nchareAtoms): nchareAtoms(_nchareAtoms)
314 {
315  maptable = _tomap;
316  if(numInst == 0) // first instance does the hard stuff
317  {
318  for(int element=0; element<nchareAtoms; element++)
319  {
320  if(availprocs->count()==0)
321  availprocs->reset();
322  maptable->set(element,availprocs->findNext());
323  }
324  }
325  else
326  { //
327  CkAbort("this should only called on the first instance");
328  }
329 }
330 
331 
332 GSMapTable::GSMapTable(MapType2 *_frommap, MapType2 *_tomap, PeList *_availprocs,
333  int _nchareG, int _nstates, int _Gstates_per_pe, bool useCuboidMap, int numInst,
334  int offsetX, int offsetY, int offsetZ):
335  nchareG(_nchareG), nstates(_nstates), Gstates_per_pe(_Gstates_per_pe)
336 {
337  reverseMap = NULL;
338  maptable = _tomap;
339  availprocs = _availprocs;
340 
341  /** The first instance creates the map and the other instances just use the
342  * map with a translation
343  */
344  if(numInst == 0) {
345  state_load = 0.0;
346  int l, m, pl, pm, srem, rem, i=0;
347 
348  /** The first thing is to find the size of blocks (chunks) of GSpace chares
349  * which will be put on each processor. This depends on the number of
350  * states to be given to each processor (input by the user)
351  *
352  * <- m ->
353  * <--- nchareG --->
354  * ***************** ^ ^
355  * | | | l
356  * | | | _
357  * | |
358  * | | nstates
359  * | | Y
360  * | | |
361  * | | | |
362  * | | | |
363  * ***************** - |________ X
364  *
365  */
366  l = Gstates_per_pe; // no of states in one chunk
367  pl = nstates / l; // no of procs on y axis
368  if(nstates % l == 0)
369  srem = 0; // remainder states
370  else
371  {
372  while(pow(2.0, (double)i) < pl)
373  i++;
374  // make it same as the nearest smaller power of 2
375  pl = (int) pow(2.0, (double)(i-1));
376  srem = nstates % pl;
377  }
378  pm = availprocs->count() / pl; // no of procs on x axis
379 
380  if(!useCuboidMap && pm == 0) {
381  CkPrintf("Choose a larger Gstates_per_pe than %d such that { no. of processors [%d] / (no. of states [%d] / Gstates_per_pe [%d]) } is > 0 \n",
382  l, availprocs->count(), nstates, l);
383  CkAssert(availprocs->count() / (nstates/l) > 0);
384  }
385  m = nchareG / pm; // no of planes in one chunk
386  rem = nchareG % pm; // remainder of planes left to be mapped
387 
388  planes_per_pe=m;
389 
390  /*if(CkMyPe()==0)
391  {
392  CkPrintf("nstates %d nchareG %d Pes %d\n", nstates, nchareG, availprocs->count());
393  CkPrintf("l %d, m %d pl %d pm %d srem %d rem %d\n", l, m, pl, pm, srem, rem);
394  }*/
395 
396  // Initialize pelist
397  int srcpe=0;
398  if(useCuboidMap)
399  {
400  int *Pecount= new int[config.numPesPerInstance];
401  bzero(Pecount, config.numPesPerInstance *sizeof(int));
402  PeList *exclusionList = NULL;
403 
404  // here we require that nchareG tiles
405  if(config.numPesPerInstance % nchareG != 0)
406  {
407  CkPrintf("To use CuboidMap nchareG %d should be set as a factor of numprocs %d using gExpandFact\n", nchareG, config.numPesPerInstance);
408  CkExit();
409  }
410  int procsPerPlane = config.numPesPerInstance/nchareG;
411  int cubeGstates_per_pe = nstates/procsPerPlane;
412  int charesperpe = nchareG*nstates/config.numPesPerInstance;
413  int cubesrem = nstates%procsPerPlane;
414  if(cubesrem)
415  cubeGstates_per_pe++;
416  if(cubesrem)
417  charesperpe++;
418 
419  CkPrintf("procsPerPlane %d Gstates_per_pe %d remainder %d\n", procsPerPlane, cubeGstates_per_pe, cubesrem);
420  for(int plane=0; plane<nchareG; plane++)
421  {
422  // slice us off our plane's processors
423  PeList *planeProcs=new PeList(availprocs, plane*procsPerPlane, procsPerPlane);
424  if(planeProcs->count()==0)
425  planeProcs->reset();
426  int destpe=planeProcs->findNext();
427  int workingGsPerPe=cubeGstates_per_pe;
428  bool unallocateRem= (cubesrem) ? true: false;
429  // non power of two systems need some exclusion logic
430  for(int state=0;state<nstates;state+=workingGsPerPe)
431  {
432  if(unallocateRem)
433  if(state>=cubesrem)
434  {
435 
436  workingGsPerPe--;
437  unallocateRem=false;
438  //CkPrintf("rem %d complete at state %d gsperpenow %d\n",cubesrem, state, workingGsPerPe);
439  }
440 
441  // we should block these better
442  for(int stateperpe=0;(stateperpe<workingGsPerPe)&&((state+stateperpe)<nstates);stateperpe++)
443  {
444  #ifdef USE_INT_MAP
445  maptable->set(state+stateperpe, plane, destpe);
446  #else
447  maptable->put(intdual(state+stateperpe, plane))=destpe;
448  #endif
449  if(cubesrem)
450  {
451  Pecount[destpe]++;
452  if(Pecount[destpe]>=charesperpe)
453  {
454  if(exclusionList==NULL)
455  {
456  exclusionList=new PeList(1);
457  exclusionList->TheList[0]=destpe;
458  exclusionList->sortIdx[0]=0;
459  }
460  else
461  {
462  exclusionList->mergeOne(destpe);
463  PeList one(1);
464  one.TheList[0]=destpe;
465  *planeProcs - one;
466  planeProcs->reindex();
467  planeProcs->reset();
468  }
469  }
470  if(((stateperpe+1<workingGsPerPe)&&((state+stateperpe+1)<nstates)) || state+workingGsPerPe<nstates)
471  {
472  // we will need another proc from this list
473  destpe=planeProcs->findNext();
474  if(planeProcs->count()==0)
475  {
476  planeProcs->reset();
477  if(planeProcs->count()==0)
478  CkPrintf("GSMap exceeding count on plane %d state %d after pe %d workingGsPerPe %d cubesrem %d\n",state,plane,destpe, workingGsPerPe, cubesrem);
479  }
480  }
481  }
482  }
483  if(!cubesrem)
484  {
485  if(planeProcs->count()==0)
486  planeProcs->reset();
487  destpe=planeProcs->findNext();
488  }
489  /*if(availprocs->count()==0)
490  {
491  CkPrintf("GSMap created on processor %d\n", CkMyPe());
492  dump();
493  CkPrintf("GSMap ran out of nodes on plane %d state %d\n", plane, state);
494  CkExit();
495  }
496  */
497  }
498  delete planeProcs;
499  }
500  delete [] Pecount;
501  if(exclusionList!=NULL)
502  delete exclusionList;
503  }
504  else
505  {
506  int destpe=availprocs->findNext();
507 
508  // foreach statechunk
509  // foreach state in chunk
510  // map it
511  // new pe
512  // done
513  //
514  //else old way
515  for(int ychunk=0; ychunk<nchareG; ychunk=ychunk+m)
516  {
517  if(ychunk==(pm-rem)*m)
518  m=m+1;
519  for(int xchunk=0; xchunk<nstates; xchunk=xchunk+l)
520  {
521  if(xchunk==(pl-srem)*l)
522  l=l+1;
523  for(int state=xchunk; state<xchunk+l && state<nstates; state++)
524  {
525  for(int plane=ychunk; plane<ychunk+m && plane<nchareG; plane++)
526  {
527  #ifdef USE_INT_MAP
528  maptable->set(state, plane,destpe);
529  #else
530  maptable->put(intdual(state, plane))=destpe;
531  #endif
532  }
533  }
534  srcpe=destpe;
535  if(availprocs->count()==0)
536  availprocs->reset();
537  destpe=availprocs->findNext();
538  }
539  }
540  }
541  #ifdef _MAP_DEBUG_
542  CkPrintf("GSMap created on processor %d\n", CkMyPe());
543  dump();
544  int size[2] = {128, 12};
545  MapFile *mf = new MapFile("GSMap", 2, size, config.numPes, "TXYZ", 2, 1, 1, 1);
546  mf->dumpMap(maptable);
547  #endif
548  } else { // not instance 0
549  if(config.torusMap)
550  {
551  int x, y, z, t, destpe;
552  CkPrintf("{%d} GS using offsets X=%d Y=%d Z=%d\n",numInst, offsetX, offsetY, offsetZ);
553  for(int state=0; state<nstates; state++)
554  for(int plane=0; plane<nchareG; plane++) {
555  topoMgr->rankToCoordinates(_frommap->get(state, plane), x, y, z, t);
556  int newx=(x+offsetX)%topoMgr->getDimNX();
557  int newy=(y+offsetY)%topoMgr->getDimNY();
558  int newz=(z+offsetZ)%topoMgr->getDimNZ();
559  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
560  maptable->set(state, plane, destpe);
561  }
562  }
563  else
564  {
565  CkPrintf("WARNING: using co-mapping for instances because I'm too lazy to partition the processors in the non topo case\n");
566  for(int state=0; state<nstates; state++)
567  for(int plane=0; plane<nchareG; plane++) {
568  maptable->set(state,plane,_frommap->get(state, plane));
569  }
570  }
571  }
572 }
573 
574 SCalcMapTable::SCalcMapTable(MapType4 *_map, PeList *_availprocs,
575  int _nstates, int _nchareG, int _grainsize,
576  bool _flag, int _scalc_per_plane,
577  int _planes_per_pe,
578  int _numChunksA,
579  int _numChunksS,
580  MapType2 *gsmap, bool useCuboidMap,
581  bool useCentroid, int boxSize):
582 
583  max_states(_nstates), nchareG(_nchareG),
584  grainsize(_grainsize), symmetric(_flag),
585  scalc_per_plane(_scalc_per_plane), planes_per_pe(_planes_per_pe),
586  numChunksAsym(_numChunksA), numChunksSym(_numChunksS)
587 {
588 
589  int scobjs_per_pe, rem;
590  int count=0, procno=0;
591  int intidx[2];
592  int lesser_scalc = 0;
593  reverseMap=NULL;
594  maptable=_map;
595  availprocs=_availprocs;
596  availprocs->reset();
597  int maxstateindex=max_states/grainsize*grainsize;
598  if(planes_per_pe == 0)
599  CkAbort("Choose a larger nChareG to avoid this crash\n");
600  if(symmetric)
601  {
602  for(int i=1; i<=max_states/grainsize; i++)
603  lesser_scalc += i;
604  scobjs_per_pe = lesser_scalc*nchareG*numChunksSym/availprocs->count();
605  rem = lesser_scalc*nchareG*numChunksSym % availprocs->count();
606  if(rem!=0)
607  scobjs_per_pe+=1;
608 #ifdef _MAP_DEBUG_
609  CkPrintf(" lesser_scalc %d *nchareG %d *numChunksSym %d %% availprocs->count() %d = rem %d and scobjs_per_pe is %d\n", lesser_scalc,scalc_per_plane,nchareG,numChunksAsym , availprocs->count(),rem, scobjs_per_pe);
610 #endif
611 
612  int srcpe=0,destpe=availprocs->findNext();
613  if(availprocs->count()==0)
614  availprocs->reset();
615 
616  //if(CkMyPe()==0) CkPrintf("scobjs_per_pe %d grainsize %d nchareG %d scalc_per_plane %d planes_per_pe %d numChunks %d rem %d\n", scobjs_per_pe, grainsize, nchareG, scalc_per_plane, planes_per_pe, numChunksSym, rem);
617  if(useCuboidMap|| useCentroid)
618  { // in the cuboid map case we place all planes box by box
619  for(int plane=0; plane<nchareG; plane++)
620  { // could restrict list to the gs box here
621 
622  PeList *thisPlaneBox;
623  if(useCentroid)
624  {
625 
626  thisPlaneBox= subListPlane(plane, max_states, gsmap);
627  thisPlaneBox->reset();
628  }
629  else
630  {
631 
632  // CkPrintf("plane %d making pelist from boxSize %d\n",plane,boxSize);
633  thisPlaneBox= new PeList(availprocs, plane*boxSize, boxSize);
634  }
635 
636  // PeList *thisPlaneBox= availprocs;
637  // thisPlaneBox->dump();
638  if(!useCentroid)
639  {
640  destpe=thisPlaneBox->findNext();
641  if(thisPlaneBox->count()==0)
642  thisPlaneBox->reset();
643 
644  }
645  for(int xchunk=0; xchunk<maxstateindex; xchunk=xchunk+grainsize)
646  for(int ychunk=xchunk; ychunk<maxstateindex; ychunk=ychunk+grainsize)
647  { // could find centroid here
648  if(useCentroid)
649  {
650  thisPlaneBox->trimUsed();
651  // CkPrintf("trimmed list for %d %d %d\n", plane, xchunk, ychunk);
652  // thisPlaneBox->dump();
653  sortByCentroid(thisPlaneBox, plane, xchunk, ychunk, grainsize, gsmap);
654  // CkPrintf("sorted by centroid\n");
655  // thisPlaneBox->dump();
656  destpe=thisPlaneBox->findNext();
657  if(thisPlaneBox->count()==0)
658  thisPlaneBox->reset();
659  count=0;
660  }
661  for(int newdim=0; newdim<numChunksSym; newdim++)
662  {
663  if(count<scobjs_per_pe)
664  {
665  // nothing to see here
666  }
667  else
668  { // new partition
669  procno++;
670  if(thisPlaneBox->count()==0)
671  thisPlaneBox->reset();
672 
673  destpe=thisPlaneBox->findNext();
674  if(thisPlaneBox->count()==0)
675  thisPlaneBox->reset();
676 
677  if(rem!=0)
678  if(procno==rem)
679  scobjs_per_pe-=1;
680  count=0;
681 
682  }
683  // CkPrintf("%d %d %d %d mapped to %d\n",plane,xchunk,ychunk,newdim, destpe);
684  CkAssert(destpe<config.numPes);
685 #ifdef USE_INT_MAP
686  maptable->set(plane, xchunk, ychunk, newdim,destpe);
687 #else
688  CkArrayIndex4D idx4d(plane, xchunk, ychunk, newdim);
689  CmiMemcpy(intidx,idx4d.index,2*sizeof(int));
690  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
691 #endif
692 
693  count++;
694 
695  }
696  }
697  delete thisPlaneBox;
698  }
699 
700  }
701  else
702  {
703  for(int pchunk=0; pchunk<nchareG; pchunk=pchunk+planes_per_pe)
704  for(int newdim=0; newdim<numChunksSym; newdim++)
705  for(int xchunk=0; xchunk<maxstateindex; xchunk=xchunk+grainsize)
706  for(int ychunk=xchunk; ychunk<maxstateindex; ychunk=ychunk+grainsize)
707  //for(int newdim=0; newdim<numChunksSym; newdim++)
708  for(int plane=pchunk; plane<pchunk+planes_per_pe && plane<nchareG; plane++)
709  {
710  CkArrayIndex4D idx4d(plane, xchunk, ychunk, newdim);
711  CmiMemcpy(intidx,idx4d.index,2*sizeof(int)); // our 4 shorts are now 2 ints
712 
713  if(count<scobjs_per_pe)
714  {
715  //if(CkMyPe()==0) CkPrintf("plane %d x %d y %d newdim %d = proc %d\n", plane, xchunk, ychunk, newdim, fp.next[0]*x*y+fp.next[1]*x+fp.next[2]);
716  CkAssert(destpe<config.numPes);
717 #ifdef USE_INT_MAP
718  maptable->set(plane, xchunk, ychunk, newdim,destpe);
719 #else
720  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
721 #endif
722 
723  count++;
724  }
725  else
726  {
727  // new partition
728  procno++;
729  srcpe=destpe;
730  if(availprocs->count()==0)
731  availprocs->reset();
732  // availprocs->sortSource(srcpe);
733  destpe=availprocs->findNext();
734  if(availprocs->count()==0)
735  availprocs->reset();
736 
737  if(rem!=0 &&scobjs_per_pe>1)
738  if(procno==rem)
739  scobjs_per_pe-=1;
740  CkAssert(destpe<config.numPes);
741 #ifdef USE_INT_MAP
742  maptable->set(plane, xchunk, ychunk, newdim,destpe);
743 #else
744  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
745 #endif
746  count=0;
747  count++;
748  }
749 
750  }
751 #ifdef _MAP_DEBUG_
752  CkPrintf("Symmetric SCalcMap created on processor %d\n", CkMyPe());
753  dump();
754 #endif
755  } // else not cuboid
756  }
757  else
758  {
759  scobjs_per_pe = scalc_per_plane*nchareG*numChunksAsym/availprocs->count();
760  rem = scalc_per_plane*nchareG*numChunksAsym % availprocs->count();
761 
762 
763  if(rem!=0)
764  scobjs_per_pe+=1;
765 #ifdef _MAP_DEBUG_
766  CkPrintf(" scalc_per_plane %d *nchareG %d *numChunksAsym %d availprocs->count() %d = rem %d and scobjs_per_pe is %d boxSize %d\n", scalc_per_plane,nchareG,numChunksAsym , availprocs->count(),rem, scobjs_per_pe, boxSize);
767 #endif
768  //if(CkMyPe()==0) CkPrintf("scobjs_per_pe %d grainsize %d nchareG %d scalc_per_plane %d planes_per_pe %d numChunksAsym %d rem %d\n", scobjs_per_pe, grainsize, nchareG, scalc_per_plane, planes_per_pe, numChunksAsym, rem);
769  int srcpe=0,destpe=0;
770  if(!useCentroid && !useCuboidMap)
771  destpe=availprocs->findNext();
772  if(availprocs->count()==0)
773  availprocs->reset();
774  if(useCuboidMap|| useCentroid)
775  { // in the cuboid map case we place all planes box by box
776  for(int plane=0; plane<nchareG; plane++)
777  { // could restrict list to the gs box here
778 
779  PeList *thisPlaneBox;
780  if(useCentroid)
781  {
782 
783  thisPlaneBox= subListPlane(plane, max_states, gsmap);
784  // CkPrintf("plane %d making pelist with %d pes from plane's pes\n",plane, thisPlaneBox->count());
785  // thisPlaneBox->dump();
786  }
787  else
788  {
789  // CkPrintf("plane %d making pelist from boxSize %d\n",plane,boxSize);
790  thisPlaneBox= new PeList(availprocs, plane*boxSize, boxSize);
791  }
792 
793  // PeList *thisPlaneBox= availprocs;
794  // thisPlaneBox->dump();
795  if(!useCentroid)
796  {
797  destpe=thisPlaneBox->findNext();
798  if(thisPlaneBox->count()==0)
799  thisPlaneBox->reset();
800  }
801 
802  for(int xchunk=0; xchunk<maxstateindex; xchunk=xchunk+grainsize)
803  for(int ychunk=0; ychunk<maxstateindex; ychunk=ychunk+grainsize)
804  { // could find centroid here
805  if(useCentroid)
806  {
807  thisPlaneBox->trimUsed();
808  // CkPrintf("plane %d xchunk %d ychunk %d trim\n",plane,xchunk,ychunk);
809  // thisPlaneBox->dump();
810  sortByCentroid(thisPlaneBox, plane, xchunk, ychunk, grainsize, gsmap);
811  // CkPrintf("plane %d xchunk %d ychunk %d sortbycentroid\n",plane,xchunk,ychunk);
812  // thisPlaneBox->dump();
813  destpe=thisPlaneBox->findNext();
814  if(thisPlaneBox->count()==0)
815  thisPlaneBox->reset();
816  count=0;
817  }
818  for(int newdim=0; newdim<numChunksAsym; newdim++)
819  {
820 
821  if(count<scobjs_per_pe)
822  {
823  // nothing to see here
824  }
825  else
826  { // new partition
827  procno++;
828  destpe=thisPlaneBox->findNext();
829  if(thisPlaneBox->count()==0)
830  thisPlaneBox->reset();
831 
832  if(rem!=0&& scobjs_per_pe>1)
833  if(procno==rem)
834  scobjs_per_pe-=1;
835  count=0;
836 
837  }
838  // CkPrintf("%d %d %d %d mapped to %d\n",plane,xchunk,ychunk,newdim, destpe);
839 #ifdef USE_INT_MAP
840  maptable->set(plane, xchunk, ychunk, newdim,destpe);
841 #else
842  CkArrayIndex4D idx4d(plane, xchunk, ychunk, newdim);
843  CmiMemcpy(intidx,idx4d.index,2*sizeof(int));
844  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
845 #endif
846 
847  count++;
848 
849  }
850  }
851  delete thisPlaneBox;
852  }
853 
854  }
855  else
856  {
857  for(int pchunk=0; pchunk<nchareG; pchunk=pchunk+planes_per_pe)
858  for(int newdim=0; newdim<numChunksAsym; newdim++)
859  for(int xchunk=0; xchunk<maxstateindex; xchunk=xchunk+grainsize)
860  for(int ychunk=0; ychunk<maxstateindex; ychunk=ychunk+grainsize)
861  for(int plane=pchunk; plane<pchunk+planes_per_pe && plane<nchareG; plane++)
862  {
863  CkArrayIndex4D idx4d(plane, xchunk, ychunk, newdim);
864  CmiMemcpy(intidx,idx4d.index,2*sizeof(int)); // our 4 shorts are now 2 ints
865 
866  if(count<scobjs_per_pe)
867  {
868  //if(CkMyPe()==0) CkPrintf("plane %d x %d y %d newdim %d= proc %d\n", plane, xchunk, ychunk, newdim, assign[0]*x*y+assign[1]*x+assign[2]);
869 #ifdef USE_INT_MAP
870  maptable->set(plane, xchunk, ychunk, newdim, destpe);
871 #else
872  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
873 #endif
874  count++;
875  }
876  else
877  { // new partition
878  procno++;
879  srcpe=destpe;
880  if(availprocs->count()==0)
881  availprocs->reset();
882  // availprocs->sortSource(srcpe);
883  destpe=availprocs->findNext();
884  if(availprocs->count()==0)
885  availprocs->reset();
886 
887  if(rem!=0)
888  if(procno==rem)
889  scobjs_per_pe-=1;
890 #ifdef USE_INT_MAP
891  maptable->set(plane, xchunk, ychunk, newdim,destpe);
892 #else
893  maptable->put(intdual(intidx[0], intidx[1]))=destpe;
894 #endif
895  count=0;
896  count++;
897  }
898 
899 
900  }
901  }
902 #ifdef _MAP_DEBUG_
903  CkPrintf("Asymmetric SCalcMap created on processor %d\n", CkMyPe());
904  dump();
905 #endif
906  }
907 
908 }
909 
910 /** helper function
911  */
912 PeList * rebuildExclusion(int *Pecount, int rsobjs_per_pe)
913 {
914  PeList *exclusionList=NULL;
915  for(int exc=0; exc<config.numPesPerInstance; exc++)
916  {
917  if(Pecount[exc]>=rsobjs_per_pe)
918  {
919  if(exclusionList==NULL)
920  {
921  exclusionList=new PeList(1);
922  exclusionList->TheList[0]=exc;
923  exclusionList->sortIdx[0]=0;
924  }
925  else
926  {
927  exclusionList->mergeOne(exc);
928  }
929  }
930  }
931  return(exclusionList);
932 }
933 
934 
935 RSMapTable::RSMapTable(MapType2 *_frommap, MapType2 *_tomap, PeList *_availprocs,
936  int _nstates, int _sizeZ, int _Rstates_per_pe, bool useCuboidMap, MapType2 *gsmap,
937  int nchareG, int numInst, int offsetX, int offsetY, int offsetZ):
938  nstates(_nstates), sizeZ(_sizeZ),
939  Rstates_per_pe(_Rstates_per_pe)
940 {
941  reverseMap = NULL;
942  maptable = _tomap;
943  availprocs = _availprocs;
944  availprocs->reset();
945 
946  if(numInst == 0) {
947  int l, m, pl, pm, srem, rem, i=0, rsobjs_per_pe;
948  int *Pecount= new int [config.numPes];
949 
950  bzero(Pecount, config.numPes*sizeof(int));
951 
952  rsobjs_per_pe = nstates*sizeZ/config.numPesPerInstance;
953  if(config.useStrictCuboid)
954  rsobjs_per_pe++; // you'll need the wiggle room
955  l = Rstates_per_pe; // no of states in one chunk
956  pl = nstates / l;
957  if(nstates % l == 0)
958  srem = 0;
959  else
960  {
961  while(pow(2.0, (double)i) < pl)
962  i++;
963  pl = (int) pow(2.0, (double)(i-1)); // make it same as the nearest smaller power of 2
964  srem = nstates % pl;
965  }
966  pm = availprocs->count() / pl;
967 
968  if(pm == 0) {
969  CkPrintf("Choose a larger Rstates_per_pe than %d such that { no. of processors [%d] / (no. of states [%d] / Rstates_per_pe [%d]) } is > 0 \n",
970  l, availprocs->count(), nstates, l);
971  CkAssert(availprocs->count() / (nstates/l) > 0);
972  }
973 
974  m = sizeZ / pm;
975  rem = sizeZ % pm;
976 
977  //CkPrintf("nstates %d sizeZ %d Pes %d rsobjs_per_pe %d\n", nstates, sizeZ, availprocs->count(), rsobjs_per_pe);
978  //CkPrintf("l %d, m %d pl %d pm %d srem %d rem %d\n", l, m,
979  // pl, pm, srem, rem);
980  int srcpe=0;
981  int destpe=availprocs->findNext();
982 
983  if(availprocs->count()==0)
984  availprocs->reset();
985  if(useCuboidMap)
986  {
987  int srem = (nstates*sizeZ) % config.numPesPerInstance;
988  // if(srem)
989  // rsobjs_per_pe++;
990 
991  // exclusion mapping has the sad side effect of increasing the
992  // number of exclusions with the state and plane number
993  // until you end up increasing the cap too high
994  // Topo mapping is less important than even distribution
995  // so if you go over the cap, use the master list instead of the
996  // state box.
997 
998  // this has the effect of creating an imbalance
999 
1000  PeList *myavail=new PeList(*availprocs);
1001  PeList *exclusionList = NULL;
1002  for(int state=0; state < nstates; state++)
1003  {
1004 
1005  int srsobjs_per_pe=rsobjs_per_pe;
1006  PeList *thisStateBox = subListState(state, nchareG, gsmap);
1007  int samplePe=thisStateBox->TheList[0];
1008  // CkPrintf("RS state %d box has %d procs rsobjs_per_pe %d\n",state,thisStateBox->count(), rsobjs_per_pe);
1009  // bool useExclude=false;
1010  bool useExclude=true;
1011  if(exclusionList!=NULL && useExclude)
1012  {
1013  *thisStateBox - *exclusionList;
1014  thisStateBox->reindex();
1015  thisStateBox->reset();
1016  }
1017  //CkPrintf("RS state %d box has %d procs after exclusion\n",state,thisStateBox->count());
1018  // CkPrintf("RS state %d pe list after exclude \n",state);
1019  // thisStateBox->dump();
1020  if(thisStateBox->count()<=0)
1021  {
1022  if(config.useStrictCuboid)
1023  {
1024  // use old scheme of bumping srsobjs_per_pe
1025  while(thisStateBox->count()<=0 && srsobjs_per_pe<=sizeZ*nstates)
1026  {
1027  CkPrintf("State %d Ran out of procs in RS centroid map increasing rs objects per proc to %d\n",state,srsobjs_per_pe);
1028  srsobjs_per_pe++;
1029  if(exclusionList!=NULL)
1030  delete exclusionList;
1031  exclusionList = rebuildExclusion(Pecount, srsobjs_per_pe);
1032  delete thisStateBox;
1033  thisStateBox = subListState(state, nchareG, gsmap);
1034  if(exclusionList!=NULL){
1035  *thisStateBox - *exclusionList;
1036  thisStateBox->reindex();
1037  thisStateBox->reset();
1038  }
1039 
1040  }
1041  }
1042  else
1043 
1044  {
1045  CkPrintf("State %d Ran out of procs in RS centroid map scheme, spilling over to master list\n",state,srsobjs_per_pe);
1046  // *myavail - *exclusionList;
1047  delete thisStateBox;
1048  thisStateBox=new PeList(*myavail);
1049  thisStateBox->reindex();
1050  thisStateBox->reset();
1051  thisStateBox->sortSource(samplePe);
1052  }
1053  }
1054  if(thisStateBox->count()==0)
1055  {
1056  useExclude=false;
1057  CkPrintf("RS state %d excluded to nil, ignoring exclusions\n",state);
1058  delete thisStateBox;
1059  thisStateBox = subListState(state, nchareG, gsmap);
1060  }
1061 
1062  // sort by centroid (not necessary)
1063 
1064  for(int plane=0; plane < sizeZ; plane++)
1065  {
1066  if(thisStateBox->count()==0)
1067  thisStateBox->reset();
1068  if(thisStateBox->count()<=0)
1069  {
1070  if(config.useStrictCuboid)
1071  {
1072  while(thisStateBox->count()<=0 && srsobjs_per_pe<=sizeZ*nstates)
1073  {
1074  srsobjs_per_pe++;
1075  CkPrintf("State %d Plane %d Ran out of procs in RS centroid map increasing rs objects per proc to %d\n",state,plane,srsobjs_per_pe);
1076  if(exclusionList!=NULL)
1077  delete exclusionList;
1078  exclusionList=rebuildExclusion(Pecount, srsobjs_per_pe);
1079  delete thisStateBox;
1080  thisStateBox = subListState(state, nchareG, gsmap);
1081  if(exclusionList!=NULL && useExclude)
1082  {
1083  *thisStateBox - *exclusionList;
1084  thisStateBox->reset();
1085  thisStateBox->reindex();
1086  }
1087  }
1088  }
1089  else
1090  {
1091  CkPrintf("State %d Ran out of procs in RS centroid map scheme, spilling over to master list\n",state,srsobjs_per_pe);
1092  if(myavail->count()<=0)
1093  {
1094  srsobjs_per_pe++;
1095  CkPrintf("State %d Ran out of procs in master, bumping srsobjs_per_pe\n",state,srsobjs_per_pe);
1096  delete thisStateBox;
1097  delete myavail;
1098  myavail= new PeList(*availprocs);
1099  if(exclusionList!=NULL)
1100  delete exclusionList;
1101 
1102  exclusionList=rebuildExclusion(Pecount, srsobjs_per_pe);
1103  if(exclusionList!=NULL)
1104  {
1105  *myavail - *exclusionList;
1106  }
1107  thisStateBox = new PeList(*myavail);
1108  }
1109  else
1110  {
1111  delete thisStateBox;
1112  thisStateBox=new PeList(*myavail);
1113  }
1114  thisStateBox->reindex();
1115  thisStateBox->reset();
1116  thisStateBox->sortSource(samplePe);
1117  }
1118  }
1119  if(useExclude && thisStateBox->count()<=0)
1120  { // surrender
1121  useExclude=false;
1122  delete thisStateBox;
1123  CkAbort("RS cuboid map hopeless please examine configuration\n");
1124 
1125  }
1126 
1127  destpe = thisStateBox->findNext();
1128 
1129  #ifdef USE_INT_MAP
1130  maptable->set(state, plane, destpe);
1131  #else
1132  maptable->put(intdual(state, plane))= destpe;
1133  #endif
1134  // if(CkMyPe()==0) CkPrintf("%d %d [%d]\n", state, plane, destpe);
1135  // CkAssert(destpe < config.numPesPerInstance);
1136  Pecount[destpe]++;
1137  if(Pecount[destpe]>=srsobjs_per_pe)
1138  {
1139  if(exclusionList==NULL)
1140  {
1141  exclusionList=new PeList(1);
1142  exclusionList->TheList[0]=destpe;
1143  exclusionList->sortIdx[0]=0;
1144  }
1145  else
1146  {
1147  exclusionList->mergeOne(destpe);
1148  PeList one(1);
1149  one.TheList[0]=destpe;
1150  *thisStateBox - one;
1151  *myavail - one;
1152  thisStateBox->reindex();
1153  thisStateBox->reset();
1154 
1155  }
1156  }
1157  }
1158  delete thisStateBox;
1159  }
1160  if(exclusionList!=NULL)
1161  delete exclusionList;
1162  if(myavail != NULL)
1163  delete myavail;
1164  }
1165  else
1166  {
1167 
1168  // this remainder scheme is odd, creates imbalance.
1169  // doesn't use all processors
1170  //destpe=availprocs->findNext();
1171  for(int ychunk=0; ychunk<sizeZ-rem; ychunk=ychunk+m)
1172  {
1173  /*if(ychunk==(pm-rem)*m)
1174  m=m+1;*/
1175  for(int xchunk=0; xchunk<nstates; xchunk=xchunk+l)
1176  {
1177  if(xchunk==(pl-srem)*l)
1178  l=l+1;
1179  for(int state=xchunk; state<xchunk+l && state<nstates; state++)
1180  {
1181  for(int plane=ychunk; plane<ychunk+m && plane<sizeZ; plane++)
1182  {
1183  #ifdef USE_INT_MAP
1184  maptable->set(state, plane, destpe);
1185  #else
1186  maptable->put(intdual(state, plane))= destpe;
1187  #endif
1188  }
1189  }
1190  destpe=availprocs->findNext();
1191  if(availprocs->count()==0)
1192  availprocs->reset();
1193 
1194  }
1195  }
1196  if(rem!=0)
1197  for(int state=0; state<nstates; state++)
1198  {
1199  for(int plane=sizeZ-rem; plane<sizeZ; plane++)
1200  {
1201  if(availprocs->count()==0)
1202  availprocs->reset();
1203  destpe=availprocs->findNext();
1204  #ifdef USE_INT_MAP
1205  maptable->set(state, plane, destpe);
1206  #else
1207  maptable->put(intdual(state, plane)) = destpe;
1208  #endif
1209  }
1210  }
1211  }
1212  delete [] Pecount;
1213  #ifdef _MAP_DEBUG_
1214 
1215  CkPrintf("RSMap created on processor %d\n", CkMyPe());
1216  dump();
1217  #endif
1218  } else { // not instance 0
1219  if(config.torusMap==1)
1220  {
1221  int x, y, z, t, destpe;
1222  CkPrintf("{%d} RS using offsets X=%d Y=%d Z=%d\n",numInst, offsetX, offsetY, offsetZ);
1223  for(int state=0; state<nstates; state++)
1224  for(int plane=0; plane<sizeZ; plane++) {
1225  topoMgr->rankToCoordinates(_frommap->get(state, plane), x, y, z, t);
1226  int newx=(x+offsetX)%topoMgr->getDimNX();
1227  int newy=(y+offsetY)%topoMgr->getDimNY();
1228  int newz=(z+offsetZ)%topoMgr->getDimNZ();
1229  destpe = topoMgr->coordinatesToRank(newx, newy, newz, t);
1230  maptable->set(state, plane, destpe);
1231  }
1232  }
1233  else
1234  {
1235  CkPrintf("WARNING: using co-mapping for instances because I'm too lazy to partition the processors in the non topo case\n");
1236  for(int state=0; state<nstates; state++)
1237  for(int plane=0; plane<sizeZ; plane++) {
1238  maptable->set(state,plane,_frommap->get(state, plane));
1239  }
1240  }
1241  }
1242 }
1243 
1244 
1245 RPPMapTable::RPPMapTable(MapType2 *_map,
1246  PeList *_availprocs, PeList *exclusion,
1247  int _nstates, int _sizeZNL, int _Rstates_per_pe,
1248  int boxSize, bool usePPmap, int nchareG,
1249  MapType2 *pp_map) :
1250  nstates(_nstates), sizeZNL(_sizeZNL),
1251  Rstates_per_pe(_Rstates_per_pe)
1252 {
1253  int states_per_pe=Rstates_per_pe;
1254  int totalChares=nstates*sizeZNL;
1255  reverseMap=NULL;
1256  maptable=_map;
1257  availprocs=_availprocs;
1258  // if(useCuboidMap)
1259  // states_per_pe=nstates/boxSize; // no of states in one chunk
1260  // else
1261  // states_per_pe=nstates/boxSize; // no of states in one chunk
1262  // else
1263  bool useExclusion=true;
1264  PeList *RPPlist=availprocs;
1265  int chares_per_pe=totalChares/config.numPesPerInstance;
1266  // pl = nstates / states_per_pe;
1267  CkPrintf("CharesPerPe %d states_per_pe %d\n",chares_per_pe, states_per_pe);
1268  if(exclusion==NULL || exclusion->count()==0 || config.numPesPerInstance <=exclusion->count() )
1269  useExclusion=false;
1270  int afterExclusion=availprocs->count();
1271  if(useExclusion)
1272  afterExclusion=availprocs->count() - exclusion->count();
1273  // CkPrintf("RPP excluded list\n");
1274  // exclusion->dump();
1275  if(useExclusion && afterExclusion > chares_per_pe*sizeZNL)
1276  { // we can fit the exclusion without blinking
1277  CkPrintf("RPP using density exclusion to avoid %d processors\n",exclusion->count());
1278  *RPPlist-*exclusion;
1279  RPPlist->reindex();
1280  }
1281  else
1282  {// so an rstates_per_pe chosen for realstate might be too big
1283  if(useExclusion && afterExclusion > (sizeZNL+nstates)*2)
1284  { // set states_per_pe to fit the exclusion
1285 
1286  states_per_pe=(int) ((float) (totalChares/afterExclusion))*0.75;
1287  CkPrintf("RPP adjusting states per pe from %d to %d to use density exclusion to stay within %d processors\n",Rstates_per_pe, states_per_pe, exclusion->count());
1288  *RPPlist-*exclusion;
1289  RPPlist->reindex();
1290  }
1291  else
1292  {
1293  useExclusion=false;
1294  CkPrintf("RPP with %d chares ignoring density exclusion which left %d out of %d processors\n",totalChares, exclusion->count(), availprocs->count());
1295  }
1296  }
1297 
1298  chares_per_pe=totalChares/RPPlist->count();
1299  // CkPrintf("nstates %d sizeZNL %d Pes %d\n", nstates, sizeZNL, RPPlist->count());
1300  int srcpe=0;
1301  // CkPrintf("nstates %d sizeZNL %d Pes %d\n", nstates, sizeZNL, RPPlist->count());
1302  // RPPlist->dump();
1303  if(usePPmap)
1304  {
1305  /*
1306  * RPP(s,*) <-> PP(s,*)
1307  */
1308  bool neednewexc=false;
1309  // make all the maps
1310  // subtract the exclusion from each
1311  // keep the smallest count as the max charesperpe
1312  // PeList **maps= new PeList* [nstates];
1313  // this code is too memory hoggy
1314  int maxcharesperpe=states_per_pe;
1315  int *usedPes= new int[config.numPes];
1316  bool *useExclude= new bool[nstates];
1317  bzero(usedPes, config.numPes * sizeof(int));
1318  for(int state=0; state < nstates ; state++)
1319  {
1320  // have variable number of exclusions per list
1321  // need to figure out what the max should be
1322  // for the border cases
1323  //maps[state]= subListState( state, nchareG, pp_map);
1324  PeList *state_map=subListState( state, nchareG, pp_map);
1325  CkAssert(state_map->count()>0);
1326  if(useExclusion)
1327  {
1328  *state_map-*exclusion;
1329  state_map->reindex();
1330  if(state_map->count()==0 || sizeZNL/state_map->count() > states_per_pe)
1331  { //not enough for exclusion
1332  delete state_map;
1333  state_map=subListState( state, nchareG, pp_map);
1334  useExclude[state]=false;
1335  }
1336  else
1337  {
1338  useExclude[state]=true;
1339  }
1340  }
1341  else
1342  {
1343  useExclude[state]=false;
1344  }
1345  int thischaresperpe=sizeZNL/state_map->count() + 1;
1346  // CkPrintf("state %d has %d charesperpe in %d pemap \n",state,thischaresperpe,state_map->count());
1347 
1348 
1349  maxcharesperpe=(thischaresperpe>maxcharesperpe) ? thischaresperpe : maxcharesperpe;
1350 
1351  // if(states_per_pe>maxcharesperpe)
1352  // maxcharesperpe=states_per_pe;
1353  delete state_map;
1354  }
1355  int totcharesperpe=sizeZNL*nstates/config.numPesPerInstance+1;
1356  maxcharesperpe=(maxcharesperpe>totcharesperpe) ? maxcharesperpe : totcharesperpe;
1357  // maxcharesperpe++;
1358  PeList *usedbyRPP=NULL;
1359  CkPrintf("RPP maxcharesperpe is %d\n",maxcharesperpe);
1360  int origmaxcharesperpe=maxcharesperpe;
1361  PeList *excludedBigmap=NULL;
1362  for(int state=0; state < nstates ; state++)
1363  {
1364  PeList *state_map=subListState( state, nchareG, pp_map);
1365  if(useExclude[state])
1366  {
1367  *state_map-*exclusion;
1368  state_map->reindex();
1369  }
1370  while(state_map->count()<=0 && maxcharesperpe<=sizeZNL*nstates)
1371  {
1372  if(config.useStrictCuboid)
1373  {
1374  CkPrintf("State %d Ran out of procs in RPP centroid map increasing rpp objects per proc to %d\n",state,maxcharesperpe);
1375  maxcharesperpe++;
1376  if(usedbyRPP!=NULL)
1377  delete usedbyRPP;
1378  usedbyRPP=rebuildExclusion(usedPes, maxcharesperpe);
1379  delete state_map;
1380  state_map = subListState(state, nchareG, pp_map);
1381  if(usedbyRPP!=NULL){
1382  *state_map - *usedbyRPP;
1383  state_map->reindex();
1384  state_map->reset();
1385  }
1386  }
1387  else
1388  {
1389  // ditch topo scheme for overflow
1390  // use the RPPlist
1391  CkPrintf("State %d Ran out of procs in RPP centroid using full RPPlist\n",state);
1392  delete state_map;
1393  if(!neednewexc && excludedBigmap!=NULL)
1394  {
1395  state_map= new PeList(*excludedBigmap);
1396  }
1397  else
1398  {
1399  state_map = new PeList(*RPPlist);
1400  if(usedbyRPP!=NULL){
1401  *state_map - *usedbyRPP;
1402  state_map->reindex();
1403  state_map->reset();
1404  if(excludedBigmap!=NULL)
1405  delete excludedBigmap;
1406  excludedBigmap=new PeList(*state_map);
1407  }
1408  }
1409  if(state_map->count()<=0)
1410  {
1411  maxcharesperpe++;
1412  CkPrintf("plane %d Ran out of procs in RPP centroid using full RPPlist and bumping maxcharesperpe to %d\n",state, maxcharesperpe);
1413  if(usedbyRPP!=NULL)
1414  delete usedbyRPP;
1415  usedbyRPP=rebuildExclusion(usedPes, maxcharesperpe);
1416  delete state_map;
1417  state_map = new PeList(*RPPlist);
1418  if(usedbyRPP!=NULL && usedbyRPP->count()>0){
1419  *state_map - *usedbyRPP;
1420  state_map->reindex();
1421  state_map->reset();
1422  }
1423  if(excludedBigmap!=NULL)
1424  delete excludedBigmap;
1425  excludedBigmap= new PeList(*state_map);
1426  neednewexc=false;
1427  }
1428  // ADD SORT here
1429  }
1430  state_map->sortSource(pp_map->get(state,0));
1431  }
1432  for(int plane=0; plane < sizeZNL; plane++)
1433  {
1434  if(state_map->count()==0)
1435  state_map->reset();
1436  while(state_map->count()<=0 && maxcharesperpe<=sizeZNL*nstates)
1437  {
1438  if(config.useStrictCuboid)
1439  {
1440  while(state_map->count()<=0 && maxcharesperpe<=sizeZNL*nstates)
1441  {
1442  CkPrintf("State %d Ran out of procs in RPP centroid map increasing rpp objects per proc to %d\n",state,maxcharesperpe);
1443  maxcharesperpe++;
1444  if(usedbyRPP!=NULL)
1445  delete usedbyRPP;
1446  usedbyRPP=rebuildExclusion(usedPes, maxcharesperpe);
1447  delete state_map;
1448  state_map = subListState(state, nchareG, pp_map);
1449  if(usedbyRPP!=NULL){
1450  *state_map - *usedbyRPP;
1451  state_map->reindex();
1452  state_map->reset();
1453  }
1454 
1455  }
1456  }
1457  else
1458  {
1459 
1460  CkPrintf("State %d Ran out of procs in RPP centroid using full RPPlist\n",state);
1461  delete state_map;
1462  if(!neednewexc && excludedBigmap!=NULL&& excludedBigmap->count()>0)
1463  {
1464  state_map= new PeList(*excludedBigmap);
1465  }
1466  else
1467  {
1468  state_map = new PeList(*RPPlist);
1469  if(usedbyRPP!=NULL){
1470  *state_map - *usedbyRPP;
1471  state_map->reindex();
1472  state_map->reset();
1473  if(excludedBigmap!=NULL)
1474  delete excludedBigmap;
1475  if(state_map->count()==0)
1476  { // man we're totally dry here.
1477  // this should be handled in the next block
1478  excludedBigmap=NULL;
1479  }
1480  else
1481  {
1482  excludedBigmap= new PeList(*state_map);
1483  neednewexc=false;
1484  }
1485  }
1486  }
1487  if(state_map->count()<=0)
1488  {
1489  maxcharesperpe++;
1490  CkPrintf("plane %d Ran out of procs in RPP centroid using full RPPlist and bumping maxcharesperpe to %d\n",state, maxcharesperpe);
1491 
1492  if(usedbyRPP!=NULL)
1493  delete usedbyRPP;
1494  usedbyRPP=rebuildExclusion(usedPes, maxcharesperpe);
1495  delete state_map;
1496  state_map = new PeList(*RPPlist);
1497  if(usedbyRPP!=NULL)
1498  {
1499  *state_map - *usedbyRPP;
1500  state_map->reindex();
1501  state_map->reset();
1502  }
1503  if(state_map->count()==0)
1504  { // man we're totally dry here.
1505  // reboot
1506  CkPrintf("plane %d Ran out of procs in RPP centroid using full RPPlist after bumping maxcharesperpe to %d, clearing used list, resetting maxcharesperpe to %d\n",state, maxcharesperpe, origmaxcharesperpe);
1507 
1508 
1509  maxcharesperpe=origmaxcharesperpe;
1510  bzero(usedPes, config.numPes * sizeof(int));
1511  if(usedbyRPP!=NULL)
1512  delete usedbyRPP;
1513  usedbyRPP=rebuildExclusion(usedPes, maxcharesperpe);
1514  delete state_map;
1515  state_map = new PeList(*RPPlist);
1516  if(usedbyRPP!=NULL)
1517  {
1518  *state_map - *usedbyRPP;
1519  state_map->reindex();
1520  state_map->reset();
1521  }
1522  }
1523 
1524  if(excludedBigmap!=NULL)
1525  delete excludedBigmap;
1526  excludedBigmap= new PeList(*state_map);
1527  neednewexc=false;
1528  }
1529  }
1530  state_map->sortSource(pp_map->get(state,0));
1531  state_map->reset();
1532  }
1533  int destpe=state_map->findNext();
1534 #ifdef USE_INT_MAP
1535  maptable->set(state, plane, destpe);
1536 #else
1537  maptable->put(intdual(state, plane))=destpe;
1538 #endif
1539  usedPes[destpe]++;
1540  if(usedPes[destpe]>maxcharesperpe)
1541  {
1542  neednewexc=true;
1543  if(usedbyRPP==NULL)
1544  {
1545  usedbyRPP= new PeList(1);
1546  usedbyRPP->TheList[0]=destpe;
1547  }
1548  else
1549  usedbyRPP->mergeOne(destpe);
1550  usedbyRPP->reindex();
1551  exclusion->mergeOne(destpe);
1552  exclusion->reindex();
1553  PeList thisOne(1);
1554  thisOne.TheList[0]=destpe;
1555  if(excludedBigmap!=NULL)
1556  {
1557  neednewexc=false;
1558  *excludedBigmap-thisOne;
1559  }
1560  *state_map - thisOne;
1561  state_map->reindex();
1562  state_map->reset();
1563  }
1564  }
1565  delete state_map;
1566  }
1567  delete usedbyRPP;
1568  delete [] usedPes;
1569  delete [] useExclude;
1570  if(excludedBigmap!=NULL)
1571  delete excludedBigmap;
1572  }
1573  else
1574  {
1575  int destpe=RPPlist->findNext();
1576  // place by planes up to chares_per_pe to give us a balanced layout
1577  int charesOnThisPe=0;
1578  int rem = totalChares % chares_per_pe;
1579  int starting_cpp=chares_per_pe;
1580  // if(rem) --chares_per_pe;
1581  CkPrintf("chares_per_pe %d rem %d\n",chares_per_pe, rem);
1582  for(int state=0; state < nstates; state++)
1583  {
1584  for(int plane=0; plane < sizeZNL; plane++)
1585  {
1586  // CkPrintf("RPP states_per_pe %d setting %d %d to pe %d\n", states_per_pe, state,plane,destpe);
1587 #ifdef USE_INT_MAP
1588  maptable->set(state, plane, destpe);
1589 #else
1590  maptable->put(intdual(state, plane))=destpe;
1591 #endif
1592  if(++charesOnThisPe==chares_per_pe)
1593  {
1594  destpe=RPPlist->findNext();
1595  charesOnThisPe=0;
1596  if(RPPlist->count()==0)
1597  RPPlist->reset();
1598  }
1599  }
1600  if(rem && RPPlist->count()<2*nstates)
1601  {
1602  chares_per_pe=starting_cpp+1;
1603  // CkPrintf("At state %d count %d bumping chares_per_pe to %d\n",state, RPPlist->count(),chares_per_pe);
1604  }
1605 
1606  }
1607  }
1608 #ifdef _MAP_DEBUG_
1609  CkPrintf("RPPMap created on processor %d\n", CkMyPe());
1610  dump();
1611 #endif
1612 }
1613 
1614 
1615 OrthoMapTable::OrthoMapTable(MapType2 *_map, PeList *_availprocs, int _nstates, int _orthograinsize, MapType4 *scalcmap, int nplanes, int numChunks, int sGrainSize, PeList *exclusionList): nstates(_nstates), orthoGrainSize(_orthograinsize)
1616 {
1617  maptable = _map;
1618  availprocs = _availprocs;
1619  int oobjs_per_pe;
1620  int srcpe = 0, destpe;
1621  bool useSublist=true;
1622  bool useExclude=true;
1623  int maxorthoindex=(nstates/orthoGrainSize-1);
1624  int northo=(maxorthoindex+1)*(maxorthoindex+1);
1625  oobjs_per_pe = northo/(config.numPesPerInstance);
1626  int *Pecount= new int [config.numPesPerInstance];
1627  bzero(Pecount, config.numPesPerInstance*sizeof(int));
1628  int s1 = 0, s2 = 0;
1629  int maxpcstateindex=(nstates/sGrainSize-1)*sGrainSize;
1630  int maxorthostateindex=(nstates/orthoGrainSize-1)*orthoGrainSize;
1631  for(int state1 = 0; state1 <= maxorthostateindex; state1 += orthoGrainSize)
1632  for(int state2 = 0; state2 <= maxorthostateindex; state2 += orthoGrainSize)
1633  {
1634  s1 = (state1/sGrainSize);
1635  s1 = s1 * sGrainSize;
1636  s1 = (s1>maxpcstateindex) ? maxpcstateindex :s1;
1637  s2 = (state2/sGrainSize);
1638  s2 = s2 * sGrainSize;
1639  s2 = (s2>maxpcstateindex) ? maxpcstateindex :s2;
1640  PeList *thisStateBox=NULL;
1641  if(useSublist)
1642  {
1643  thisStateBox = subListState2(s1, s2, nplanes, numChunks, scalcmap);
1644  useExclude = true;
1645  *thisStateBox - *exclusionList;
1646  thisStateBox->reindex();
1647  thisStateBox->reset();
1648  }
1649  else
1650  {
1651  thisStateBox=availprocs;
1652  }
1653  if(thisStateBox->count() == 0)
1654  { // the sublist scheme failed
1655  CkPrintf("Ortho %d %d ignoring SubList\n", state1, state2);
1656  if(thisStateBox!=availprocs)
1657  delete thisStateBox;
1658  thisStateBox = availprocs;
1659  //*thisStateBox - *exclusionList;
1660  //thisStateBox->reindex();
1661  //useSublist=false;
1662  }
1663 
1664  if(thisStateBox->count() == 0)
1665  {
1666  CkPrintf("Ortho %d %d ignoring exclusion\n", state1, state2);
1667  if(thisStateBox!=availprocs)
1668  delete thisStateBox;
1669  thisStateBox = subListState2(s1, s2, nplanes, numChunks, scalcmap);
1670  useExclude = false;
1671  }
1672 
1673  /*sortByCentroid(thisStateBox, nplanes, s1, s2, numChunks, scalcmap);
1674  destpe=thisStateBox->findNext();
1675  if(thisStateBox->count()==0)
1676  thisStateBox->reset();
1677  */
1678 
1679  destpe=minDistCentroid(thisStateBox, nplanes, s1, s2, numChunks, scalcmap);
1680  int os1 = (state1/orthoGrainSize);
1681  os1 = (os1>maxorthoindex) ? maxorthoindex :os1;
1682  int os2=(state2/orthoGrainSize);
1683  os2 = (os2>maxorthoindex) ? maxorthoindex :os2;
1684 #ifdef USE_INT_MAP
1685  maptable->set(os1, os2, destpe);
1686 #else
1687  maptable->put(intdual(os1, os2))=destpe;
1688 #endif
1689  Pecount[destpe]++;
1690  if(Pecount[destpe]>=oobjs_per_pe)
1691  {
1692  exclusionList->mergeOne(destpe);
1693  if(useExclude)
1694  {
1695  PeList one(1);
1696  one.TheList[0]=destpe;
1697  *availprocs - one;
1698  availprocs->reindex();
1699  availprocs->reset();
1700  }
1701  }
1702  if(thisStateBox!=availprocs)
1703  delete thisStateBox;
1704  }
1705  delete [] Pecount;
1706 }
1707 
1708 OrthoHelperMapTable::OrthoHelperMapTable(MapType2 *_map, int _nstates, int _orthograinsize, MapType2 *omap, PeList *_avail, PeList *exclude): nstates(_nstates), orthoGrainSize(_orthograinsize)
1709 {
1710  maptable = _map;
1711  int destpe = 0;
1712  // map orthohelper near but not on ortho by removing all ortho procs
1713  availprocs= new PeList(*_avail);
1714  *availprocs-*exclude;
1715  // CkPrintf("exlude for helpers is \n");
1716  // exclude->dump();
1717  bool useExclude=true;
1718  if(availprocs->count() < ((nstates/orthoGrainSize)*(nstates/orthoGrainSize)))
1719  {
1720  CkPrintf("There aren't enough processors to effectively parallelize orthoHelpers, you should disable orthoHelpers!!!\n");
1721  delete availprocs;
1722  availprocs= new PeList(*_avail);
1723  availprocs->reset();
1724  useExclude=false;
1725  }
1726  int maxorthoindex=(nstates/orthoGrainSize-1);
1727  for(int state1 = 0; state1 <= maxorthoindex; state1++)
1728  for(int state2 = 0; state2 <= maxorthoindex; state2++)
1729  {
1730 
1731 #ifdef USE_INT_MAP
1732  destpe=availprocs->minDist(omap->get(state1, state2));
1733  maptable->set(state1, state2, destpe);
1734 #else
1735  destpe=availprocs->minDist(omap->get(intdual(state1, state2)));
1736  maptable->put(intdual(state1, state2))=destpe;
1737 #endif
1738  if(useExclude)
1739  {
1740 
1741  PeList one(1);
1742  one.TheList[0]=destpe;
1743  *availprocs - one;
1744  availprocs->reindex();
1745  }
1746 
1747  }
1748  delete availprocs;
1749 }
1750 
1751 RhoRSMapTable::RhoRSMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoR, int _rhoRsubplanes, int max_states, bool useCentroid, MapType2 *rsmap, PeList *exclude): nchareRhoR(_nchareRhoR), rhoRsubplanes(_rhoRsubplanes)
1752 {
1753  reverseMap=NULL;
1754  maptable=_map;
1755  availprocs=_availprocs;
1756  int rrsobjs_per_pe, rem;
1757  int srcpe=0;
1758  int numChares=nchareRhoR*rhoRsubplanes;
1759  int pesused=0;
1760  if(availprocs->count()==1)
1761  {
1762  rrsobjs_per_pe= numChares;
1763  rem=0;
1764  }
1765  else
1766  {
1767  rrsobjs_per_pe= numChares/(availprocs->count());
1768  rem = numChares % (availprocs->count());
1769  if(numChares<availprocs->count())
1770  {
1771  rem=0;
1772  rrsobjs_per_pe=1;
1773  }
1774  if(rem!=0)
1775  rrsobjs_per_pe += 1;
1776  }
1777 
1778  if(availprocs->count()==0)
1779  availprocs->reset();
1780  int destpe;
1781  int *Pecount = new int [config.numPes];
1782  bzero(Pecount, config.numPes*sizeof(int));
1783 
1784  //if(CkMyPe()==0) CkPrintf("nchareRhoR %d rrsobjs_per_pe %d rem %d\n", nchareRhoR, rrsobjs_per_pe, rem);
1785  if(useCentroid)
1786  {
1787  PeList *exclusionList = NULL;
1788  //CkAssert(numChares<availprocs->count());
1789 
1790  for(int chunk=0; chunk<nchareRhoR; chunk++)
1791  {
1792  PeList *thisPlaneBox = subListPlane(chunk, max_states, rsmap);
1793  bool useExclude=true;
1794  if(exclusionList!=NULL) {
1795  *thisPlaneBox - *exclusionList;
1796  thisPlaneBox->reindex();
1797  }
1798 
1799  if(thisPlaneBox->count()==0)
1800  {
1801  CkPrintf("Rho RS %d ignoring plane sublist\n",chunk);
1802  thisPlaneBox = new PeList(*availprocs);
1803  if(exclusionList!=NULL) {
1804  *thisPlaneBox - *exclusionList;
1805  thisPlaneBox->reindex();
1806  }
1807  if(thisPlaneBox->count()==0)
1808  {
1809  CkPrintf("Rho RS %d ignoring plane sublist and exclusion\n",chunk);
1810  thisPlaneBox = new PeList(*availprocs);
1811  }
1812  }
1813  sortByCentroid(thisPlaneBox, chunk, max_states, rsmap);
1814  // CkPrintf("RhoR %d has %d procs from RS plane\n",chunk,thisPlaneBox->count());
1815 
1816  destpe=thisPlaneBox->findNext();
1817  if(thisPlaneBox->count()==0)
1818  thisPlaneBox->reset();
1819  for(int subplane=0 ; subplane<rhoRsubplanes ; subplane++)
1820  {
1821 #ifdef USE_INT_MAP
1822  maptable->set(chunk, subplane, destpe);
1823 #else
1824  maptable->put(intdual(chunk, subplane))=destpe;
1825 #endif
1826  Pecount[destpe]++;
1827  if(Pecount[destpe]>=rrsobjs_per_pe)
1828  {
1829  if(exclusionList==NULL)
1830  {
1831  exclusionList=new PeList(1);
1832  exclusionList->TheList[0]=destpe;
1833  }
1834  else
1835  exclusionList->mergeOne(destpe);
1836  if(useExclude && thisPlaneBox->size>1)
1837  {
1838  *thisPlaneBox - *exclusionList;
1839  thisPlaneBox->reindex();
1840  }
1841  sortByCentroid(thisPlaneBox, chunk, max_states, rsmap);
1842  }
1843  destpe=thisPlaneBox->findNext();
1844  if(thisPlaneBox->count()==0)
1845  thisPlaneBox->reset();
1846  }
1847  delete thisPlaneBox;
1848  }
1849  // now include the partially filled processors
1850  for(int i=0; i<config.numPesPerInstance;i++)
1851  {
1852  if(Pecount[i]>0)
1853  {
1854  pesused++;
1855  if(Pecount[i] <rrsobjs_per_pe)
1856  {
1857  if(exclusionList==NULL)
1858  {
1859  exclusionList=new PeList(1);
1860  exclusionList->TheList[0]=i;
1861  }
1862  else
1863  exclusionList->mergeOne(i);
1864  }
1865  }
1866  exclusionList->reindex();
1867  }
1868  if(exclusionList!=NULL)
1869  {
1870  exclude->append(*exclusionList);
1871  delete exclusionList;
1872  }
1873  }
1874  else
1875  {
1876  int nprocs=0, objs=0;
1877  destpe=availprocs->findNext();
1878  if(availprocs->count()==0)
1879  availprocs->reset();
1880  for(int chunk=0; chunk<nchareRhoR; chunk++)
1881  {
1882  for(int subplane=0; subplane<rhoRsubplanes; subplane++)
1883  {
1884  if(rem!=0)
1885  if(nprocs==rem)
1886  rrsobjs_per_pe -= 1;
1887 #ifdef USE_INT_MAP
1888  maptable->set(chunk, subplane, destpe);
1889 #else
1890  maptable->put(intdual(chunk, subplane))=destpe;
1891 #endif
1892  objs++;
1893  if(objs==rrsobjs_per_pe)
1894  {
1895  destpe=availprocs->findNext();
1896  if(availprocs->count()==0)
1897  availprocs->reset();
1898  objs=0;
1899  nprocs++;
1900  }
1901  }
1902  }
1903  }
1904  delete [] Pecount;
1905  CkPrintf("Built RhoRS Map [%d, %d] on %d processors\n",nchareRhoR,rhoRsubplanes, pesused );
1906 #ifdef _MAP_DEBUG_
1907  CkPrintf("RhoRSMap created on processor %d\n", CkMyPe());
1908  dump();
1909 #endif
1910 
1911 }
1912 /**
1913  * RhoG and RhoR are mutually all to all. They should be mapped such
1914  * that they are relatively near but exclusive. Meaning they should
1915  * share no processors if there are enough to go around. Given that
1916  * RhoR is already placed. We can take the centroid of that map. Then
1917  * sort the available list based on distance to that centroid. Giving
1918  * us the cloud of available processors around the RhoRS processors as
1919  * our preferred placement.
1920  */
1921 
1922 RhoGSMapTable::RhoGSMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoG, bool useCentroid, MapType2 *rhorsmap, PeList *exclude): nchareRhoG(_nchareRhoG)
1923 {
1924  reverseMap=NULL;
1925  maptable=_map;
1926  availprocs=_availprocs;
1927  int rgsobjs_per_pe, rem;
1928 
1929  if(availprocs->count()==0)
1930  availprocs->reset();
1931 
1932  //if(CkMyPe()==0) CkPrintf("nchareRhoG %d rgsobjs_per_pe %d rem
1933  //%d\n", nchareRhoG, rgsobjs_per_pe, rem);
1934  if(useCentroid)
1935  {
1936  // get centroid of rsmap use it to sort the avail list
1937  availprocs->sortSource(rhorsmap->getCentroid(config.torusMap));
1938  availprocs->reset();
1939  }
1940  PeList *avail= new PeList(*availprocs);
1941  *avail-*exclude;
1942  if(avail->count()>nchareRhoG)
1943  {
1944  // try an exclusion
1945  CkPrintf("RhoG excluding %d from avail %d\n",exclude->count(), availprocs->count());
1946  *availprocs-*exclude;
1947  availprocs->reindex();
1948  // CkPrintf("avail now %d\n", availprocs->count());
1949  }
1950  else
1951  {
1952  CkPrintf("cannot use exclusion in rhog\n");
1953  availprocs->reset();
1954  }
1955  delete avail;
1956 
1957  if(availprocs->count()==1)
1958  {
1959  rgsobjs_per_pe= nchareRhoG;
1960  rem=0;
1961  }
1962  else
1963  {
1964  rgsobjs_per_pe= nchareRhoG/(availprocs->count());
1965  rem = nchareRhoG % (availprocs->count());
1966  if(nchareRhoG<availprocs->count())
1967  {
1968  rem=0;
1969  rgsobjs_per_pe=1;
1970  }
1971  if(rem!=0)
1972  rgsobjs_per_pe += 1;
1973  }
1974 
1975  int destpe=availprocs->findNext();
1976  if(availprocs->count()==0)
1977  availprocs->reset();
1978 
1979  for(int chunk=0; chunk<nchareRhoG; chunk+=rgsobjs_per_pe)
1980  {
1981  if(rem>1)
1982  if(chunk==rem*rgsobjs_per_pe)
1983  rgsobjs_per_pe -= 1;
1984  for(int i=chunk;((i<chunk+rgsobjs_per_pe)&&(i<nchareRhoG));i++)
1985  {
1986 #ifdef USE_INT_MAP
1987  maptable->set(i, 0, destpe);
1988 #else
1989  maptable->put(intdual(i, 0))=destpe;
1990 #endif
1991  }
1992  exclude->mergeOne(destpe);
1993  if(chunk+1<nchareRhoG)
1994  destpe=availprocs->findNext();
1995  if(availprocs->count()==0)
1996  availprocs->reset();
1997  }
1998 #ifdef _MAP_DEBUG_
1999  CkPrintf("RhoGSMap created on processor %d\n", CkMyPe());
2000  dump();
2001 #endif
2002 }
2003 
2004 
2005 RhoRHartMapTable::RhoRHartMapTable(MapType3 *_map, PeList *_availprocs, int _nchareRhoRHart, int rhoRsubplanes, int nchareHartAtmT,PeList *exclude ): nchareRhoRHart(_nchareRhoRHart)
2006 {
2007  reverseMap=NULL;
2008  maptable=_map;
2009  availprocs=_availprocs;
2010  int rrsobjs_per_pe, rem;
2011  int srcpe=0;
2012  int numChares=nchareRhoRHart*rhoRsubplanes*nchareHartAtmT;
2013  if(availprocs->count()==0)
2014  availprocs->reset();
2015  PeList *avail= new PeList(*availprocs);
2016  *avail-*exclude;
2017  if(avail->count()>numChares)
2018  {
2019  // try an exclusion
2020  CkPrintf("RhoRHart excluding %d from avail %d\n",exclude->count(), availprocs->count());
2021  *availprocs-*exclude;
2022  availprocs->reindex();
2023  // CkPrintf("avail now %d\n", availprocs->count());
2024  }
2025  else
2026  {
2027  // CkPrintf("cannot use exclusion in rhoRhart\n");
2028  availprocs->reset();
2029  }
2030  delete avail;
2031 
2032  if(availprocs->count()==1)
2033  {
2034  rrsobjs_per_pe= numChares;
2035  rem=0;
2036  }
2037  else
2038  {
2039  rrsobjs_per_pe= numChares/(availprocs->count());
2040  rem = numChares % (availprocs->count());
2041  if(numChares<availprocs->count())
2042  {
2043  rrsobjs_per_pe=1;
2044  rem=0;
2045  }
2046  if(rem!=0)
2047  rrsobjs_per_pe += 1;
2048  }
2049 
2050  int destpe=availprocs->findNext();
2051  srcpe=destpe;
2052  if(availprocs->count()==0)
2053  availprocs->reset();
2054 
2055  //if(CkMyPe()==0) CkPrintf("nchareRhoR %d rrsobjs_per_pe %d rem %d\n", nchareRhoRHart, rrsobjs_per_pe, rem);
2056 
2057  int nprocs=0, objs=0;
2058  destpe=availprocs->findNext();
2059  if(availprocs->count()==0)
2060  availprocs->reset();
2061 
2062  for(int atmtype=0; atmtype< nchareHartAtmT;atmtype++)
2063  {
2064  for(int chunk=0; chunk<nchareRhoRHart; chunk++)
2065  {
2066  for(int subplane=0; subplane<rhoRsubplanes; subplane++)
2067  {
2068  if(rem!=0)
2069  if(nprocs==rem)
2070  rrsobjs_per_pe -= 1;
2071 #ifdef USE_INT_MAP
2072  maptable->set(chunk, subplane, atmtype,destpe);
2073 #else
2074  maptable->put(inttriple(chunk, subplane, atmtype))=destpe;
2075 #endif
2076  objs++;
2077  exclude->mergeOne(destpe);
2078  if(objs>=rrsobjs_per_pe)
2079  {
2080 
2081  destpe=availprocs->findNext();
2082  if(availprocs->count()==0)
2083  availprocs->reset();
2084  objs=0;
2085  nprocs++;
2086  }
2087  }
2088  }
2089  }
2090 #ifdef _MAP_DEBUG_
2091  CkPrintf("RhoRHartMap created on processor %d\n", CkMyPe());
2092  dump();
2093 #endif
2094 
2095 }
2096 
2097 RhoGHartMapTable::RhoGHartMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoGHart, int nchareHartAtmT, int useCentroid, MapType3 *rhartmap, PeList *exclude): nchareRhoGHart(_nchareRhoGHart)
2098 {
2099  int npes;
2100  reverseMap=NULL;
2101  maptable=_map;
2102  PeList *availprocs=_availprocs;
2103  PeList *avail= new PeList(*availprocs);
2104  avail->reset();
2105  exclude->reset();
2106  *avail-*exclude;
2107  int numchares=nchareRhoGHart*nchareHartAtmT;
2108  bool excluded=true;
2109  if(avail->count()>numchares)
2110  {
2111  // try an exclusion
2112  CkPrintf("RhoGHart excluding %d from avail %d\n",exclude->count(), availprocs->count());
2113  *availprocs-*exclude;
2114  availprocs->reindex();
2115  }
2116  else
2117  {
2118  excluded=false;
2119  CkPrintf("cannot use exclusion in rhoghart\n");
2120  availprocs->reset();
2121  }
2122  delete avail;
2123  if(availprocs->count()==0)
2124  availprocs->reset();
2125  npes=availprocs->count();
2126  int rghobjs_per_pe, rem;
2127 
2128  if(availprocs->count()==1)
2129  {
2130  rghobjs_per_pe= numchares;
2131  rem=0;
2132  }
2133  else
2134  {
2135  rghobjs_per_pe= numchares/npes;
2136  rem = numchares % npes;
2137  if(numchares < npes)
2138  {
2139  rem=0;
2140  rghobjs_per_pe=1;
2141  }
2142  if(rem!=0)
2143  rghobjs_per_pe += 1;
2144  }
2145  if(useCentroid && excluded && rhartmap!=NULL)
2146  {
2147  // get centroid of rhartmap use it to sort the avail list
2148  availprocs->reset();
2149  availprocs->sortSource(rhartmap->getCentroid(config.torusMap));
2150  availprocs->reset();
2151  }
2152  int destpe=availprocs->findNext();
2153  if(availprocs->count()==0)
2154  availprocs->reset();
2155  for(int atmtype=0; atmtype< nchareHartAtmT;atmtype++)
2156  {
2157  for(int chunk=0; chunk<nchareRhoGHart; chunk+=rghobjs_per_pe)
2158  {
2159  // CkAssert(exclude->exists(destpe)<0);
2160  if(rem>1 && rghobjs_per_pe>1)
2161  if(chunk==rem*rghobjs_per_pe)
2162  rghobjs_per_pe -= 1;
2163  for(int i=chunk;((i<chunk+rghobjs_per_pe)&&(i<nchareRhoGHart));i++)
2164  {
2165 #ifdef USE_INT_MAP
2166  maptable->set(i, atmtype, destpe);
2167 #else
2168  maptable->put(intdual(i, atmtype))=destpe;
2169 #endif
2170  }
2171  exclude->mergeOne(destpe);
2172  // availprocs->sortSource(srcpe);
2173  if(chunk+1<nchareRhoGHart)
2174  destpe=availprocs->findNext();
2175  else
2176  chunk=nchareRhoGHart; //get us out of here
2177  if(availprocs->count()==0)
2178  availprocs->reset();
2179 
2180  }
2181  }
2182 #ifdef _MAP_DEBUG_
2183  CkPrintf("RhoGHartMap created on processor %d\n", CkMyPe());
2184  dump();
2185 #endif
2186 
2187 }
2188 
2189 /* We receive a copy of the RhoRS and then act independently with the
2190  bulk of our communication with VdWGS. Exclusion mapping will try to
2191  put us on different processors from RhoRS. Our connection to RS is
2192  more tenuous, so centroid mapping to RS is not as compelling. */
2193 
2194 VdWRSMapTable::VdWRSMapTable(MapType3 *_map, PeList *_availprocs, int _nchareRhoR, int _rhoRsubplanes, int _nchareVdW, int max_states, PeList *exclude): nchareRhoR(_nchareRhoR), rhoRsubplanes(_rhoRsubplanes)
2195 ,nchareVdW(_nchareVdW) {
2196  reverseMap=NULL;
2197  maptable=_map;
2198  availprocs=_availprocs;
2199  int rvdwobjs_per_pe, rem;
2200  int srcpe=0;
2201  int numChares=nchareRhoR*rhoRsubplanes* nchareVdW;
2202  int pesused=0;
2203  if(availprocs->count()==1)
2204  {
2205  rvdwobjs_per_pe= numChares;
2206  rem=0;
2207  }
2208  else
2209  {
2210  rvdwobjs_per_pe= numChares/(availprocs->count());
2211  rem = numChares % (availprocs->count());
2212  if(numChares<availprocs->count())
2213  {
2214  rem=0;
2215  rvdwobjs_per_pe=1;
2216  }
2217  if(rem!=0)
2218  rvdwobjs_per_pe += 1;
2219  }
2220 
2221  if(availprocs->count()==0)
2222  availprocs->reset();
2223  int destpe;
2224  int nprocs=0, objs=0;
2225  destpe=availprocs->findNext();
2226  if(availprocs->count()==0)
2227  availprocs->reset();
2228  for(int chunk=0; chunk<nchareRhoR; chunk++)
2229  {
2230  for(int subplane=0; subplane<rhoRsubplanes; subplane++)
2231  {
2232  for(int vdw=0; vdw<nchareVdW; vdw++)
2233  {
2234  if(rem!=0)
2235  if(nprocs==rem)
2236  rvdwobjs_per_pe -= 1;
2237 #ifdef USE_INT_MAP
2238  maptable->set(chunk, subplane, vdw, destpe);
2239 #else
2240  maptable->put(inttriple(chunk, subplane,vdw))=destpe;
2241 #endif
2242  objs++;
2243  if(objs==rvdwobjs_per_pe)
2244  {
2245  destpe=availprocs->findNext();
2246  if(availprocs->count()==0)
2247  availprocs->reset();
2248  objs=0;
2249  nprocs++;
2250  }
2251  }
2252  }
2253  }
2254  CkPrintf("Built VdWRS Map [%d, %d, %d]\n",nchareRhoR,rhoRsubplanes, nchareVdW);
2255 #ifdef _MAP_DEBUG_
2256  CkPrintf("VdWRSMap created on processor %d\n", CkMyPe());
2257  dump();
2258 #endif
2259 
2260 }
2261 /**
2262  * VdWG and VdWR are mutually all to all. They should be mapped such
2263  * that they are relatively near but exclusive. Meaning they should
2264  * share no processors if there are enough to go around.
2265 
2266  * Given that VdWR is already placed we could map relative to it. But
2267  * the cross communication within each of R and G could dominate so
2268  * we'll start with a simple map and optimize it when we see how this
2269  * new beast works.
2270 
2271  */
2272 
2273 VdWGSMapTable::VdWGSMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoG, int _nchareVdW, PeList *exclude): nchareRhoG(_nchareRhoG), nchareVdW(_nchareVdW)
2274 {
2275  reverseMap=NULL;
2276  maptable=_map;
2277  availprocs=_availprocs;
2278  int rgsobjs_per_pe, rem;
2279 
2280  if(availprocs->count()==0)
2281  availprocs->reset();
2282  PeList *avail= new PeList(*availprocs);
2283  *avail-*exclude;
2284  int numVdWObjs=nchareRhoG*nchareVdW;
2285  if(avail->count()>numVdWObjs)
2286  {
2287  // try an exclusion
2288  CkPrintf("RhoG excluding %d from avail %d\n",exclude->count(), availprocs->count());
2289  *availprocs-*exclude;
2290  availprocs->reindex();
2291  // CkPrintf("avail now %d\n", availprocs->count());
2292  }
2293  else
2294  {
2295  CkPrintf("cannot use exclusion in rhog\n");
2296  availprocs->reset();
2297  }
2298  delete avail;
2299 
2300  if(availprocs->count()==1)
2301  {
2302  rgsobjs_per_pe= numVdWObjs;
2303  rem=0;
2304  }
2305  else
2306  {
2307  rgsobjs_per_pe= numVdWObjs/(availprocs->count());
2308  rem = numVdWObjs % (availprocs->count());
2309  if(numVdWObjs<availprocs->count())
2310  {
2311  rem=0;
2312  rgsobjs_per_pe=1;
2313  }
2314  if(rem!=0)
2315  rgsobjs_per_pe += 1;
2316  }
2317 
2318  int destpe=availprocs->findNext();
2319  if(availprocs->count()==0)
2320  availprocs->reset();
2321 
2322  int thischunk=0;
2323  for(int i=0; i<nchareRhoG; i++)
2324  {
2325  for(int j=0; j<nchareVdW; j++)
2326  {
2327 
2328 #ifdef USE_INT_MAP
2329  maptable->set(i, j, destpe);
2330 #else
2331  maptable->put(intdual(i, j))=destpe;
2332 #endif
2333  thischunk++;
2334  if(thischunk>rgsobjs_per_pe)
2335  {
2336  exclude->mergeOne(destpe);
2337  destpe=availprocs->findNext();
2338  if(availprocs->count()==0)
2339  availprocs->reset();
2340  }
2341  }
2342  }
2343 #ifdef _MAP_DEBUG_
2344  CkPrintf("VdWGSMap created on processor %d\n", CkMyPe());
2345  dump();
2346 #endif
2347 }
2348 
2349 
2350 
2351 void MapTable2::makeReverseMap()
2352 {
2353 #ifndef USE_INT_MAP
2354  CkHashtableIterator *it=maptable->iterator();
2355  it->seekStart();
2356  intdual *key;
2357  reverseMap= new CkVec <intdual> [config.numPesPerInstance];
2358  while(it->hasNext())
2359  {
2360  it->next((void **) &key);
2361  int proc =maptable->get(key[0]);
2362  reverseMap[proc].push_back(key[0]);
2363  }
2364  delete it;
2365 #endif
2366 }
2367 
2368 
2369 
2370 PeList *subListPlane(int plane, int nstates, MapType2 *smap)
2371 {
2372 
2373  PeList *thisPlane = new PeList(1);
2374  thisPlane->TheList[0]=smap->get(0,plane);
2375  for(int state=1; state<nstates; state++)
2376  {
2377  thisPlane->mergeOne(smap->get(state,plane));
2378  }
2379  thisPlane->reset();
2380  thisPlane->reindex();
2381  return(thisPlane);
2382 }
2383 
2384 PeList *subListState(int state, int nplanes, MapType2 *smap)
2385 {
2386  // CkPrintf("in sublist state %d\n",state);
2387  PeList *thisState= new PeList(1);
2388  thisState->TheList[0]=smap->get(state,0);
2389  thisState->sortIdx[0]=0;
2390  for(int plane=1; plane < nplanes; plane++)
2391  {
2392  int pe=smap->get(state, plane);
2393  thisState->mergeOne(pe);
2394  }
2395  thisState->reindex();
2396  thisState->reset();
2397  return(thisState);
2398 }
2399 
2400 PeList *subListState2(int state1, int state2, int nplanes, int numChunks, MapType4 *smap)
2401 {
2402  PeList *thisState = new PeList(1);
2403  thisState->TheList[0]=smap->get(0, state1, state2, 0);
2404  thisState->sortIdx[0]=0;
2405  for(int plane=0; plane<nplanes; plane++)
2406  for(int chunk=0; chunk<numChunks; chunk++)
2407  {
2408  int pe = smap->get(plane, state1, state2, chunk);
2409  thisState->mergeOne(pe);
2410  }
2411  thisState->reindex();
2412  thisState->reset();
2413  return(thisState);
2414 }
2415 
2416 
2417 
2418 void RhoRSMapTable::sortByCentroid(PeList *avail, int plane, int nstates, MapType2 *rsmap)
2419 {
2420  int points=0, bestPe;
2421  if(config.torusMap==1) {
2422  int sumX=0, sumY=0, sumZ=0;
2423  for(int state=0;state<nstates;state++)
2424  {
2425  int X, Y, Z, T;
2426  topoMgr->rankToCoordinates(rsmap->get(state, plane), X, Y, Z, T);
2427  sumX+=X;
2428  sumY+=Y;
2429  sumZ+=Z;
2430  points++;
2431  }
2432  int avgX=sumX/points;
2433  int avgY=sumY/points;
2434  int avgZ=sumZ/points;
2435  bestPe=topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
2436  avail->sortSource(bestPe);
2437  avail->reset();
2438  }
2439  else {
2440  int sumPe=0;
2441  for(int state=0;state<nstates;state++)
2442  {
2443  sumPe+=rsmap->get(state,plane);
2444  points++;
2445  }
2446  bestPe=sumPe/points;
2447  avail->sortSource(bestPe);
2448  avail->reset();
2449  }
2450 }
2451 
2452 
2453 void SCalcMapTable::sortByCentroid(PeList *avail, int plane, int stateX, int stateY, int grainsize, MapType2 *gsmap)
2454 {
2455  int points=0, bestPe;
2456 
2457  if(config.torusMap==1) {
2458  int sumX=0, sumY=0, sumZ=0;
2459  for(int state=stateX;(state<stateX+grainsize)&&(state<config.nstates);state++)
2460  {
2461  int X, Y, Z, T;
2462  topoMgr->rankToCoordinates(gsmap->get(state, plane), X, Y, Z, T);
2463  sumX+=X;
2464  sumY+=Y;
2465  sumZ+=Z;
2466  points++;
2467  }
2468  for(int state=stateY;(state<stateY+grainsize)&&(state<config.nstates);state++)
2469  {
2470  int X, Y, Z, T;
2471  topoMgr->rankToCoordinates(gsmap->get(state, plane), X, Y, Z, T);
2472  sumX+=X;
2473  sumY+=Y;
2474  sumZ+=Z;
2475  points++;
2476  }
2477  int avgX=sumX/points;
2478  int avgY=sumY/points;
2479  int avgZ=sumZ/points;
2480  bestPe=topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
2481  }
2482  else {
2483  int sumPe=0;
2484  for(int state=stateX;(state<stateX+grainsize)&&(state<config.nstates);state++)
2485  {
2486  sumPe+=gsmap->get(state,plane);
2487  points++;
2488  }
2489  for(int state=stateY;(state<stateY+grainsize)&&(state<config.nstates);state++)
2490  {
2491  sumPe+=gsmap->get(state,plane);
2492  points++;
2493  }
2494  bestPe=sumPe/points;
2495  }
2496  avail->sortSource(bestPe);
2497  avail->reset();
2498 
2499 }
2500 
2501 
2502 void OrthoMapTable::sortByCentroid(PeList *avail, int nplanes, int state1, int state2, int numChunks, MapType4 *smap)
2503 {
2504  int points=0, bestPe;
2505  if(config.torusMap==1) {
2506  int sumX = 0, sumY = 0, sumZ = 0;
2507 
2508  for(int plane=0; plane<nplanes; plane++)
2509  for(int chunk=0; chunk<numChunks; chunk++)
2510  {
2511  int X, Y, Z, T;
2512  topoMgr->rankToCoordinates(smap->get(plane, state1, state2, chunk), X, Y, Z, T);
2513  sumX += X;
2514  sumY += Y;
2515  sumZ += Z;
2516  points++;
2517  }
2518  int avgX = sumX/points;
2519  int avgY = sumY/points;
2520  int avgZ = sumZ/points;
2521  bestPe = topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
2522  avail->sortSource(bestPe);
2523  avail->reset();
2524  }
2525  else {
2526  int sumPe = 0;
2527  for(int plane=0; plane<nplanes; plane++)
2528  for(int chunk=0; chunk<numChunks; chunk++)
2529  {
2530  sumPe += smap->get(plane, state1, state2, chunk);
2531  points++;
2532  }
2533  bestPe = sumPe/points;
2534  avail->sortSource(bestPe);
2535  avail->reset();
2536  }
2537 }
2538 
2539 int OrthoMapTable::minDistCentroid(PeList *avail, int nplanes, int state1, int state2, int numChunks, MapType4 *smap)
2540 {
2541  int points=0, bestPe;
2542  if(config.torusMap==1) {
2543  int sumX = 0, sumY = 0, sumZ = 0;
2544 
2545  for(int plane=0; plane<nplanes; plane++)
2546  for(int chunk=0; chunk<numChunks; chunk++)
2547  {
2548  int X, Y, Z, T;
2549  topoMgr->rankToCoordinates(smap->get(plane, state1, state2, chunk), X, Y, Z, T);
2550  sumX += X;
2551  sumY += Y;
2552  sumZ += Z;
2553  points++;
2554  }
2555  int avgX = sumX/points;
2556  int avgY = sumY/points;
2557  int avgZ = sumZ/points;
2558  bestPe = topoMgr->coordinatesToRank(avgX, avgY, avgZ, 0);
2559  return(avail->minDist(bestPe));
2560  }
2561  else {
2562  int sumPe = 0;
2563  for(int plane=0; plane<nplanes; plane++)
2564  for(int chunk=0; chunk<numChunks; chunk++)
2565  {
2566  sumPe += smap->get(plane, state1, state2, chunk);
2567  points++;
2568  }
2569  bestPe = sumPe/points;
2570  return(avail->minDist(bestPe));
2571  }
2572 }
2573 
PeList
Definition: PeList.h:33
rebuildExclusion
PeList * rebuildExclusion(int *Pecount, int rsobjs_per_pe)
helper function
Definition: MapTable.C:912
MapFile.h
Author: Abhinav S Bhatele Date Created: December 28th, 2006.
GSMapTable::GSMapTable
GSMapTable(MapType2 *_frommap, MapType2 *_tomap, PeList *_availprocs, int _nchareG, int _nstates, int _Gstates_per_pe, bool useCuboidMap, int numInst, int offsetX, int offsetY, int offsetZ)
Definition: MapTable.C:332
IntMap.h
Author: Eric J Bohm Date Created: June 4th, 2006.
PeList::minDist
int minDist(int source)
return a processor at minumum distance from source
Definition: PeList.C:201
IntMap4
Definition: IntMap.h:26
VdWGSMapTable::VdWGSMapTable
VdWGSMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoG, int _nchareVdW, PeList *exclude)
VdWG and VdWR are mutually all to all.
Definition: MapTable.C:2273
MapTable.h
Author: Eric J Bohm Date Created: June 4th, 2006.
RhoGSMapTable::RhoGSMapTable
RhoGSMapTable(MapType2 *_map, PeList *_availprocs, int _nchareRhoG, bool useCentroid, MapType2 *rhorsmap, PeList *exclude)
RhoG and RhoR are mutually all to all.
Definition: MapTable.C:1922
PeList.h
Author: Eric J Bohm Date Created: May 31, 2006.