Charm++: Charm Source Code Documentation

00001 
00007 #include "ParFUM.h"
00008 #include "ParFUM_internals.h"
00009 #include "ParFUM_SA.h"
00010 #include "bulk_adapt_ops.h"
00011 
00012 CProxy_ParFUMShadowArray meshSA;
00013 #ifdef DEBUG
00014 #undef DEBUG
00015 #endif
00016 #define DEBUG(x) x
00017 //#define DEBUG(x) 
00018 
00019 
00020 void ParFUM_SA_Init(int meshId) {
00021   CkArrayID ParfumSAId;
00022   int idx;
00023   int size;
00024   FEM_Mesh_allocate_valid_attr(meshId, FEM_ELEM+0);
00025   FEM_Mesh_allocate_valid_attr(meshId, FEM_NODE);
00026   _registerParFUM_SA();
00027   TCharm *tc=TCharm::get();
00028   MPI_Comm comm = MPI_COMM_WORLD;
00029   MPI_Comm_rank(comm,&idx);
00030   MPI_Comm_size(comm,&size);
00031   if(idx==0) {
00032     CkArrayOptions opts;
00033     opts.bindTo(tc->getProxy()); //bind to the current proxy
00034     ParfumSAId = CProxy_ParFUMShadowArray::ckNew(size, idx, opts);
00035   }
00036   MPI_Bcast(&ParfumSAId, sizeof(CkArrayID), MPI_BYTE, 0, comm);
00037   meshSA = ParfumSAId;
00038   meshSA[idx].insert(size, idx);
00039   FEM_Mesh *m = FEM_Mesh_lookup(meshId,"ParFUM_SA_Init");
00040 
00041   //initializing the lock code
00042   FEM_DataAttribute *lockAttr = (FEM_DataAttribute *) m->node.lookup(FEM_ADAPT_LOCK,"ParFUM_SA_Init");
00043   AllocTable2d<int> &lockTable = lockAttr->getInt();
00044   for(int i=0;i<lockAttr->getMax();i++){
00045     lockTable[i][0] = -1; // locking chunkID
00046     lockTable[i][1] = -1; // locking region
00047   }
00048 
00049   FEMMeshMsg *msg = new FEMMeshMsg(m,tc,meshId); 
00050   meshSA[idx].setFemMesh(msg);
00051   return;
00052 }
00053 
00054 
00055 ParFUMShadowArray::ParFUMShadowArray(int s, int i) {
00056   numChunks = s;
00057   idx = i;
00058   fmMesh = NULL;
00059   regionCount = 0;
00060   pendingLock.localID = -1;  
00061   holdingLock.localID = -1;
00062 }
00063 
00064 ParFUMShadowArray::ParFUMShadowArray(CkMigrateMessage *m) {
00065   tc = NULL;
00066   fmMesh = NULL;
00067   regionCount = 0;
00068   pendingLock.localID = -1;
00069 }
00070 
00071 ParFUMShadowArray::~ParFUMShadowArray() {
00072 }
00073 
00074 
00075 void ParFUMShadowArray::pup(PUP::er &p) {
00076   p|numChunks;
00077   p|idx;
00078   p|tproxy;
00079 }
00080 
00081 enum {FEM_globalID=33};
00082 void ParFUMShadowArray::ckJustMigrated(void) {
00083   ArrayElement1D::ckJustMigrated();
00084   tc = tproxy[idx].ckLocal();
00085   CkVec<TCharm::UserData> &v=tc->sud;
00086   FEM_chunk *c = (FEM_chunk*)(v[FEM_globalID].getData());
00087   fmMesh = c->getMesh("ckJustMigrated");
00088   fmMesh->setParfumSA(this);
00089 }
00090 
00091 void ParFUMShadowArray::setFemMesh(FEMMeshMsg *msg) {
00092   fmMesh = msg->m;
00093   tc = msg->t;
00094   tproxy = tc->getProxy();
00095   fmMesh->setParfumSA(this);
00096   bulkAdapt = new BulkAdapt(msg->meshid,fmMesh,idx,thisProxy);
00097   return;
00098 }
00099 
00104 void ParFUMShadowArray::sort(int *chkList, int chkListSize) {
00105   for(int i=0; i<chkListSize; i++) {
00106     for(int j=i+1; j<chkListSize; j++) {
00107       if(chkList[j] < chkList[i]) {
00108   int tmp = chkList[i];
00109   chkList[i] = chkList[j];
00110   chkList[j] = tmp;
00111       }
00112     }
00113   }
00114   return;
00115 }
00116 
00117 
00118 void uniquify(CkVec<int> &vec){
00119   if(vec.length() != 0){
00120     vec.quickSort(8);
00121     int count=0;
00122     for(int i=1;i<vec.length();i++){
00123       if(vec[count] == vec[i]){
00124       }else{
00125         count++;
00126         if(i != count){
00127           vec[count] = vec[i];
00128         }
00129       }
00130     }
00131     vec.resize(count+1);
00132   }
00133 }
00134 
00135 
00136 bool hasGreaterPrio(RegionID x, RegionID y) {
00137   return ((x.prio > y.prio) || ((x.prio == y.prio) && (x.localID < y.localID)) 
00138       || ((x.prio == y.prio) && (x.localID == y.localID) && (x.chunkID < y.chunkID)));
00139 }
00140 
00144 int ParFUMShadowArray::lockRegion(int numElements,adaptAdj *elements,RegionID *regionID,double prio){
00145   //create a regionID for this region
00146   if (regionID->localID == -1) {
00147     regionID->chunkID = idx;
00148     regionID->localID = regionCount;
00149     regionID->prio = prio;
00150     regionCount++;
00151   }
00152 
00153   CkPrintf("[%d] INCOMING: (%d,%d,%6.4f)\n", idx, regionID->chunkID,regionID->localID,regionID->prio);
00154   CkPrintf("[%d] HOLDING:  (%d,%d,%6.4f)\n", idx, holdingLock.chunkID,holdingLock.localID,holdingLock.prio);
00155   CkPrintf("[%d] PENDING:  (%d,%d,%6.4f)\n", idx, pendingLock.chunkID,pendingLock.localID,pendingLock.prio);
00156 
00157   if (holdingLock.localID != -1) { // someone holds a lock on this chunk
00158     CkAssert((holdingLock == (*regionID)) || (holdingLock.chunkID != regionID->chunkID));
00159     if (holdingLock == (*regionID)) { // this regionID already has the lock
00160       DEBUG(printf("[%d] lockRegion SUCCEEDED: This region: (%d,%d,%6.4f) is the holding lock.\n", 
00161            idx, regionID->chunkID,regionID->localID,regionID->prio));
00162     }
00163     else if (hasGreaterPrio(holdingLock, *regionID)) { // it is locked; fail
00164       DEBUG(printf("[%d] lockRegion FAILED: Partition is locked by higher prio region: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00165            idx,holdingLock.chunkID,holdingLock.localID,holdingLock.prio,
00166            regionID->chunkID,regionID->localID,regionID->prio));
00167       return 0;
00168     }
00169     else { // holder is lower prio
00170       if (pendingLock.localID != -1) { // there is a pending region
00171     if (pendingLock == (*regionID)) {
00172       DEBUG(printf("[%d] lockRegion PENDING: Partition is already pending with: (%d,%d,%6.4f), lock held by (%d,%d,%6.4f)\n", 
00173                idx,regionID->chunkID,regionID->localID,regionID->prio,holdingLock.chunkID,holdingLock.localID,holdingLock.prio));
00174 
00175       return 1;
00176     }
00177     else if (hasGreaterPrio(pendingLock, *regionID)) { //pending has prio, fail
00178       DEBUG(printf("[%d] lockRegion FAILED: Partition has pending lock of higher prio region: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00179                idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00180                regionID->chunkID,regionID->localID,regionID->prio));
00181       return 0;
00182     }
00183     else { // pending is lower priority, replace with this region
00184       pendingLock = (*regionID);
00185       DEBUG(printf("[%d] lockRegion PENDING: Partition has pending lower prio region: (%d,%d,%6.4f) replaced with: (%d,%d,%6.4f)\n", 
00186                idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00187                regionID->chunkID,regionID->localID,regionID->prio));
00188       return 1;
00189     }
00190       }
00191       else { // there is no pending region; make this pending
00192     pendingLock = (*regionID);
00193     DEBUG(printf("[%d] lockRegion PENDING: Partition has no pending region; we have: (%d,%d,%6.4f)\n", 
00194              idx,regionID->chunkID,regionID->localID,regionID->prio));
00195     return 1;
00196       }
00197     }
00198   }
00199   else { // no one holds the lock
00200     if (pendingLock.localID != -1) { // there is a pending lock
00201       CkAssert((pendingLock == (*regionID)) || (pendingLock.chunkID != regionID->chunkID));
00202       if (pendingLock == (*regionID)) { // this region was pending; take the lock
00203     holdingLock = (*regionID);
00204     pendingLock.localID = pendingLock.chunkID = -1;
00205     DEBUG(printf("[%d] lockRegion SUCCEEDED: This region: (%d,%d,%6.4f) was pending, now has lock.\n", 
00206            idx, regionID->chunkID,regionID->localID,regionID->prio));
00207       }
00208       else if (hasGreaterPrio(pendingLock, *regionID)) { // this region has lower prio to pending
00209     DEBUG(printf("[%d] lockRegion FAILED: Pending region should lock: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00210            idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00211            regionID->chunkID,regionID->localID,regionID->prio));
00212     return 0;
00213       }
00214       else { // this region has higher prio; take the lock
00215     holdingLock = (*regionID);
00216     DEBUG(printf("[%d] lockRegion SUCCEEDED: This region gets lock: pending (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00217            idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00218            regionID->chunkID,regionID->localID,regionID->prio));
00219       }
00220     }
00221     else { //no one is pending; take the lock
00222       holdingLock = (*regionID);
00223       DEBUG(printf("[%d] lockRegion SUCCEEDED: This region gets lock with: (%d,%d,%6.4f)\n", 
00224            idx, regionID->chunkID,regionID->localID,regionID->prio));
00225     }
00226   }
00227 
00228   LockRegion *region = new LockRegion;
00229   region->myID = *regionID;
00230   DEBUG(CkPrintf("[%d] LockRegion (%d,%d) created \n",idx,regionID->chunkID,regionID->localID));
00231   regionTable.put(*regionID) = region;
00232   
00233   //collect all the local nodes in this list of elements into one list
00234   collectLocalNodes(numElements,elements,region->localNodes);
00235   
00236   //Try to lock all the local nodes. If any fails unlock all and return
00237   //add this region to the regionTable if the locals have all been locked
00238   FEM_Node &fmNode = fmMesh->node;
00239   IDXL_Side &shared = fmNode.shared;
00240   /*
00241   bool success = lockLocalNodes(region);
00242   DEBUG(CkPrintf("[%d] LockRegion %d lockLocalNodes success=%d  \n",idx,regionID->localID,success));
00243   if(!success){ //we could not lock at least one of the local nodes
00244     unlockLocalNodes(region);
00245     delete region;
00246     pendingLock.localID = -1;
00247     return 0;
00248   }else{ //all local nodes were sucessfully locked
00249     regionTable.put(*regionID) = region;
00250   }
00251   */
00252   
00253   //find list of chunks that share nodes (have a shared idxl) with the local nodes of this region
00254   region->sharedIdxls.push_back(idx);
00255   for(int i=0;i<region->localNodes.length();i++){
00256     int node = region->localNodes[i];
00257     const IDXL_Rec *rec=shared.getRec(node);
00258     if(rec!=NULL){ //Put in the list of shared Nodes
00259       for(int j=0;j<rec->getShared();j++){
00260     region->sharedIdxls.push_back(rec->getChk(j));
00261       }
00262     }
00263   }
00264   uniquify(region->sharedIdxls);    
00265   DEBUG(CkPrintf("[%d] LockRegion %d number of SharedIdxls %d \n",idx,regionID->localID,region->sharedIdxls.length()));
00266   if(region->sharedIdxls.length() == 1){ //no remote chunks are invovled in this region's locking
00267     DEBUG(CkPrintf("[%d] LockRegion %d successfully locked with no remote messages \n",idx,regionID->localID));
00268     pendingLock.localID = -1;
00269     return 2;
00270   }
00271   
00272   /*
00273   //lock the local shared idxls
00274   success = lockSharedIdxls(region);
00275   if(!success){ //we could not lock one of the idxls owned by us
00276     //The idxls that were locked are already been unlocked; we need to unlock only the local nodes
00277     unlockLocalNodes(region);
00278     regionTable.remove(*regionID);
00279     delete region;
00280     pendingLock.localID = -1;
00281     return 0;
00282   }
00283   */
00284 
00285   //separate the list of remote elements by chunk
00286   for(int i=0;i<numElements;i++){
00287     if(elements[i].partID != idx){ //remote element
00288       CkVec<adaptAdj> *list = region->remoteElements.get(elements[i].partID);
00289       if(list == NULL){ //first element of this type
00290     list = new CkVec<adaptAdj>;
00291     DEBUG(printf("[%d] Adding remoteElements %p for partition %d\n",idx,list,elements[i].partID););
00292     region->remoteElements.put(elements[i].partID) = list;
00293     CkAssert(region->remoteElements.get(elements[i].partID) ==list);
00294       }
00295       list->push_back(elements[i]);
00296     }
00297   }
00298   
00299   //send message to lock all the idxls for all the chunks and corresponding remote elements
00300   DEBUG(printf("[%d] Begin locking remote nodes and IDXLs for (%d,%d)...\n", idx, regionID->chunkID, regionID->localID));
00301   for(int i=0;i<region->sharedIdxls.length();i++){
00302     int chunk = region->sharedIdxls[i];
00303     if(chunk == idx) { // local chunk is already locked
00304       continue;
00305     }
00306     CkVec<adaptAdj> *list = region->remoteElements.get(chunk);
00307     adaptAdj *elementsForChunk=NULL;
00308     int numElementsForChunk=0;
00309     if(list != NULL){
00310       elementsForChunk = list->getVec();
00311       numElementsForChunk = list->size();
00312     }
00313     thisProxy[chunk].lockRegionForRemote(region->myID,region->sharedIdxls.getVec(),region->sharedIdxls.size(),elementsForChunk,numElementsForChunk);        
00314   }
00315   region->numReplies = 1; //this chunk itself has locked all its local nodes and idxl
00316   region->success = 2;
00317   region->tid = CthSelf();
00318   CthSuspend();
00319   
00320   if(region->success==2){
00321     DEBUG(printf("[%d] Remote locking successful for (%d,%d)...\n", idx, regionID->chunkID, regionID->localID););
00322     return 2;
00323   }
00324   else if (region->success == 1) {
00325     DEBUG(printf("[%d] Remote locking pending for (%d,%d). Trying again...\n", idx, regionID->chunkID, regionID->localID););
00326     return 1;
00327   }
00328   else {
00329     DEBUG(printf("[%d] Remote locking failed for (%d,%d)...\n", idx, regionID->chunkID, regionID->localID););
00330     unpendRegion(*regionID);
00331     unlockRegion(*regionID);
00332     regionTable.remove(*regionID);
00333     //delete region;
00334     holdingLock.localID = -1;
00335     return 0;
00336   }
00337 }
00338 
00339 
00342 void ParFUMShadowArray::collectLocalNodes(int numElements,adaptAdj *elements,CkVec<int> &localNodes){
00343   for(int i=0;i<numElements;i++){
00344     //check if the element is local
00345     if(elements[i].partID < 0 || elements[i].localID < 0) {
00346       CkAssert(elements[i].partID < 0 && elements[i].localID < 0);
00347       continue;
00348     }
00349     if(elements[i].partID == idx){
00350       int elemType = elements[i].elemType;
00351       const FEM_Elem &elem = fmMesh->getElem(elemType);
00352       const int *conn = elem.connFor(elements[i].localID);
00353       const int numNodes = elem.getNodesPer();
00354       DEBUG(printf("Nodes for local element %d: ",elements[i].localID));
00355       for(int j=0;j<numNodes;j++){
00356         int node = conn[j];
00357         CkAssert(node >= 0);
00358         localNodes.push_back(node);
00359     DEBUG(printf("%d ",node));
00360       }
00361       DEBUG(printf("\n"));
00362     }
00363   }
00364   //now we need to uniquify the list of localNodes
00365   uniquify(localNodes);
00366 }
00367 
00368 
00369 bool ParFUMShadowArray::lockLocalNodes(LockRegion *region){
00370   FEM_Node &fmNode = fmMesh->node;
00371   AllocTable2d<int> &lockTable = ((FEM_DataAttribute *)fmNode.lookup(FEM_ADAPT_LOCK,"lockRegion"))->getInt();
00372   
00373   bool success=true;
00374   for(int i=0;i<region->localNodes.length();i++){
00375     int node = region->localNodes[i];
00376     
00377     if(lockTable[node][0]!= -1){
00378       //the node is locked
00379       DEBUG(printf("[%d] Node %d was locked \n",thisIndex,node));
00380       success = false;
00381       break;
00382     }
00383     else {
00384       //the node is unlocked
00385       lockTable[node][0] = region->myID.chunkID;
00386       lockTable[node][1] = region->myID.localID;
00387       
00388       DEBUG(printf("[%d] Locking Node %d \n",thisIndex,node));
00389     }
00390   }
00391   return success;
00392 }
00393 
00394 
00399 bool ParFUMShadowArray::lockSharedIdxls(LockRegion *region){
00400   bool success=true;
00401   int lockedIdx=-1;
00402   for(int i=0;i<region->sharedIdxls.length();i++){
00403     int chunkID = region->sharedIdxls[i];
00404     if(idx < chunkID){
00405       IDXL_List *list =  fmMesh->node.shared.getIdxlListN(chunkID);
00406       if(list != NULL){
00407     if(list->isLocked()){ //list is locked 
00408       success = false;
00409       lockedIdx= i;
00410       break;
00411     }else{
00412       list->lockIdxl();
00413     }
00414       }
00415     }
00416   }
00417 
00418   if(!success){
00419     for(int i=0;i<lockedIdx;i++){
00420       int chunkID = region->sharedIdxls[i];
00421       if(idx < chunkID){
00422     IDXL_List *list =  fmMesh->node.shared.getIdxlListN(chunkID);
00423     if(list != NULL){
00424       if(list->isLocked()){
00425         list->unlockIdxl();
00426       }
00427     }
00428       }
00429     }
00430   }
00431   return success;
00432 }
00433 
00436 void ParFUMShadowArray::lockRegionForRemote(RegionID regionID,int *sharedIdxls,int numSharedIdxls,adaptAdj *elements,int numElements){
00437   LockRegion *region = new LockRegion;
00438   region->myID = regionID;
00439   static int tag=0;
00440   int success;
00441 
00442   DEBUG(printf("lockRegionForRemote: regionID=(%d,%d,%6.4f) holdingLock=(%d,%d,%6.4f) pendingLock=(%d,%d,%6.4f)\n", regionID.chunkID, regionID.localID, regionID.prio, holdingLock.chunkID, holdingLock.localID, holdingLock.prio, pendingLock.chunkID, pendingLock.localID, pendingLock.prio));
00443   if (holdingLock.localID != -1) { // someone holds a lock on this chunk
00444     CkAssert((holdingLock == regionID) || (holdingLock.chunkID != regionID.chunkID));
00445     if (holdingLock == regionID) { // this regionID already has the lock
00446       success=2;
00447       DEBUG(printf("[%d] lockRegionForRemote SUCCEEDED: This region: (%d,%d,%6.4f) is the holding lock.\n", 
00448            idx, regionID.chunkID,regionID.localID,regionID.prio));
00449     }
00450     else if (hasGreaterPrio(holdingLock, regionID)) { // it is locked; fail
00451       success=0;
00452       DEBUG(printf("[%d] lockRegionForRemote FAILED: Partition is locked by higher prio region: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00453            idx,holdingLock.chunkID,holdingLock.localID,holdingLock.prio,
00454            regionID.chunkID,regionID.localID,regionID.prio));
00455     }
00456     else {
00457       if (pendingLock.localID != -1) {
00458     if (pendingLock == regionID) {
00459       success = 1;
00460       DEBUG(printf("[%d] lockRegion PENDING: Partition is already pending with: (%d,%d,%6.4f), lock held by (%d,%d,%6.4f)\n", 
00461                idx,regionID.chunkID,regionID.localID,regionID.prio,holdingLock.chunkID,holdingLock.localID,holdingLock.prio));
00462     }
00463     else if (hasGreaterPrio(pendingLock, regionID)) {
00464       success=0;
00465       DEBUG(printf("[%d] lockRegion FAILED: Partition has pending lock of higher prio region: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00466                idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00467                regionID.chunkID,regionID.localID,regionID.prio));
00468     }
00469     else {
00470       pendingLock = regionID;
00471       success=1;
00472       DEBUG(printf("[%d] lockRegion PENDING: Partition has pending lower prio region: (%d,%d,%6.4f) replaced with: (%d,%d,%6.4f)\n", 
00473                idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00474                regionID.chunkID,regionID.localID,regionID.prio));
00475     }
00476       }
00477       else { // there is no pending region; make this pending
00478     pendingLock = regionID;
00479     success=1;
00480     DEBUG(printf("[%d] lockRegion PENDING: Partition has no pending region; we have: (%d,%d,%6.4f)\n", 
00481              idx,regionID.chunkID,regionID.localID,regionID.prio));
00482       }
00483     }
00484   }
00485   else { // no one holds the lock
00486     if (pendingLock.localID != -1) { // there is a pending lock
00487       CkAssert((pendingLock == regionID) || (pendingLock.chunkID != regionID.chunkID));
00488       if (pendingLock == regionID) {
00489     holdingLock = regionID;
00490     pendingLock.localID = pendingLock.chunkID = -1;
00491     success=2;
00492     DEBUG(printf("[%d] lockRegion SUCCEEDED: This region: (%d,%d,%6.4f) was pending, now has lock.\n", 
00493            idx, regionID.chunkID,regionID.localID,regionID.prio));
00494       }
00495       else if (hasGreaterPrio(pendingLock, regionID)) {
00496     success=0;
00497     DEBUG(printf("[%d] lockRegion FAILED: Pending region should lock: (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00498              idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00499              regionID.chunkID,regionID.localID,regionID.prio));
00500       }
00501       else {
00502     holdingLock = regionID;
00503     success=2;
00504     DEBUG(printf("[%d] lockRegion SUCCEEDED: This region gets lock: pending (%d,%d,%6.4f) we have: (%d,%d,%6.4f)\n", 
00505              idx,pendingLock.chunkID,pendingLock.localID,pendingLock.prio,
00506              regionID.chunkID,regionID.localID,regionID.prio));
00507 
00508       }
00509     }
00510     else {
00511       holdingLock = regionID;
00512       success=2;
00513       DEBUG(printf("[%d] lockRegion SUCCEEDED: This region gets lock with: (%d,%d,%6.4f)\n", 
00514            idx, regionID.chunkID,regionID.localID,regionID.prio));
00515       
00516     }
00517   }
00518 
00519   /*
00520   //make a list of all the nodes that are specified in the elements
00521   collectLocalNodes(numElements,elements,region->localNodes);
00522   //try locking all the local nodes,
00523   success = 0;
00524   if (lockLocalNodes(region)) {
00525     success = 2;
00526     for(int i=0;i<numSharedIdxls;i++){
00527       region->sharedIdxls.push_back(sharedIdxls[i]);
00528     }
00529     if (!(lockSharedIdxls(region)))
00530       success = 0;
00531   }
00532 
00533   // could not lock either the shared idxls or the local nodes; need to unlock local nodes before replying
00534   if (success == 0) {
00535     unlockLocalNodes(region);
00536     pendingLock.localID = -1;
00537     delete region;
00538   }
00539   else if (success == 1) {
00540     unlockLocalNodes(region);
00541     delete region;
00542   }
00543   else{ //if the locking was successful, store the region
00544     regionTable.put(regionID) = region;
00545   }
00546   */
00547     
00548   if (success == 2) {
00549     regionTable.put(regionID) = region;
00550   }
00551 
00552   DEBUG(CkPrintf("[%d] Lockregion (%d,%d,%6.4f) success=%d sending lockReply (tag=%d) to %d\n",thisIndex,regionID.chunkID,regionID.localID,regionID.prio,success,tag,regionID.chunkID));
00553   //send a reply back to the requesting chunk with the result of the lock attempt
00554   thisProxy[regionID.chunkID].lockReply(idx,regionID,success,tag,success);
00555   DEBUG(CkPrintf("[%d] Lockregion (%d,%d,%6.4f) success=%d sent lockReply (tag=%d) to %d\n",thisIndex,regionID.chunkID,regionID.localID,regionID.prio,success,tag,regionID.chunkID));
00556   tag++;
00557 }
00558 
00565 void ParFUMShadowArray::lockReply(int remoteChunk,RegionID regionID, int foosuccess, int tag, int otherSuccess){
00566   LockRegion *region = regionTable.get(regionID);
00567   DEBUG(CkPrintf("[%d] Lockregion reply (tag=%d) received (%d,%d,%6.4f) foosuccess=%d otherSuccess=%d from chunk %d region=%x\n",thisIndex,tag,regionID.chunkID,regionID.localID,regionID.prio,foosuccess, otherSuccess, remoteChunk, region));
00568   CkAssert(region != NULL);
00569   region->numReplies++;
00570   if (region->success > otherSuccess)
00571     region->success = otherSuccess;
00572   if(region->numReplies == region->sharedIdxls.size()){
00573     if(region->success == 2){
00574       CthAwaken(region->tid);
00575     }
00576     else if (region->success == 1) {
00577       CthAwaken(region->tid);
00578     }   
00579     else {
00580       unlockRegion(region);
00581     }   
00582   }
00583 }
00584 
00585 void ParFUMShadowArray::unpendRegion(RegionID regionID){
00586   CkAssert(regionID.chunkID == idx);
00587   LockRegion *region = regionTable.get(regionID);
00588   if (region!=NULL) {
00589     for(int i=0;i<region->sharedIdxls.length();i++){
00590       int chunk = region->sharedIdxls[i];
00591       if(chunk != idx){
00592     thisProxy[chunk].unpendForRemote(region->myID);
00593       }
00594     }
00595     region->numReplies = 1; //this chunk has unlocked all the stuff on it
00596     if(region->sharedIdxls.length() == 1){ //there are no remote chunks involved in this region
00597     }else{ //wait for unlock to finish on other chunks
00598       CthSuspend();
00599     }
00600     regionTable.remove(regionID);
00601     delete region;
00602   }
00603   if (holdingLock == regionID) 
00604     holdingLock.localID = -1;
00605   if (pendingLock == regionID)
00606     pendingLock.localID = -1;
00607 }
00608 
00609 void ParFUMShadowArray::unpendForRemote(RegionID regionID){
00610   LockRegion *region = regionTable.get(regionID);
00611   if (region != NULL) {
00612     delete region;
00613     regionTable.remove(regionID);
00614   }
00615   if (holdingLock == regionID) 
00616     holdingLock.localID = -1;
00617   if (pendingLock == regionID)
00618     pendingLock.localID = -1;
00619   thisProxy[regionID.chunkID].unlockReply(idx,regionID);
00620 }
00621 
00622 void ParFUMShadowArray::unlockRegion(RegionID regionID){
00623   CkAssert(regionID.chunkID == idx);
00624   CkAssert((regionID == holdingLock) || (regionID.localID == -1));
00625   if (regionID.localID != -1) {
00626     LockRegion *region = regionTable.get(regionID);
00627     CkAssert(region!=NULL);
00628     unlockRegion(region);
00629     if(region->sharedIdxls.length() == 1){ //there are no remote chunks involved in this region
00630     }else{ //wait for unlock to finish on other chunks
00631       CthSuspend();
00632     }
00633     holdingLock.localID = -1;
00634     regionTable.remove(regionID);
00635     delete region;
00636   }
00637   DEBUG(CkPrintf("[%d] LockRegion (%d,%d,%6.4f) unlocked\n",idx,regionID.chunkID,regionID.localID,regionID.prio));
00638   CkPrintf("[%d] INCOMING: (%d,%d,%6.4f)\n", idx, regionID.chunkID,regionID.localID,regionID.prio);
00639   CkPrintf("[%d] HOLDING:  (%d,%d,%6.4f)\n", idx, holdingLock.chunkID,holdingLock.localID,holdingLock.prio);
00640   CkPrintf("[%d] PENDING:  (%d,%d,%6.4f)\n", idx, pendingLock.chunkID,pendingLock.localID,pendingLock.prio);
00641 }
00642 
00644 void ParFUMShadowArray::unlockRegion(LockRegion *region){
00645   //unlock the local nodes
00646   //unlockLocalNodes(region);
00647   //unlock the sharedIdxls on this chunk
00648   //unlockSharedIdxls(region);
00649   //free the idxl and elements on remote chunks
00650   for(int i=0;i<region->sharedIdxls.length();i++){
00651     int chunk = region->sharedIdxls[i];
00652     if(chunk != idx){
00653       thisProxy[chunk].unlockForRemote(region->myID);
00654     }
00655   }
00656   region->numReplies = 1; //this chunk has unlocked all the stuff on it
00657 }
00658 
00659 void ParFUMShadowArray::unlockLocalNodes(LockRegion *region){
00660   //unlock the localNodes
00661   FEM_Node &fmNode = fmMesh->node;
00662   IDXL_Side &shared = fmNode.shared;
00663   AllocTable2d<int> &lockTable = ((FEM_DataAttribute *)fmNode.lookup(FEM_ADAPT_LOCK,"lockRegion"))->getInt();
00664   
00665   for(int i=0;i<region->localNodes.length();i++){
00666     //is the node locked and has it been locked by this region
00667     if(lockTable[region->localNodes[i]][0] == region->myID.chunkID && lockTable[region->localNodes[i]][1] == region->myID.localID){ //the node had been locked by this region earlier; unlocking it now
00668       lockTable[region->localNodes[i]][0] = -1;
00669       lockTable[region->localNodes[i]][1] = -1;
00670       
00671       DEBUG(printf("[%d] Un Locking Node  %d \n",thisIndex,region->localNodes[i]));
00672     }
00673   }
00674 }
00675 
00677 void ParFUMShadowArray::unlockSharedIdxls(LockRegion *region){
00678   for(int i=0;i<region->sharedIdxls.length();i++){
00679     int chunkID = region->sharedIdxls[i];
00680     if(idx < chunkID){
00681       IDXL_List *list =  fmMesh->node.shared.getIdxlListN(chunkID);
00682       if(list != NULL){
00683     if(list->isLocked()){
00684       list->unlockIdxl();
00685     }
00686     else {
00687       CmiAbort("IDXL was not locked. Attempting to unlock it?");
00688     }
00689       }
00690     }
00691   }
00692 }
00693 
00697 void ParFUMShadowArray::unlockForRemote(RegionID regionID){
00698   if ((regionID == holdingLock) || (holdingLock.localID == -1)) {
00699     LockRegion *region = regionTable.get(regionID);
00700     if(region != NULL){
00701       //unlockLocalNodes(region);
00702       //unlockSharedIdxls(region);
00703       regionTable.remove(regionID);
00704       delete region;
00705     }
00706     CkPrintf("[%d] INCOMING: (%d,%d,%6.4f)\n", idx, regionID.chunkID,regionID.localID,regionID.prio);
00707     CkPrintf("[%d] HOLDING:  (%d,%d,%6.4f)\n", idx, holdingLock.chunkID,holdingLock.localID,holdingLock.prio);
00708     CkPrintf("[%d] PENDING:  (%d,%d,%6.4f)\n", idx, pendingLock.chunkID,pendingLock.localID,pendingLock.prio);
00709     if (holdingLock == regionID) 
00710       holdingLock.localID = -1;
00711     DEBUG(CkPrintf("[%d] Lockregion (%d,%d,%6.4f) unlocked \n",thisIndex,regionID.chunkID,regionID.localID,regionID.prio));
00712   }
00713   thisProxy[regionID.chunkID].unlockReply(idx,regionID);
00714 }
00715 
00719 void ParFUMShadowArray::unlockReply(int remoteChunk,RegionID regionID){
00720   LockRegion *region = regionTable.get(regionID);
00721   CkAssert(region != NULL);
00722   region->numReplies++;
00723   if(region->numReplies == region->sharedIdxls.length()){
00724     CthAwaken(region->tid);
00725   }
00726 }
00727 
00728 
00732 FEM_Comm *ParFUMShadowArray::FindIdxlSide(int idxlType) {
00733   if(idxlType == 0) { //shared node
00734     return &(fmMesh->node.shared);
00735   }
00736   else if(idxlType == 1) { //ghost node send 
00737     return &(fmMesh->node.ghostSend);
00738   }
00739   else if(idxlType == 2) { //ghost node recv 
00740     return &(fmMesh->node.ghost->ghostRecv);
00741   }
00742   else if((idxlType-3)%2 == 0) { //ghost elem send 
00743     int elemType = (idxlType-3)/2;
00744     return &(fmMesh->elem[elemType].ghostSend);
00745   }
00746   else if((idxlType-4)%2 == 0) { //ghost elem recv 
00747     int elemType = (idxlType-4)/2;
00748     return &(fmMesh->elem[elemType].ghost->ghostRecv);
00749   }
00750   return NULL;
00751 }
00752 
00753 
00762 int ParFUMShadowArray::IdxlAddPrimary(int localId, int sharedChk, int idxlType){
00763   if(idxlType>0 && idxlType%2==0) {
00764     localId = FEM_From_ghost_index(sharedChk);
00765   }
00766   return (FindIdxlSide(idxlType))->addNode(localId,sharedChk);
00767 }
00768 
00773 bool ParFUMShadowArray::IdxlAddSecondary(int localId, int sharedChk, int sharedIdx, int idxlType){
00774   if(idxlType>0 && idxlType%2==0) {
00775     localId = FEM_From_ghost_index(sharedChk);
00776   }
00777   return (FindIdxlSide(idxlType))->setNode(localId,sharedChk,sharedIdx);
00778 }
00779 
00783 int ParFUMShadowArray::IdxlRemovePrimary(int localId, int sharedChk, int idxlType){
00784   if(idxlType>0 && idxlType%2==0) {
00785     localId = FEM_From_ghost_index(sharedChk);
00786   }
00787   return (FindIdxlSide(idxlType))->removeNode(localId,sharedChk);
00788 }
00789 
00793 bool ParFUMShadowArray::IdxlRemoveSecondary(int sharedChk, int sharedIdx, int idxlType){
00794   return (FindIdxlSide(idxlType))->unsetNode(sharedChk,sharedIdx);
00795 }
00796 
00800 int ParFUMShadowArray::IdxlLookUpPrimary(int localId, int sharedChk, int idxlType){
00801   if(idxlType>0 && idxlType%2==0) {
00802     localId = FEM_From_ghost_index(sharedChk);
00803   }
00804   return (FindIdxlSide(idxlType))->existsNode(localId,sharedChk);
00805 }
00806 
00810 int ParFUMShadowArray::IdxlLookUpSecondary(int sharedChk, int sharedIdx, int idxlType){
00811   return (FindIdxlSide(idxlType))->getNode(sharedChk,sharedIdx);
00812 }
00813 
00814 
00815 // BULK ADAPT ENTRY METHODS START HERE:
00816 adaptAdjMsg *ParFUMShadowArray::remote_bulk_edge_bisect_2D(adaptAdj &nbrElem, adaptAdj &splitElem, int new_idxl, int n1_idxl, int n2_idxl, int partitionID)
00817 {
00818   adaptAdjMsg *am = new adaptAdjMsg;
00819   am->elem = bulkAdapt->remote_edge_bisect_2D(nbrElem, splitElem, new_idxl, n1_idxl, n2_idxl, partitionID);
00820   return am;
00821 }
00822 
00823 longestMsg *ParFUMShadowArray::isLongest(int elem, int elemType, double len)
00824 {
00825   longestMsg *m = new longestMsg;
00826   m->longest = bulkAdapt->isLongest(elem, elemType, len);
00827   return m;
00828 }
00829 
00830 void ParFUMShadowArray::remote_adaptAdj_replace(adaptAdj &elem, adaptAdj &oldElem, adaptAdj &newElem)
00831 {
00832   bulkAdapt->remote_adaptAdj_replace(elem, oldElem, newElem);
00833 }
00834 
00835 void ParFUMShadowArray::remote_edgeAdj_replace(int remotePartID, adaptAdj &adj, 
00836                            adaptAdj &elem, 
00837                            adaptAdj &splitElem, int n1_idxl,
00838                            int n2_idxl)
00839 {
00840   bulkAdapt->remote_edgeAdj_replace(remotePartID, adj, elem, splitElem, 
00841                     n1_idxl, n2_idxl);
00842 }
00843 
00844 void ParFUMShadowArray::remote_edgeAdj_add(int remotePartID, adaptAdj &adj, 
00845                        adaptAdj &splitElem, int n1_idxl, 
00846                        int n2_idxl)
00847 {
00848   bulkAdapt->remote_edgeAdj_add(remotePartID, adj, splitElem, n1_idxl, n2_idxl);
00849 }
00850 
00851 void ParFUMShadowArray::recv_split_3D(int pos, int tableID, adaptAdj &elem, 
00852                       adaptAdj &splitElem)
00853 {
00854   bulkAdapt->recv_split_3D(pos, tableID, elem, splitElem);
00855 }
00856 
00857 void ParFUMShadowArray::handle_split_3D(int remotePartID, int pos, int tableID,
00858                     adaptAdj &elem, RegionID lockRegionID, int n1_idxl, 
00859                     int n2_idxl, int n5_idxl)
00860 {
00861   CkAssert(holdingLock == lockRegionID);
00862   bulkAdapt->handle_split_3D(remotePartID, pos, tableID, elem, lockRegionID, n1_idxl, 
00863                  n2_idxl, n5_idxl);
00864 }
00865 
00866 void ParFUMShadowArray::recv_splits(int tableID, int expectedSplits)
00867 {
00868   while (!(bulkAdapt->all_splits_received(tableID, expectedSplits)))
00869     CthYield();
00870 }
00871 
00872 void ParFUMShadowArray::update_asterisk_3D(int remotePartID, int i, 
00873                        adaptAdj &elem, int numElemPairs, 
00874                        adaptAdj *elemPairs, RegionID lockRegionID,
00875                        int n1_idxl, int n2_idxl, int n5_idxl)
00876 {
00877   CkAssert(holdingLock == lockRegionID);
00878   bulkAdapt->update_asterisk_3D(remotePartID, i, elem, numElemPairs, elemPairs,
00879                 lockRegionID, n1_idxl, n2_idxl, n5_idxl);
00880 }
00881 
00882 
00883 #include "ParFUM_SA.def.h"
libs/ck-libs/ParFUM/ParFUM_SA.C
