| version 1.22 | version 1.23 |
|---|
| |
| | |
| #include "ParFUM.h" | #include "ParFUM.h" |
| #include "ParFUM_internals.h" | #include "ParFUM_internals.h" |
| | #include "MsaHashtable.h" |
| | |
| #include "../parmetis/parmetis.h" | #include "../parmetis/parmetis.h" |
| | |
| |
| call parmetis | call parmetis |
| */ | */ |
| double parStartTime = CkWallTimer(); | double parStartTime = CkWallTimer(); |
| | MSA1DINT::Read &rPtr = eptrMSA.syncToRead(wPtr); |
| | MSA1DINT::Read &rInd = eindMSA.syncToRead(wInd); |
| printf("starting FEM_call_parmetis \n"); | printf("starting FEM_call_parmetis \n"); |
| struct partconndata *partdata = FEM_call_parmetis(data,comm_context); | struct partconndata *partdata = FEM_call_parmetis(data.nelem, rPtr, rInd, comm_context); |
| | |
| printf("done with parmetis %d FEM_Mesh %d in %.6lf \n",CmiMemoryUsage(),sizeof(FEM_Mesh),CkWallTimer()-parStartTime); | printf("done with parmetis %d FEM_Mesh %d in %.6lf \n",CmiMemoryUsage(),sizeof(FEM_Mesh),CkWallTimer()-parStartTime); |
| | |
| |
| | |
| DEBUG(printf("[%d] Memory usage on vp 0 close to max %d \n",CkMyPe(),CmiMemoryUsage())); | DEBUG(printf("[%d] Memory usage on vp 0 close to max %d \n",CkMyPe(),CmiMemoryUsage())); |
| //Free up the eptr and eind MSA arrays stored in data | //Free up the eptr and eind MSA arrays stored in data |
| | delete &rPtr; |
| | delete &rInd; |
| data.arr1.FreeMem(); | data.arr1.FreeMem(); |
| data.arr2.FreeMem(); | data.arr2.FreeMem(); |
| nodepart.FreeMem(); | nodepart.FreeMem(); |
| |
| These will be used later to give each partitioned mesh | These will be used later to give each partitioned mesh |
| its elements and data. | its elements and data. |
| */ | */ |
| | |
| FEM_Mesh mypiece; | FEM_Mesh mypiece; |
| MPI_Recv_pup(mypiece,masterRank,MESH_CHUNK_TAG,(MPI_Comm)comm_context); | MPI_Recv_pup(mypiece,masterRank,MESH_CHUNK_TAG,(MPI_Comm)comm_context); |
| | |
| /* | /* |
| call parmetis and get the resuts back from it | call parmetis and get the resuts back from it |
| */ | */ |
| struct partconndata *partdata = FEM_call_parmetis(data,comm_context); | MSA1DINT::Read &rPtr = data.arr1.syncToRead(data.arr1.getInitialWrite()); |
| | MSA1DINT::Read &rInd = data.arr2.syncToRead(data.arr1.getInitialWrite()); |
| | struct partconndata *partdata = FEM_call_parmetis(data.nelem, rPtr, rInd, comm_context); |
| | |
| /* | /* |
| write to the msa that contains the partitions to which a node belongs | write to the msa that contains the partitions to which a node belongs |
| |
| //printf("[%d] Number of elements in my partitioned mesh %d number of nodes %d \n",myRank,me.m->nElems(),me.m->node.size()); | //printf("[%d] Number of elements in my partitioned mesh %d number of nodes %d \n",myRank,me.m->nElems(),me.m->node.size()); |
| | |
| //Free up the eptr and eind MSA arrays stored in data | //Free up the eptr and eind MSA arrays stored in data |
| | delete &rPtr; |
| | delete &rInd; |
| data.arr1.FreeMem(); | data.arr1.FreeMem(); |
| data.arr2.FreeMem(); | data.arr2.FreeMem(); |
| nodepart.FreeMem(); | nodepart.FreeMem(); |
| |
| parmetis. It returns the partition and the connectivity of | parmetis. It returns the partition and the connectivity of |
| the elements for which this processor is responsible. | the elements for which this processor is responsible. |
| */ | */ |
| struct partconndata * FEM_call_parmetis(struct conndata &data,FEM_Comm_t comm_context){ | struct partconndata * FEM_call_parmetis(int nelem, MSA1DINT::Read &rPtr, MSA1DINT::Read &rInd, FEM_Comm_t comm_context) |
| | { |
| int myRank,numChunks; | int myRank,numChunks; |
| MPI_Comm_size((MPI_Comm)comm_context,&numChunks); | MPI_Comm_size((MPI_Comm)comm_context,&numChunks); |
| MPI_Comm_rank((MPI_Comm)comm_context,&myRank); | MPI_Comm_rank((MPI_Comm)comm_context,&myRank); |
| | |
| int nelem = data.nelem; | |
| /* | /* |
| Setup the elmdist array. All processors | Setup the elmdist array. All processors |
| get equal number of elements. This is not | get equal number of elements. This is not |
| |
| DEBUG(printf("\n")); | DEBUG(printf("\n")); |
| int startindex = elmdist[myRank]; | int startindex = elmdist[myRank]; |
| int endindex = elmdist[myRank+1]; | int endindex = elmdist[myRank+1]; |
| data.arr1.sync(); | |
| data.arr2.sync(); | |
| int numindices = endindex - startindex; | int numindices = endindex - startindex; |
| int *eptr = new int[numindices+1]; | int *eptr = new int[numindices+1]; |
| /* | /* |
| Read the msa arrays to extract the data | Read the msa arrays to extract the data |
| Store it in the eptr and eind arrays | Store it in the eptr and eind arrays |
| */ | */ |
| int startConn = data.arr1.get(startindex); | int startConn = rPtr.get(startindex); |
| int endConn = data.arr1.get(endindex); | int endConn = rPtr.get(endindex); |
| int numConn = endConn - startConn; | int numConn = endConn - startConn; |
| int *eind = new int[numConn]; | int *eind = new int[numConn]; |
| DEBUG(printf("%d startindex %d endindex %d startConn %d endConn %d \n",myRank,startindex,endindex,startConn,endConn)); | DEBUG(printf("%d startindex %d endindex %d startConn %d endConn %d \n",myRank,startindex,endindex,startConn,endConn)); |
| for(int i=startindex;i<endindex;i++){ | for(int i=startindex;i<endindex;i++){ |
| int conn1 = data.arr1.get(i); | int conn1 = rPtr.get(i); |
| int conn2 = data.arr1.get(i+1); | int conn2 = rPtr.get(i+1); |
| eptr[i-startindex] = conn1 - startConn; | eptr[i-startindex] = conn1 - startConn; |
| for(int j=conn1;j<conn2;j++){ | for(int j=conn1;j<conn2;j++){ |
| eind[j-startConn] = data.arr2.get(j); | eind[j-startConn] = rInd.get(j); |
| } | } |
| } | } |
| eptr[numindices] = endConn - startConn; | eptr[numindices] = endConn - startConn; |
| |
| } | } |
| printf("\n"); | printf("\n"); |
| */ | */ |
| data.arr1.sync(); | |
| data.arr2.sync(); | |
| int wgtflag=0,numflag=0,ncon=1,ncommonnodes=2,options[5],edgecut=0; | int wgtflag=0,numflag=0,ncon=1,ncommonnodes=2,options[5],edgecut=0; |
| double ubvec = 1.05; | double ubvec = 1.05; |
| double *tpwgts = new double[numChunks]; | double *tpwgts = new double[numChunks]; |
| |
| } | } |
| } | } |
| | |
| /* | |
| horrible bubble sort, replace by quicksort : done | |
| */ | |
| void sortNodeList(NodeList &lnodes){ | void sortNodeList(NodeList &lnodes){ |
| CkVec<NodeElem> *vec = lnodes.vec; | CkVec<NodeElem> *vec = lnodes.vec; |
| /* for(int i=0;i<vec->size();i++){ | |
| for(int j=i+1;j<vec->size();j++){ | |
| if((*vec)[i].global > (*vec)[j].global){ | |
| NodeElem t = (*vec)[i]; | |
| (*vec)[i] = (*vec)[j]; | |
| (*vec)[j] = t; | |
| } | |
| } | |
| }*/ | |
| vec->quickSort(); | vec->quickSort(); |
| } | } |
| | |