Charm++: Charm Source Code Documentation

00001 /*
00002 Multiblock CFD Framework for Charm++
00003 Parallel Programming Lab, Univ. of Illinois 2000
00004 
00005 This file implements a C, C++, or Fortran-callable
00006 library for parallel multiblock CFD computations.
00007 
00008 We partition the user's grid using the makemblock programs.
00009 We run the user's (timeloop) driver routine in a 
00010 thread (in the style of AMPI), so communication looks
00011 blocking to the user.  Internally, we suspend the
00012 user's driver thread when communication is needed,
00013 then resume the thread when the results arrive.
00014  */
00015 #include "mblock_impl.h"
00016 #include <limits.h>
00017 #include <float.h> /*for FLT_MIN on non-Suns*/
00018 
00019 /* TCHARM Semaphore ID */
00020 #define MBLOCK_TCHARM_SEMAID 0x003B70C6 /* __MBLOCK */
00021 
00022 #if 0
00023 /*Many debugging statements:*/
00024 #define DBG(x) ckout<<"["<<thisIndex<<"] MBLK> "<<x<<endl;
00025 
00026 #else
00027 /*No debugging statements*/
00028 #define DBG(x) /*empty*/
00029 #endif
00030 
00031 //_mblkptr gives the current chunk, and is only
00032 // valid in routines called from driver().
00033 CtvStaticDeclare(MBlockChunk *, _mblkptr);
00034 
00035 CDECL void driver(void);
00036 FDECL void FTN_NAME(DRIVER,driver)(void);
00037 
00038 static void callDrivers(void) {
00039         driver();
00040         FTN_NAME(DRIVER,driver)();
00041 }
00042 
00043 static int callDrivers_idx = -1;
00044 
00045 void MBlockNodeInit(void) 
00046 {
00047   CmiAssert(callDrivers_idx == -1);
00048   callDrivers_idx = TCHARM_Register_thread_function((TCHARM_Thread_data_start_fn)callDrivers);
00049 }
00050 
00051 //Startup routine to use if the user doesn't register one
00052 static void MBlockFallbackSetup(void)
00053 {
00054         TCHARM_Create(TCHARM_Get_num_chunks(),callDrivers_idx);
00055 }
00056 
00057 void MBlockProcInit(void) 
00058 {
00059   CtvInitialize(MBlockChunk *, _mblkptr);
00060   TCHARM_Set_fallback_setup(MBlockFallbackSetup);
00061 }
00062 
00063 CDECL void MBLK_Init(int comm) {
00064   if (TCHARM_Element()==0) {
00065      CkArrayID threads; int nThreads;
00066      CkArrayOptions opt=TCHARM_Attach_start(&threads,&nThreads);
00067      CkArrayID aid=CProxy_MBlockChunk::ckNew(threads,opt);
00068   }
00069   MBlockChunk *c=(MBlockChunk *)TCharm::get()->semaGet(MBLOCK_TCHARM_SEMAID);
00070 }
00071 FORTRAN_AS_C(MBLK_INIT,MBLK_Init,mblk_init, (int *comm), (*comm))
00072 
00073 /******************** Reduction Support **********************/
00074 
00075 template<class d>
00076 void sum(const int len, d* lhs, const d* rhs)
00077 {
00078   int i;
00079   for(i=0;i<len;i++) {
00080     *lhs++ += *rhs++;
00081   }
00082 }
00083 
00084 #ifdef __CYGWIN__
00085 #undef max 
00086 #undef min
00087 #endif
00088 
00089 template<class d>
00090 void max(const int len, d* lhs, const d* rhs)
00091 {
00092   int i;
00093   for(i=0;i<len;i++) {
00094     *lhs = (*lhs > *rhs) ? *lhs : *rhs;
00095     lhs++; rhs++;
00096   }
00097 }
00098 
00099 template<class d>
00100 void min(const int len, d* lhs, const d* rhs)
00101 {
00102   int i;
00103   for(i=0;i<len;i++) {
00104     *lhs = (*lhs < *rhs) ? *lhs : *rhs;
00105     lhs++; rhs++;
00106   }
00107 }
00108 
00109 template<class d>
00110 void assign(const int len, d* lhs, d val)
00111 {
00112   int i;
00113   for(i=0;i<len;i++) {
00114     *lhs = val;
00115   }
00116 }
00117 
00118 static inline void
00119 initialize(const DType& dt, void *lhs, int op)
00120 {
00121   switch(op) {
00122     case MBLK_SUM:
00123       switch(dt.base_type) {
00124         case MBLK_BYTE : 
00125           assign(dt.vec_len,(unsigned char*)lhs, (unsigned char)0); 
00126           break;
00127         case MBLK_INT : assign(dt.vec_len,(int*)lhs, 0); break;
00128         case MBLK_REAL : assign(dt.vec_len,(float*)lhs, (float)0.0); break;
00129         case MBLK_DOUBLE : assign(dt.vec_len,(double*)lhs, 0.0); break;
00130       }
00131       break;
00132     case MBLK_MAX:
00133       switch(dt.base_type) {
00134         case MBLK_BYTE : 
00135           assign(dt.vec_len,(unsigned char*)lhs, (unsigned char)CHAR_MIN); 
00136           break;
00137         case MBLK_INT : assign(dt.vec_len,(int*)lhs, INT_MIN); break;
00138         case MBLK_REAL : assign(dt.vec_len,(float*)lhs, (float)FLT_MIN); break;
00139         case MBLK_DOUBLE : assign(dt.vec_len,(double*)lhs, DBL_MIN); break;
00140       }
00141       break;
00142     case MBLK_MIN:
00143       switch(dt.base_type) {
00144         case MBLK_BYTE : 
00145           assign(dt.vec_len,(unsigned char*)lhs, (unsigned char)CHAR_MAX); 
00146           break;
00147         case MBLK_INT : assign(dt.vec_len,(int*)lhs, INT_MAX); break;
00148         case MBLK_REAL : assign(dt.vec_len,(float*)lhs, FLT_MAX); break;
00149         case MBLK_DOUBLE : assign(dt.vec_len,(double*)lhs, DBL_MAX); break;
00150       }
00151       break;
00152   }
00153 }
00154 
00155 typedef void (*combineFn)(const int len,void *lhs,const void *rhs);
00156 
00157 typedef void (*combineFn_BYTE)(const int len,unsigned char *lhs,const unsigned char *rhs);
00158 typedef void (*combineFn_INT)(const int len,int *lhs,const int *rhs);
00159 typedef void (*combineFn_REAL)(const int len,float *lhs,const float *rhs);
00160 typedef void (*combineFn_DOUBLE)(const int len,double *lhs,const double *rhs);
00161 
00162 
00163 static combineFn
00164 combine(const DType& dt, int op)
00165 {
00166   switch(op) {
00167 //This odd-looking define selects the appropriate templated type
00168     // of "fn", casts it to a void* type, and returns it.
00169 #define combine_switch(fn) \
00170       switch(dt.base_type) {\
00171         case MBLK_BYTE : return (combineFn)(combineFn_BYTE)fn;\
00172         case MBLK_INT : return (combineFn)(combineFn_INT)fn;\
00173         case MBLK_REAL : return (combineFn)(combineFn_REAL)fn;\
00174         case MBLK_DOUBLE : return (combineFn)(combineFn_DOUBLE)fn;\
00175       }\
00176       break;
00177     case MBLK_SUM: combine_switch(sum);
00178     case MBLK_MIN: combine_switch(min);
00179     case MBLK_MAX: combine_switch(max);
00180   }
00181   return NULL;
00182 }
00183 
00184 /******************************* MBLOCKCHUNK *********************************/
00185 
00186 void MBlockChunk::commonInit(void)
00187 {
00188   nfields = 0;
00189   seqnum = 0;
00190   b=NULL;
00191 
00192   for (int bc=0;bc<MBLK_MAXBC;bc++)
00193     bcs[bc].fn=NULL;
00194 }
00195 
00196 MBlockChunk::MBlockChunk(const CkArrayID &threads_)
00197 {
00198   commonInit();
00199 
00200   threads=threads_;
00201   migInit();
00202   
00203   update.nRecd = 0;
00204   update.wait_seqnum = -1;
00205   messages = CmmNew();
00206   
00207   thread->semaPut(MBLOCK_TCHARM_SEMAID,this);
00208 }
00209 
00210 MBlockChunk::MBlockChunk(CkMigrateMessage *msg)
00211         :CBase_MBlockChunk(msg)
00212 {
00213   commonInit();
00214   
00215   messages=NULL;
00216 }
00217 
00218 //Update fields after migration
00219 void MBlockChunk::migInit(void)
00220 {
00221   thread=threads[thisIndex].ckLocal();
00222   if (thread==NULL) CkAbort("MBlock can't locate its thread!\n");
00223   CtvAccessOther(thread->getThread(),_mblkptr)=this;
00224 }
00225 
00226 void MBlockChunk::ckJustMigrated(void)
00227 {
00228   ArrayElement1D::ckJustMigrated();
00229   migInit();
00230 }
00231 
00232 MBlockChunk::~MBlockChunk() //Destructor-- deallocate memory
00233 {
00234         CmmFree(messages);
00235         delete b;
00236 }
00237 
00238 void 
00239 MBlockChunk::read(const char *prefix,int nDim) 
00240 {
00241   if (nDim!=3) CkAbort("MBlock> Currently only 3D calculations are supported.\n");
00242   delete b;
00243   b = new block(prefix, thisIndex);
00244   int n = b->getPatches();
00245   nRecvPatches = 0;
00246   for(int i=0;i<n;i++) {
00247     patch *p = b->getPatch(i);
00248     if(p->type == patch::internal)
00249       nRecvPatches++;
00250   }
00251 }
00252 
00253 void
00254 MBlockChunk::recv(MBlockDataMsg *dm)
00255 {
00256   if(dm->seqnum == update.wait_seqnum) {
00257     DBG( "recv: handling ("<<dm->seqnum<<")" )
00258     update_field(dm); // update the appropriate field value
00259     if(update.nRecd==nRecvPatches) {
00260       update.wait_seqnum = -1; //This update is complete
00261       thread->resume();
00262     }
00263   } else if (dm->seqnum>seqnum) {
00264     DBG( "recv: stashing early ("<<dm->seqnum<<")" )
00265     CmmPut(messages, 1, &(dm->seqnum), dm);
00266   } else
00267     CkAbort("MBLOCK MBlockChunk received message from the past!\n");
00268 }
00269 
00270 //Send the values for my source patches out
00271 void
00272 MBlockChunk::send(int fid,const extrudeMethod &meth,const void *grid)
00273 {
00274   const DType &dt=fields[fid]->dt;
00275   int len = dt.length();
00276   const char *fStart=(const char *)grid;
00277   fStart+=dt.init_offset;
00278   blockDim d = fields[fid]->dim;
00279   int n = b->getPatches();
00280   for(int m=0;m<n;m++) {
00281     patch *p = b->getPatch(m);
00282     if (p->type != patch::internal)
00283       continue;
00284     internalBCpatch *ip = (internalBCpatch*) p;
00285     blockSpan src=ip->getExtents(meth,fields[fid]->forVoxel,-1);
00286     int dest = ip->dest;
00287     int num = src.getDim().getSize();
00288     int msgLen=len*num;
00289     MBlockDataMsg *msg = new (&msgLen, 0) MBlockDataMsg(seqnum, thisIndex, fid, 
00290         ip->destPatch); 
00291     CHK(msg);
00292     blockLoc dsize; //Size in destination space
00293     blockLoc so; //Source origin-- corresponds to dest. start
00294     for (int axis=0;axis<3;axis++) {
00295       dsize[ip->orient[axis]]=src.getDim()[axis];
00296       if (ip->orient.dIsFlipped(axis)) 
00297         so[axis]=src.end[axis]-1;
00298       else
00299         so[axis]=src.start[axis]; 
00300     }
00301     blockLoc iDir=ip->orient.destAxis(0);
00302     blockLoc jDir=ip->orient.destAxis(1);
00303     blockLoc kDir=ip->orient.destAxis(2);
00304     
00305     //Loop over the destination space
00306     void *data = msg->data;
00307     for(int k=0;k<dsize[2];k++) 
00308       for(int j=0;j<dsize[1];j++)
00309         for(int i=0;i<dsize[0];i++)     
00310         {
00311           blockLoc from=so+i*iDir+j*jDir+k*kDir;
00312 #ifndef CMK_OPTIMIZE  /*Perform bounds check*/
00313           if (!src.contains(from))
00314             CkAbort("Read location out of bounds in mblock::MBlockChunk::send!\n");
00315 #endif
00316           const void *src = (const void *)(fStart+d[from]*dt.distance);
00317           memcpy(data, src, len);
00318           data = (void*) ((char*)data + len);
00319         }
00320     thisProxy[dest].recv(msg);
00321     DBG("  sending "<<num<<" data items ("<<seqnum<<") to processor "<<dest)
00322   }
00323 }
00324 
00325 
00326 //Update my ghost cells based on these values
00327 void
00328 MBlockChunk::update_field(MBlockDataMsg *msg)
00329 {
00330   DBG("    update_field ("<<msg->seqnum<<")")
00331   const DType &dt=fields[msg->fid]->dt;
00332   int length=dt.length();
00333   char *aStart=(char *)update.ogrid;
00334   char *aEnd=aStart+fields[msg->fid]->arrayBytes;
00335   char *fStart=aStart+dt.init_offset;
00336   char *data = (char *) msg->data;
00337   internalBCpatch *p = (internalBCpatch*) (b->getPatch(msg->patchno));
00338   blockSpan dest=p->getExtents(update.m,fields[msg->fid]->forVoxel);
00339 
00340   blockDim dim = fields[msg->fid]->dim;
00341   //Loop over the destination-space patch
00342   for(int k=dest.start[2]; k<dest.end[2]; k++)
00343     for(int j=dest.start[1]; j<dest.end[1]; j++)
00344       for(int i=dest.start[0]; i<dest.end[0]; i++) 
00345       {
00346         char *dest = fStart+dim.c_index(i,j,k)*dt.distance;
00347 #ifndef CMK_OPTIMIZE /*Perform bounds check*/
00348         if (dest<aStart || dest>=aEnd)
00349           CkAbort("MBlockChunk::update_field would write out of bounds!\n");
00350 #endif
00351         memcpy(dest, data, length);
00352         data = data + length;
00353       }
00354   delete msg;
00355   update.nRecd++;
00356 }
00357 
00358 void
00359 MBlockChunk::start_update(int fid,const extrudeMethod &m,void *ingrid, void *outgrid)
00360 {
00361   //Update sequence number
00362   seqnum++;
00363   update.nRecd=0;
00364   update.wait_seqnum=seqnum;
00365   update.m=m;
00366   update.ogrid=outgrid;
00367 
00368   DBG("update("<<seqnum<<") {")
00369   // first send my field values to all the processors that need it
00370   send(fid, m,ingrid);
00371 
00372   // now, if any of the field values have been received already,
00373   // process them
00374   MBlockDataMsg *dm;
00375   while ((dm = (MBlockDataMsg*)CmmGet(messages, 1, &seqnum, 0))!=NULL) {
00376     DBG("   handling stashed message")
00377     update_field(dm);
00378   }
00379   if (update.nRecd == nRecvPatches)
00380     update.wait_seqnum = -1; // we now have everything
00381 
00382   DBG("} update("<<seqnum<<")")
00383 }
00384 
00385 int
00386 MBlockChunk::test_update(void)
00387 {
00388   return (update.wait_seqnum==(-1))?MBLK_DONE : MBLK_NOTDONE;
00389 }
00390 
00391 int
00392 MBlockChunk::wait_update(void)
00393 {
00394   if(update.wait_seqnum != (-1)) {
00395     DBG("sleeping for "<<nRecvPatches-nRecd<<" update messages")
00396     thread->suspend();
00397     DBG("got all update messages for "<<seqnum)  
00398   }
00399   return MBLK_SUCCESS;
00400 }
00401 
00402 void
00403 MBlockChunk::reduce_field(int fid, const void *grid, void *outbuf, int op)
00404 {
00405   // first reduce over local gridpoints
00406   const DType &dt = fields[fid]->dt;
00407   const void *src = (const void *) ((const char *) grid + dt.init_offset);
00408   ::initialize(dt,outbuf,op);
00409   combineFn fn=combine(dt,op);
00410   int dim[3];
00411   MBLK_Get_blocksize(dim);
00412   blockDim d = fields[fid]->dim;
00413   for(int k=0; k<dim[2]; k++)
00414     for(int j=0; j<dim[1]; j++)
00415       for(int i=0; i<dim[0]; i++)
00416         fn(dt.vec_len, outbuf, 
00417             ((const char *)src + d.c_index(i,j,k)*dt.distance));
00418   // and now reduce over partitions
00419   reduce(fid, outbuf, outbuf, op);
00420 }
00421 
00422 void
00423 MBlockChunk::reduce(int fid, const void *inbuf, void *outbuf, int op)
00424 {
00425   const DType &dt = fields[fid]->dt;
00426   int len = dt.length();
00427   if(ckGetArraySize()==1) {
00428     memcpy(outbuf,inbuf,len);
00429     return;
00430   }
00431   CkReduction::reducerType rtype;
00432   switch(op) {
00433     case MBLK_SUM:
00434       switch(dt.base_type) {
00435         case MBLK_INT: rtype = CkReduction::sum_int; break;
00436         case MBLK_REAL: rtype = CkReduction::sum_float; break;
00437         case MBLK_DOUBLE: rtype = CkReduction::sum_double; break;
00438       }
00439       break;
00440     case MBLK_MAX:
00441       switch(dt.base_type) {
00442         case MBLK_INT: rtype = CkReduction::max_int; break;
00443         case MBLK_REAL: rtype = CkReduction::max_float; break;
00444         case MBLK_DOUBLE: rtype = CkReduction::max_double; break;
00445       }
00446       break;
00447     case MBLK_MIN:
00448       switch(dt.base_type) {
00449         case MBLK_INT: rtype = CkReduction::min_int; break;
00450         case MBLK_REAL: rtype = CkReduction::min_float; break;
00451         case MBLK_DOUBLE: rtype = CkReduction::min_double; break;
00452       }
00453       break;
00454   }
00455   reduce_output = outbuf;
00456   contribute(len, (void *)inbuf, rtype, 
00457         CkCallback(index_t::reductionResult(0),thisProxy) );
00458   thread->suspend();
00459 }
00460 
00461 void
00462 MBlockChunk::reductionResult(CkReductionMsg *m)
00463 {
00464   memcpy(reduce_output,m->getData(),m->getSize());
00465   delete m;
00466   reduce_output=NULL;
00467   thread->resume();
00468 }
00469 
00470 /********** Thread/migration support ************/
00471 static MBlockChunk *getCurMBlockChunk(void) 
00472 {
00473   MBlockChunk *cptr=CtvAccess(_mblkptr);
00474   if (cptr==NULL) 
00475     CkAbort("MBlock does not appear to be initialized!");
00476   return cptr;
00477 }
00478 
00479 
00480 /******************************* C Bindings **********************************/
00481 
00482 CDECL int 
00483 MBLK_Read(const char *prefix,int nDim) {
00484   getCurMBlockChunk() -> read(prefix,nDim);
00485   return MBLK_SUCCESS;
00486 }
00487 
00488 CDECL int 
00489 MBLK_Get_nblocks(int *n)
00490 {
00491   *n = TCHARM_Num_elements();
00492   return MBLK_SUCCESS;
00493 }
00494 
00495 CDECL int 
00496 MBLK_Get_myblock(int *m)
00497 {
00498   *m = TCHARM_Element();
00499   return MBLK_SUCCESS;
00500 }
00501 
00502 CDECL int 
00503 MBLK_Get_blocksize(int *dim)
00504 {
00505   MBLOCKAPI("MBLK_Get_blocksize");
00506   MBlockChunk *cptr = getCurMBlockChunk();
00507   blockDim d = cptr->b->getDim();
00508   /*Subtract one to get to voxel coordinates*/
00509   dim[0] = d[0]-1; dim[1] = d[1]-1; dim[2] = d[2]-1;
00510   return MBLK_SUCCESS;
00511 }
00512 
00513 CDECL int 
00514 MBLK_Get_nodelocs(const int *nodedim,double *nodeloc) {
00515   MBLOCKAPI("MBLK_Get_nodelocs");
00516   block *b=getCurMBlockChunk()->b;
00517   blockDim d = b->getDim();
00518   for (int i=0;i<3;i++)
00519     if (nodedim[i]!=d[i]) {
00520       CkError("MBLK_Get_nodelocs: your nodedim is %d; mine is %d!\n",
00521         nodedim[i],d[i]);
00522       return MBLK_FAILURE;
00523     }
00524   
00525   blockLoc j;
00526   BLOCKSPAN_FOR(j,blockSpan(blockLoc(0,0,0),d)) {
00527      int idx=d[j];
00528      vector3d v=b->getLoc(j);
00529      nodeloc[3*idx+0]=v.x;
00530      nodeloc[3*idx+1]=v.y;
00531      nodeloc[3*idx+2]=v.z;
00532   }
00533   
00534   return MBLK_SUCCESS;
00535 }
00536 
00537 CDECL double 
00538 MBLK_Timer(void)
00539 {
00540   return TCHARM_Wall_timer();
00541 }
00542 
00543 CDECL void 
00544 MBLK_Print(const char *str)
00545 {
00546   MBLOCKAPI("MBLK_Print");
00547   CkPrintf("[%d] %s\n", TCHARM_Element(), str);
00548 }
00549 
00550 CDECL int 
00551 MBLK_Register(void *_ud,MBLK_PupFn _pup_ud, int *rid)
00552 {
00553   *rid=TCHARM_Register(_ud,_pup_ud);
00554   return MBLK_SUCCESS;
00555 }
00556 
00557 CDECL int 
00558 MBLK_Get_registered(int n, void **b)
00559 {
00560   *b=TCHARM_Get_userdata(n);
00561   return MBLK_SUCCESS;
00562 }
00563 
00564 CDECL int 
00565 MBLK_Migrate(void)
00566 {
00567   MBLOCKAPI("MBLK_Migrate");
00568   TCharm::get()->migrate();
00569   return MBLK_SUCCESS;
00570 }
00571 
00572 CDECL int 
00573 MBLK_Create_field(int *dims,int isVoxel,
00574                   int base_type, int vec_len, 
00575                   int init_offset, int distance,
00576                   int *fid)
00577 {
00578   MBLOCKAPI("MBLK_Create_field");
00579   field_t *f=new field_t(DType(base_type,vec_len,init_offset,distance),
00580                          blockDim(dims[0],dims[1],dims[2]),isVoxel==1);
00581   *fid = getCurMBlockChunk()->add_field(f);
00582   return ((*fid==(-1)) ? MBLK_FAILURE : MBLK_SUCCESS);
00583 }
00584 
00585 CDECL int
00586 MBLK_Update_field(int fid, int ghostWidth, void *grid)
00587 {
00588   MBLOCKAPI("MBLK_Update_field");
00589   MBLK_Iupdate_field(fid, ghostWidth, grid, grid);
00590   return MBLK_Wait_update();
00591 }
00592 
00593 CDECL int
00594 MBLK_Iupdate_field(int fid, int ghostWidth, void *ingrid, void *outgrid)
00595 {
00596   MBLOCKAPI("MBLK_Iupdate_field");
00597   getCurMBlockChunk()->start_update(fid,extrudeMethod(ghostWidth),ingrid,outgrid);
00598   return MBLK_SUCCESS;
00599 }
00600 
00601 CDECL int
00602 MBLK_Test_update(int *status)
00603 {
00604   MBLOCKAPI("MBLK_Test_update");
00605   *status=getCurMBlockChunk()->test_update();
00606   return MBLK_SUCCESS; 
00607 }
00608 
00609 CDECL int
00610 MBLK_Wait_update(void)
00611 {
00612   MBLOCKAPI("MBLK_Wait_update");
00613   return getCurMBlockChunk()->wait_update();
00614 }
00615 
00616 CDECL int
00617 MBLK_Reduce_field(int fid, void *ingrid, void *outbuf, int op)
00618 {
00619   MBLOCKAPI("MBLK_Reduce_field");
00620   getCurMBlockChunk()->reduce_field(fid, ingrid, outbuf, op);
00621   return MBLK_SUCCESS;
00622 }
00623 
00624 CDECL int
00625 MBLK_Reduce(int fid, void *inbuf, void *outbuf, int op)
00626 {
00627   MBLOCKAPI("MBLK_Reduce");
00628   getCurMBlockChunk()->reduce(fid, inbuf, outbuf, op);
00629   return MBLK_SUCCESS;
00630 }
00631 
00632 CDECL int
00633 MBLK_Register_bc(const int bcnum, int ghostWidth,const MBLK_BcFn bcfn)
00634 {
00635   MBLOCKAPI("MBLK_Register_bc");
00636   return getCurMBlockChunk()->register_bc(bcnum, bcfn, extrudeMethod(ghostWidth), false);
00637 }
00638 
00639 CDECL int
00640 MBLK_Apply_bc(const int bcnum, void *p1,void *p2)
00641 {
00642   MBLOCKAPI("MBLK_Apply_bc");
00643   int retval =  getCurMBlockChunk()->apply_bc(bcnum, p1,p2);
00644   return retval;
00645 }
00646 
00647 CDECL int
00648 MBLK_Apply_bc_all(void *p1,void *p2)
00649 {
00650   MBLOCKAPI("MBLK_Apply_bc_all");
00651   int retval =  getCurMBlockChunk()->apply_bc_all(p1,p2);
00652   return retval;
00653 }
00654 
00655 CDECL void 
00656 MBLK_Print_block(void)
00657 {
00658   MBLOCKAPI("MBLK_Print_block");
00659   MBlockChunk *cptr = getCurMBlockChunk();
00660   cptr->print();
00661 }
00662 
00663 /************************ Fortran Bindings *********************************/
00664 
00665 // Utility functions for Fortran
00666 FDECL int FTN_NAME(FOFFSETOF,foffsetof)
00667   (void *first, void *second)
00668 {
00669   return (int)((char *)second - (char*)first);
00670 }
00671 
00672 FDECL void FTN_NAME(MBLK_READ, mblk_read)
00673   (const char *str, int *nDim,int *ret, int len)
00674 {
00675   /* skip over silly Fortran space-padded string */
00676   while (str[len-1]==' ') len--;
00677   char *tmpstr = new char[len+1]; CHK(tmpstr);
00678   memcpy(tmpstr,str,len);
00679   tmpstr[len] = '\0';
00680   *ret = MBLK_Read(tmpstr,*nDim);
00681   delete[] tmpstr;
00682 }
00683 
00684 FDECL void FTN_NAME(MBLK_GET_NBLOCKS, mblk_get_nblocks)
00685   (int *n, int *ret)
00686 {
00687   *ret = MBLK_Get_nblocks(n);
00688 }
00689 
00690 FDECL void FTN_NAME(MBLK_GET_MYBLOCK, mblk_get_myblock)
00691   (int *m, int *ret)
00692 {
00693   *ret = MBLK_Get_myblock(m);
00694 }
00695 static void c2f_index3d(int *idx) {
00696   idx[0]++; idx[1]++; idx[2]++;
00697 }
00698 FDECL void FTN_NAME(MBLK_GET_BLOCKSIZE, mblk_get_blocksize)
00699   (int *dims, int *ret)
00700 {
00701   *ret = MBLK_Get_blocksize(dims);
00702 }
00703 
00704 FDECL void FTN_NAME(MBLK_GET_NODELOCS,mblk_get_nodelocs)
00705   (const int *nodedim,double *nodeloc,int *ret) 
00706 {
00707   *ret = MBLK_Get_nodelocs(nodedim,nodeloc);
00708 }
00709 
00710 FDECL double FTN_NAME(MBLK_TIMER, mblk_timer)
00711   (void)
00712 {
00713   return MBLK_Timer();
00714 }
00715 
00716 FDECL void FTN_NAME(MBLK_PRINT,mblk_print)
00717   (char *str, int len)
00718 {
00719   char *tmpstr = new char[len+1]; CHK(tmpstr);
00720   memcpy(tmpstr,str,len);
00721   tmpstr[len] = '\0';
00722   MBLK_Print(tmpstr);
00723   delete[] tmpstr;
00724 }
00725 
00726 FDECL void FTN_NAME(MBLK_PRINT_BLOCK,mblk_print_block)
00727   (void)
00728 {
00729   MBLK_Print_block();
00730 }
00731 
00732 FDECL void FTN_NAME(MBLK_CREATE_FIELD, mblk_create_field)
00733   (int *dims,int *forVox,int *b, int *l, int *o, int *d, int *f, int *ret)
00734 {
00735   *ret = MBLK_Create_field(dims,*forVox,*b, *l, *o, *d, f);
00736 }
00737 
00738 FDECL void FTN_NAME(MBLK_UPDATE_FIELD, mblk_update_field)
00739   (int *fid, int *ghostWidth,void *grid, int *ret)
00740 {
00741   *ret = MBLK_Update_field(*fid,*ghostWidth, grid);
00742 }
00743 
00744 FDECL void FTN_NAME(MBLK_IUPDATE_FIELD, mblk_iupdate_field)
00745   (int *fid, int *ghostWidth, void *igrid, void *ogrid, int *ret)
00746 {
00747   *ret = MBLK_Iupdate_field(*fid,*ghostWidth, igrid, ogrid);
00748 }
00749 
00750 FDECL void FTN_NAME(MBLK_TEST_UPDATE, mblk_test_update)
00751   (int *status, int *ret)
00752 {
00753   *ret = MBLK_Test_update(status);
00754 }
00755 
00756 FDECL void FTN_NAME(MBLK_WAIT_UPDATE, mblk_wait_update)
00757   (int *ret)
00758 {
00759   *ret = MBLK_Wait_update();
00760 }
00761 
00762 FDECL void FTN_NAME(MBLK_REDUCE_FIELD, mblk_reduce_field)
00763   (int *fid, void *grid, void *out, int *op, int *ret)
00764 {
00765   *ret = MBLK_Reduce_field(*fid, grid, out, *op);
00766 }
00767 
00768 FDECL void FTN_NAME(MBLK_REDUCE, mblk_reduce)
00769   (int *fid, void *in, void *out, int *op, int *ret)
00770 {
00771   *ret = MBLK_Reduce_field(*fid, in, out, *op);
00772 }
00773 
00774 FDECL void FTN_NAME(MBLK_REGISTER_BC, mblk_register_bc)
00775   (int *bcnum, int *ghostWidth,MBLK_BcFn bcfn, int *ret)
00776 {
00777   MBLOCKAPI("MBLK_register_bc");
00778   *ret=getCurMBlockChunk()->register_bc(*bcnum, bcfn, extrudeMethod(*ghostWidth), true);        
00779 }
00780 
00781 FDECL void FTN_NAME(MBLK_APPLY_BC, mblk_apply_bc)
00782   (int *bcnum, void *p1,void *p2, int *ret)
00783 {
00784   *ret = MBLK_Apply_bc(*bcnum, p1,p2);
00785 }
00786 
00787 FDECL void FTN_NAME(MBLK_APPLY_BC_ALL, mblk_apply_bc_all)
00788   (void *p1,void *p2, int *ret)
00789 {
00790   *ret = MBLK_Apply_bc_all(p1,p2);
00791 }
00792 
00793 FDECL void FTN_NAME(MBLK_REGISTER, mblk_register)
00794   (void *block, MBLK_PupFn pupfn, int *rid, int *ret)
00795 {
00796   *ret = MBLK_Register(block, pupfn, rid);
00797 }
00798 
00799 FDECL void FTN_NAME(MBLK_MIGRATE, mblk_migrate)
00800   (int *ret)
00801 {
00802   *ret = MBLK_Migrate();
00803 }
00804 
00805 
00806 void
00807 MBlockChunk::print(void)
00808 {
00809   CkPrintf("-------------------- Block %d --------------------\n",thisIndex);
00810   b->print();
00811   CkPrintf("--------------------------------------------------\n",thisIndex);
00812 }  
00813   
00814 static void cmm_pup_mblock_message(pup_er p,void **msg) {
00815         CkPupMessage(*(PUP::er *)p,msg,1);
00816         if (pup_isDeleting(p)) delete (MBlockDataMsg *)*msg;
00817 }
00818 
00819 void
00820 MBlockChunk::pup(PUP::er &p)
00821 {
00822   threads.pup(p);
00823   messages=CmmPup(&p,messages,cmm_pup_mblock_message);
00824 
00825   if(p.isUnpacking())
00826     b = new block();
00827   b->pup(p);
00828 //Pup all other fields
00829   p(nfields);
00830   for (int i=0;i<nfields;i++) {
00831     if (p.isUnpacking()) fields[i]=new field_t;
00832     fields[i]->pup(p);
00833   }
00834   p((char*)bcs, MBLK_MAXBC*sizeof(bcs[0]));
00835   p(seqnum);
00836   p(update.wait_seqnum);
00837   p(update.nRecd);
00838   p(nRecvPatches);
00839 }
00840 
00841 #include "mblock.def.h"
libs/ck-libs/mblock/mblock.C
