Charm++: Charm Source Code Documentation

00001 
00005 CmiCommHandle CmiSendNetworkFunc(int destNode, int size, char *msg, int mode);
00006 
00007 static void handleOneBcastMsg(int size, char *msg) {
00008     CmiAssert(CMI_BROADCAST_ROOT(msg)!=0);
00009 #if CMK_OFFLOAD_BCAST_PROCESS
00010     if (CMI_BROADCAST_ROOT(msg)>0) {
00011         PCQueuePush(CsvAccess(procBcastQ), msg);
00012     } else {
00013 #if CMK_NODE_QUEUE_AVAILABLE
00014         PCQueuePush(CsvAccess(nodeBcastQ), msg);
00015 #endif
00016     }
00017 #else
00018     if (CMI_BROADCAST_ROOT(msg)>0) {
00019         processProcBcastMsg(size, msg);
00020     } else {
00021 #if CMK_NODE_QUEUE_AVAILABLE
00022         processNodeBcastMsg(size, msg);
00023 #endif
00024     }
00025 #endif
00026 }
00027 
00028 static void processBcastQs() {
00029 #if CMK_OFFLOAD_BCAST_PROCESS
00030     char *msg;
00031     do {
00032         msg = PCQueuePop(CsvAccess(procBcastQ));
00033         if (!msg) break;
00034         MACHSTATE2(4, "[%d]: process a proc-level bcast msg %p begin{", CmiMyNode(), msg);
00035         processProcBcastMsg(CMI_MSG_SIZE(msg), msg);
00036         MACHSTATE2(4, "[%d]: process a proc-level bcast msg %p end}", CmiMyNode(), msg);
00037     } while (1);
00038 #if CMK_NODE_QUEUE_AVAILABLE
00039     do {
00040         msg = PCQueuePop(CsvAccess(nodeBcastQ));
00041         if (!msg) break;
00042         MACHSTATE2(4, "[%d]: process a node-level bcast msg %p begin{", CmiMyNode(), msg);
00043         processNodeBcastMsg(CMI_MSG_SIZE(msg), msg);
00044         MACHSTATE2(4, "[%d]: process a node-level bcast msg %p end}", CmiMyNode(), msg);
00045     } while (1);
00046 #endif
00047 #endif
00048 }
00049 
00050 static INLINE_KEYWORD void processProcBcastMsg(int size, char *msg) {
00051     /* Since this function is only called on intermediate nodes,
00052      * the rank of this msg should be 0.
00053      */
00054     CmiAssert(CMI_DEST_RANK(msg)==0);
00055     /*CmiPushPE(CMI_DEST_RANK(msg), msg);*/
00056 
00057 #if CMK_BROADCAST_SPANNING_TREE
00058     SendSpanningChildrenProc(size, msg);
00059 #elif CMK_BROADCAST_HYPERCUBE
00060     SendHyperCubeProc(size, msg);
00061 #endif
00062 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00063       /* same message may be sent out, make a copy of it */
00064     if (CmiNumNodes()>1 && CmiGetReference(msg)>1) {
00065       void *newmsg;
00066       newmsg = CopyMsg(msg, size);
00067       CmiFree(msg);
00068       msg = newmsg;
00069     }
00070 #endif
00071     CmiPushPE(0, msg);
00072 
00073 }
00074 
00075 
00076 #if CMK_NODE_QUEUE_AVAILABLE
00077 static INLINE_KEYWORD void processNodeBcastMsg(int size, char *msg) {
00078 #if CMK_BROADCAST_SPANNING_TREE
00079     SendSpanningChildrenNode(size, msg);
00080 #elif CMK_BROADCAST_HYPERCUBE
00081     SendHyperCubeNode(size, msg);
00082 #endif
00083 
00084     /* In SMP mode, this push operation needs to be executed
00085      * after forwarding broadcast messages. If it is executed
00086      * earlier, then during the bcast msg forwarding period,
00087      * the msg could be already freed on the worker thread.
00088      * As a result, the forwarded message could be wrong!
00089      * 
00090      */
00091     CmiPushNode(msg);
00092 }
00093 #endif
00094 
00095 static void SendSpanningChildren(int size, char *msg, int rankToAssign, int startNode) {
00096 #if CMK_BROADCAST_SPANNING_TREE
00097     int i, oldRank;
00098     char *newmsg;
00099 
00100     oldRank = CMI_DEST_RANK(msg);
00101     /* doing this is to avoid the multiple assignment in the following for loop */
00102     CMI_DEST_RANK(msg) = rankToAssign;
00103     /* first send msgs to other nodes */
00104     CmiAssert(startNode >=0 &&  startNode<CmiNumNodes());
00105     for (i=1; i<=BROADCAST_SPANNING_FACTOR; i++) {
00106         int nd = CmiMyNode()-startNode;
00107         if (nd<0) nd+=CmiNumNodes();
00108         nd = BROADCAST_SPANNING_FACTOR*nd + i;
00109         if (nd > CmiNumNodes() - 1) break;
00110         nd += startNode;
00111         nd = nd%CmiNumNodes();
00112         CmiAssert(nd>=0 && nd!=CmiMyNode());
00113 #if CMK_BROADCAST_USE_CMIREFERENCE
00114         CmiReference(msg);
00115         CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
00116 #else
00117         newmsg = CopyMsg(msg, size);
00118         CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
00119 #endif
00120     }
00121     CMI_DEST_RANK(msg) = oldRank;
00122 #endif
00123 }
00124 
00125 static void SendHyperCube(int size,  char *msg, int rankToAssign, int startNode) {
00126 #if CMK_BROADCAST_HYPERCUBE
00127     int i, cnt, tmp, relDist, oldRank;
00128     const int dims=CmiNodesDim;
00129 
00130     oldRank = CMI_DEST_RANK(msg);
00131     /* doing this is to avoid the multiple assignment in the following for loop */
00132     CMI_DEST_RANK(msg) = rankToAssign;
00133 
00134     /* first send msgs to other nodes */
00135     relDist = CmiMyNode()-startNode;
00136     if (relDist < 0) relDist += CmiNumNodes();
00137 
00138     /* Sending scheme example: say we have 9 nodes, and the msg is sent from 0
00139      * The overall sending steps will be as follows:
00140      * 0-->8, 0-->4, 0-->2, 0-->1
00141      *               4-->6, 4-->5
00142      *                      2-->3
00143      *                      6-->7
00144      * So for node id as N=A+2^B, it will forward the broadcast (B-1) msg to in
00145      * the order as: N+2^(B-1), N+2^(B-2),..., N+1 except node 0, where B is
00146      * the first position of bit 1 in the binary format of the number of N
00147      * counting from the right with count starting from 0.
00148      * On node 0, the value "B" should be CmiNodesDim
00149      */
00150     /* Calculate 2^B */
00151     if(relDist==0) cnt = 1<<dims;
00152     else cnt = relDist & ((~relDist)+1);
00153     /*CmiPrintf("ND[%d]: send bcast msg with cnt=%d\n", CmiMyNode(), cnt);*/
00154     /* Begin to send msgs */
00155     for(cnt>>=1; cnt>0; cnt>>=1){
00156         int nd = relDist + cnt;
00157         if (nd >= CmiNumNodes()) continue;
00158         nd = (nd+startNode)%CmiNumNodes();
00159         /*CmiPrintf("ND[%d]: send to node %d\n", CmiMyNode(), nd);*/
00160         CmiAssert(nd>=0 && nd!=CmiMyNode());
00161 #if CMK_BROADCAST_USE_CMIREFERENCE
00162         CmiReference(msg);
00163         CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
00164 #else
00165         char *newmsg = CopyMsg(msg, size);
00166         CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
00167 #endif
00168     }
00169     CMI_DEST_RANK(msg) = oldRank;
00170 #endif
00171 }
00172 
00173 static void SendSpanningChildrenProc(int size, char *msg) {
00174     int startnode = CMI_BROADCAST_ROOT(msg)-1;
00175     SendSpanningChildren(size, msg, 0, startnode);
00176 #if CMK_SMP
00177     /* second send msgs to my peers on this node */
00178     SendToPeers(size, msg);
00179 #endif
00180 }
00181 
00182 /* send msg along the hypercube in broadcast. (Sameer) */
00183 static void SendHyperCubeProc(int size, char *msg) {
00184     int startpe = CMI_BROADCAST_ROOT(msg)-1;
00185     int startnode = CmiNodeOf(startpe);
00186 #if CMK_SMP
00187     if (startpe > CmiNumPes()) startnode = startpe - CmiNumPes();
00188 #endif
00189     SendHyperCube(size, msg, 0, startnode);
00190 #if CMK_SMP
00191     /* second send msgs to my peers on this node */
00192     SendToPeers(size, msg);
00193 #endif
00194 }
00195 
00196 #if CMK_NODE_QUEUE_AVAILABLE
00197 static void SendSpanningChildrenNode(int size, char *msg) {
00198     int startnode = -CMI_BROADCAST_ROOT(msg)-1;
00199     SendSpanningChildren(size, msg, DGRAM_NODEMESSAGE, startnode);
00200 }
00201 static void SendHyperCubeNode(int size, char *msg) {
00202     int startnode = -CMI_BROADCAST_ROOT(msg)-1;
00203     SendHyperCube(size, msg, DGRAM_NODEMESSAGE, startnode);
00204 }
00205 #endif
00206 
00207 #if USE_COMMON_SYNC_BCAST
00208 /* Functions regarding broadcat op that sends to every one else except me */
00209 void CmiSyncBroadcastFn1(int size, char *msg) {
00210     int i, mype;
00211 
00212     CQdCreate(CpvAccess(cQdState), CmiNumPes()-1);
00213     /*record the rank to avoid re-sending the msg in  spanning tree or hypercube*/
00214     CMI_DEST_RANK(msg) = CmiMyRank();
00215 
00216 #if CMK_BROADCAST_SPANNING_TREE
00217     CMI_SET_BROADCAST_ROOT(msg, CmiMyNode()+1);
00218     SendSpanningChildrenProc(size, msg);
00219 #elif CMK_BROADCAST_HYPERCUBE
00220     CMI_SET_BROADCAST_ROOT(msg, CmiMyNode()+1);
00221     SendHyperCubeProc(size, msg);
00222 #else
00223     mype = CmiMyPe();
00224     #if CMK_SMP
00225     /* In SMP, this function may be called from comm thread with a larger pe */
00226     if(mype > _Cmi_numpes){
00227         for(i=0; i<_Cmi_numpes; i++)
00228                 CmiSyncSendFn(i, size, msg);
00229         return;
00230     }
00231     #endif
00232         
00233     for ( i=mype+1; i<_Cmi_numpes; i++ )
00234         CmiSyncSendFn(i, size, msg) ;
00235         
00236     for ( i=0; i<mype; i++ )
00237         CmiSyncSendFn(i, size, msg) ;
00238 #endif
00239 
00240     /*CmiPrintf("In  SyncBroadcast broadcast\n");*/
00241 }
00242 
00243 void CmiSyncBroadcastFn(int size, char *msg) {
00244     void *newmsg = msg;
00245 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00246       /* need to copy the msg in case the msg is on the stack */
00247       /* and we only need to copy when sending out network */
00248     if (CmiNumNodes()>1) newmsg = CopyMsg(msg, size);
00249 #endif
00250     CmiSyncBroadcastFn1(size, newmsg);
00251 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00252     if (newmsg != msg) CmiFree(newmsg);
00253 #endif
00254 }
00255 
00256 void CmiFreeBroadcastFn(int size, char *msg) {
00257     CmiSyncBroadcastFn1(size,msg);
00258     CmiFree(msg);
00259 }
00260 #else
00261 #define  CmiSyncBroadcastFn1(s,m)      CmiSyncBroadcastFn(s,m)
00262 #endif
00263 
00264 #if USE_COMMON_ASYNC_BCAST
00265 /* FIXME: should use spanning or hypercube, but luckily async is never used */
00266 CmiCommHandle CmiAsyncBroadcastFn(int size, char *msg) {
00267     /*CmiPrintf("In  AsyncBroadcast broadcast\n");*/
00268     CmiAbort("CmiAsyncBroadcastFn should never be called");
00269     return 0;
00270 }
00271 #endif
00272 
00273 /* Functions regarding broadcat op that sends to every one */
00274 void CmiSyncBroadcastAllFn(int size, char *msg) {
00275     void *newmsg = msg;
00276 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00277       /* need to copy the msg in case the msg is on the stack */
00278       /* and we only need to copy when sending out network */
00279     if (CmiNumNodes()>1) newmsg = CopyMsg(msg, size);
00280 #endif
00281     CmiSyncSendFn(CmiMyPe(), size, newmsg) ;
00282     CmiSyncBroadcastFn1(size, newmsg);
00283 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00284     if (newmsg != msg) CmiFree(newmsg);
00285 #endif
00286 }
00287 
00288 void CmiFreeBroadcastAllFn(int size, char *msg) {
00289     CmiSyncBroadcastFn1(size, msg);
00290 #if CMK_BROADCAST_SPANNING_TREE && CMK_BROADCAST_USE_CMIREFERENCE
00291       /* need to copy the msg in case the msg is on the stack */
00292       /* and we only need to copy when sending out network */
00293     if (CmiNumNodes()>1 && CmiGetReference(msg)>1) {
00294       void *newmsg = CopyMsg(msg, size);
00295       CmiFree(msg);
00296       msg = newmsg;
00297     }
00298 #endif
00299     CmiSendSelf(msg);
00300 }
00301 
00302 CmiCommHandle CmiAsyncBroadcastAllFn(int size, char *msg) {
00303     CmiSendSelf(CopyMsg(msg, size));
00304     return CmiAsyncBroadcastFn(size, msg);
00305 }
00306 
00307 #if CMK_NODE_QUEUE_AVAILABLE
00308 #if USE_COMMON_SYNC_BCAST
00309 void CmiSyncNodeBroadcastFn(int size, char *msg) {
00310     int mynode = CmiMyNode();
00311     int i;
00312     CQdCreate(CpvAccess(cQdState), CmiNumNodes()-1);
00313 #if CMK_BROADCAST_SPANNING_TREE
00314     CMI_SET_BROADCAST_ROOT(msg, -CmiMyNode()-1);
00315     SendSpanningChildrenNode(size, msg);
00316 #elif CMK_BROADCAST_HYPERCUBE
00317     CMI_SET_BROADCAST_ROOT(msg, -CmiMyNode()-1);
00318     SendHyperCubeNode(size, msg);
00319 #else
00320     for (i=mynode+1; i<CmiNumNodes(); i++)
00321         CmiSyncNodeSendFn(i, size, msg);
00322     for (i=0; i<mynode; i++)
00323         CmiSyncNodeSendFn(i, size, msg);
00324 #endif
00325 }
00326 
00327 void CmiFreeNodeBroadcastFn(int size, char *msg) {
00328     CmiSyncNodeBroadcastFn(size, msg);
00329     CmiFree(msg);
00330 }
00331 #endif
00332 
00333 #if USE_COMMON_ASYNC_BCAST
00334 CmiCommHandle CmiAsyncNodeBroadcastFn(int size, char *msg) {
00335     CmiSyncNodeBroadcastFn(size, msg);
00336     return 0;
00337 }
00338 #endif
00339 
00340 void CmiSyncNodeBroadcastAllFn(int size, char *msg) {
00341     CmiSyncNodeSendFn(CmiMyNode(), size, msg);
00342     CmiSyncNodeBroadcastFn(size, msg);
00343 }
00344 
00345 CmiCommHandle CmiAsyncNodeBroadcastAllFn(int size, char *msg) {
00346     CmiSendNodeSelf(CopyMsg(msg, size));
00347     return CmiAsyncNodeBroadcastFn(size, msg);
00348 }
00349 
00350 void CmiFreeNodeBroadcastAllFn(int size, char *msg) {
00351     CmiSyncNodeBroadcastFn(size, msg);
00352     /* Since it's a node-level msg, the msg could be executed on any other
00353      * procs on the same node. This means, the push of this msg to the
00354      * node-level queue could be immediately followed a pop of this msg on
00355      * other cores on the same node even when this msg has not been sent to
00356      * other nodes. This is the reason CmiSendNodeSelf must be called after
00357      * CmiSyncNodeBroadcastFn 
00358      */
00359     CmiSendNodeSelf(msg);
00360 }
00361 #endif
00362 /* ##### End of Functions Related with Message Sending OPs ##### */
00363 
00364 #if ! CMK_MULTICAST_LIST_USE_COMMON_CODE
00365 
00366 void CmiSyncListSendFn(int npes, int *pes, int len, char *msg)
00367 {
00368     LrtsSyncListSendFn(npes, pes, len, msg);
00369 }
00370 
00371 CmiCommHandle CmiAsyncListSendFn(int npes, int *pes, int len, char *msg)
00372 {
00373     return LrtsAsyncListSendFn(npes, pes, len, msg);
00374 }
00375 
00376 void CmiFreeListSendFn(int npes, int *pes, int len, char *msg)
00377 {
00378     LrtsFreeListSendFn(npes, pes, len, msg);
00379 }
00380 
00381 #endif
arch/util/machine-broadcast.c
