arch/util/machine-smp.c

Go to the documentation of this file.
00001 
00065 /*
00066 for SMP versions:
00067 
00068 CmiStateInit
00069 CmiNodeStateInit
00070 CmiGetState
00071 CmiGetStateN
00072 CmiYield
00073 CmiStartThreads
00074 
00075 CmiIdleLock_init
00076 CmiIdleLock_sleep
00077 CmiIdleLock_addMessage
00078 CmiIdleLock_checkMessage
00079 */
00080 
00081 #include "machine-smp.h"
00082 
00083 void CmiStateInit(int pe, int rank, CmiState state);
00084 void CommunicationServerInit();
00085 
00086 /************************ Win32 kernel SMP threads **************/
00087 static struct CmiStateStruct Cmi_default_state; /* State structure to return during startup */
00088 
00089 #if CMK_SHARED_VARS_NT_THREADS
00090 
00091 CmiNodeLock CmiMemLock_lock;
00092 static HANDLE comm_mutex;
00093 #define CmiCommLockOrElse(x) /*empty*/
00094 #define CmiCommLock() (WaitForSingleObject(comm_mutex, INFINITE))
00095 #define CmiCommUnlock() (ReleaseMutex(comm_mutex))
00096 
00097 static DWORD Cmi_state_key = 0xFFFFFFFF;
00098 static CmiState     Cmi_state_vector = 0;
00099 
00100 #if 0
00101 #  define CmiGetState() ((CmiState)TlsGetValue(Cmi_state_key))
00102 #else
00103 CmiState CmiGetState()
00104 {
00105   CmiState result;
00106   result = (CmiState)TlsGetValue(Cmi_state_key);
00107   if(result == 0) {
00108         return &Cmi_default_state;
00109         /* PerrorExit("CmiGetState: TlsGetValue");*/
00110   }
00111   return result;
00112 }
00113 #endif
00114 
00115 CmiNodeLock CmiCreateLock(void)
00116 {
00117   HANDLE hMutex = CreateMutex(NULL, FALSE, NULL);
00118   return hMutex;
00119 }
00120 
00121 void CmiDestroyLock(CmiNodeLock lk)
00122 {
00123   CloseHandle(lk);
00124 }
00125 
00126 void CmiYield(void) 
00127 { 
00128   Sleep(0);
00129 }
00130 
00131 #define CmiGetStateN(n) (Cmi_state_vector+(n))
00132 
00133 /*
00134 static DWORD WINAPI comm_thread(LPVOID dummy)
00135 {  
00136   if (Cmi_charmrun_fd!=-1)
00137     while (1) CommunicationServerThread(5);
00138   return 0;
00139 }
00140 
00141 static DWORD WINAPI call_startfn(LPVOID vindex)
00142 {
00143   int index = (int)vindex;
00144  
00145   CmiState state = Cmi_state_vector + index;
00146   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc");
00147   if(TlsSetValue(Cmi_state_key, (LPVOID)state) == 0) PerrorExit("TlsSetValue");
00148 
00149   ConverseRunPE(0);
00150   return 0;
00151 }
00152 */
00153 
00154 static DWORD WINAPI call_startfn(LPVOID vindex)
00155 {
00156   int index = (int)vindex;
00157  
00158   CmiState state = Cmi_state_vector + index;
00159   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc");
00160   if(TlsSetValue(Cmi_state_key, (LPVOID)state) == 0) PerrorExit("TlsSetValue");
00161 
00162   ConverseRunPE(0);
00163 #if 0
00164   if (index<_Cmi_mynodesize)
00165           ConverseRunPE(0); /*Regular worker thread*/
00166   else { /*Communication thread*/
00167           CommunicationServerInit();
00168           if (Cmi_charmrun_fd!=-1)
00169                   while (1) CommunicationServerThread(5);
00170   } 
00171 #endif
00172   return 0;
00173 }
00174 
00175 
00176 /*Classic sense-reversing barrier algorithm.
00177 FIXME: This should be the barrier implementation for 
00178 all thread types.
00179 */
00180 static volatile HANDLE barrier_mutex;
00181 static volatile int    barrier_wait[2] = {0,0};
00182 static volatile int    barrier_which = 0;
00183 
00184 void CmiNodeBarrierCount(int nThreads) {
00185   int doWait = 1;
00186   int which;
00187 
00188   while (WaitForSingleObject(barrier_mutex, INFINITE)!=WAIT_OBJECT_0);
00189   which=barrier_which;
00190   barrier_wait[which]++;
00191   if (barrier_wait[which] == nThreads) {
00192     barrier_which = !which;
00193     barrier_wait[barrier_which] = 0;/*Reset new counter*/
00194     doWait = 0;
00195   }
00196   while (!ReleaseMutex(barrier_mutex));
00197 
00198   if (doWait)
00199       while(barrier_wait[which] != nThreads)
00200                   sleep(0);/*<- could also just spin here*/
00201 }
00202 
00203 static void CmiStartThreads(char **argv)
00204 {
00205   int     i,tocreate;
00206   DWORD   threadID;
00207   HANDLE  thr;
00208 
00209   CmiMemLock_lock=CmiCreateLock();
00210   comm_mutex = CmiCreateLock();
00211   barrier_mutex = CmiCreateLock();
00212 
00213   Cmi_state_key = TlsAlloc();
00214   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc main");
00215   
00216   Cmi_state_vector =
00217     (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct));
00218   
00219   for (i=0; i<_Cmi_mynodesize; i++)
00220     CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i));
00221   /*Create a fake state structure for the comm. thread*/
00222 /*  CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */
00223   CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize));
00224   
00225 #if CMK_MULTICORE
00226   if (!Cmi_commthread)
00227     tocreate = _Cmi_mynodesize-1;
00228   else
00229 #endif
00230   tocreate = _Cmi_mynodesize;
00231   for (i=1; i<=tocreate; i++) {
00232     if((thr = CreateThread(NULL, 0, call_startfn, (LPVOID)i, 0, &threadID)) 
00233        == NULL) PerrorExit("CreateThread");
00234     CloseHandle(thr);
00235   }
00236   
00237   if(TlsSetValue(Cmi_state_key, (LPVOID)Cmi_state_vector) == 0) 
00238     PerrorExit("TlsSetValue");
00239 }
00240 
00241 static void CmiDestoryLocks()
00242 {
00243   CloseHandle(comm_mutex);
00244   CloseHandle(CmiMemLock_lock);
00245   CmiMemLock_lock = 0;
00246   CloseHandle(barrier_mutex);
00247 }
00248 
00249 /***************** Pthreads kernel SMP threads ******************/
00250 #elif CMK_SHARED_VARS_POSIX_THREADS_SMP
00251 
00252 CmiNodeLock CmiMemLock_lock;
00253 int _Cmi_noprocforcommthread=0;/*this variable marks if there is an extra processor for comm thread
00254 in smp*/
00255 
00256 #if CMK_TLS_THREAD && CMK_USE_TLS_THREAD
00257 static __thread struct CmiStateStruct     Cmi_mystate;
00258 static CmiState     *Cmi_state_vector;
00259 
00260 CmiState CmiGetState() {
00261         return &Cmi_mystate;
00262 }
00263 #define CmiGetStateN(n) Cmi_state_vector[n]
00264 
00265 #else
00266 
00267 static pthread_key_t Cmi_state_key=(pthread_key_t)(-1);
00268 static CmiState     Cmi_state_vector;
00269 
00270 #if 0
00271 #define CmiGetState() ((CmiState)pthread_getspecific(Cmi_state_key))
00272 #else
00273 CmiState CmiGetState() {
00274         CmiState ret=(CmiState)pthread_getspecific(Cmi_state_key);
00275         if (ret==NULL) {
00276                 return &Cmi_default_state;
00277         }
00278         return ret;
00279 }
00280 
00281 #endif
00282 #define CmiGetStateN(n) (Cmi_state_vector+(n))
00283 #endif
00284 
00285 
00286 CmiNodeLock CmiCreateLock(void)
00287 {
00288   CmiNodeLock lk = (CmiNodeLock)malloc(sizeof(pthread_mutex_t));
00289   _MEMCHECK(lk);
00290   pthread_mutex_init(lk,(pthread_mutexattr_t *)0);
00291   return lk;
00292 }
00293 
00294 void CmiDestroyLock(CmiNodeLock lk)
00295 {
00296   pthread_mutex_destroy(lk);
00297   free(lk);
00298 }
00299 
00300 void CmiYield(void) { sched_yield(); }
00301 
00302 int barrier = 0;
00303 pthread_cond_t barrier_cond = PTHREAD_COND_INITIALIZER;
00304 pthread_mutex_t barrier_mutex = PTHREAD_MUTEX_INITIALIZER;
00305 
00306 void CmiNodeBarrierCount(int nThreads)
00307 {
00308   static unsigned int volatile level = 0;
00309   unsigned int cur;
00310   pthread_mutex_lock(&barrier_mutex);
00311   cur = level;
00312   /* CmiPrintf("[%d] CmiNodeBarrierCount: %d of %d level:%d\n", CmiMyPe(), barrier, nThreads, level); */
00313   barrier++;
00314   if(barrier != nThreads) {
00315       /* occasionally it wakes up without having reach the count */
00316     while (cur == level)
00317       pthread_cond_wait(&barrier_cond, &barrier_mutex);
00318   }
00319   else{
00320     barrier = 0;
00321     level++;  /* !level;  */
00322     pthread_cond_broadcast(&barrier_cond);
00323   }
00324   pthread_mutex_unlock(&barrier_mutex);
00325 }
00326 
00327 static CmiNodeLock comm_mutex;
00328 
00329 #define CmiCommLockOrElse(x) /*empty*/
00330 
00331 #if 1
00332 /*Regular comm. lock*/
00333 #  define CmiCommLock() CmiLock(comm_mutex)
00334 #  define CmiCommUnlock() CmiUnlock(comm_mutex)
00335 #else
00336 /*Verbose debugging comm. lock*/
00337 static int comm_mutex_isLocked=0;
00338 void CmiCommLock(void) {
00339         if (comm_mutex_isLocked) 
00340                 CmiAbort("CommLock: already locked!\n");
00341         CmiLock(comm_mutex);
00342         comm_mutex_isLocked=1;
00343 }
00344 void CmiCommUnlock(void) {
00345         if (!comm_mutex_isLocked)
00346                 CmiAbort("CommUnlock: double unlock!\n");
00347         comm_mutex_isLocked=0;
00348         CmiUnlock(comm_mutex);
00349 }
00350 #endif
00351 
00352 /*
00353 static void comm_thread(void)
00354 {
00355   while (1) CommunicationServer(5);
00356 }
00357 
00358 static void *call_startfn(void *vindex)
00359 {
00360   int index = (int)vindex;
00361   CmiState state = Cmi_state_vector + index;
00362   pthread_setspecific(Cmi_state_key, state);
00363   ConverseRunPE(0);
00364   return 0;
00365 }
00366 */
00367 
00368 static void *call_startfn(void *vindex)
00369 {
00370   size_t index = (size_t)vindex;
00371 #if CMK_TLS_THREAD && CMK_USE_TLS_THREAD
00372   if (index<_Cmi_mynodesize) 
00373     CmiStateInit(index+Cmi_nodestart, index, &Cmi_mystate);
00374   else
00375     CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,&Cmi_mystate);
00376   Cmi_state_vector[index] = &Cmi_mystate;
00377 #else
00378   CmiState state = Cmi_state_vector + index;
00379   pthread_setspecific(Cmi_state_key, state);
00380 #endif
00381 
00382   ConverseRunPE(0);
00383 #if 0
00384   if (index<_Cmi_mynodesize) 
00385           ConverseRunPE(0); /*Regular worker thread*/
00386   else 
00387   { /*Communication thread*/
00388           CommunicationServerInit();
00389           if (Cmi_charmrun_fd!=-1)
00390                   while (1) CommunicationServer(5,COM_SERVER_FROM_SMP);
00391   }
00392 #endif  
00393   return 0;
00394 }
00395 
00396 static void CmiStartThreads(char **argv)
00397 {
00398   pthread_t pid;
00399   size_t i;
00400   int ok, tocreate;
00401   pthread_attr_t attr;
00402 
00403   CmiMemLock_lock=CmiCreateLock();
00404   comm_mutex=CmiCreateLock();
00405   smp_mutex = CmiCreateLock();
00406 
00407 #if ! (CMK_TLS_THREAD && CMK_USE_TLS_THREAD)
00408   pthread_key_create(&Cmi_state_key, 0);
00409   Cmi_state_vector =
00410     (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct));
00411   for (i=0; i<_Cmi_mynodesize; i++)
00412     CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i));
00413   /*Create a fake state structure for the comm. thread*/
00414 /*  CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */
00415   CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize));
00416 #else
00417   Cmi_state_vector = (CmiState *)calloc(_Cmi_mynodesize+1, sizeof(CmiState));
00418   CmiStateInit(Cmi_nodestart, 0, &Cmi_mystate);
00419   Cmi_state_vector[0] = &Cmi_mystate;
00420 #endif
00421 
00422 #if CMK_MULTICORE
00423   if (!Cmi_commthread)
00424     tocreate = _Cmi_mynodesize-1;
00425   else
00426 #endif
00427   tocreate = _Cmi_mynodesize;
00428   for (i=1; i<=tocreate; i++) {
00429     pthread_attr_init(&attr);
00430     pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
00431     ok = pthread_create(&pid, &attr, call_startfn, (void *)i);
00432     if (ok<0) PerrorExit("pthread_create"); 
00433     pthread_attr_destroy(&attr);
00434   }
00435 #if ! (CMK_TLS_THREAD && CMK_USE_TLS_THREAD)
00436   pthread_setspecific(Cmi_state_key, Cmi_state_vector);
00437 #endif
00438 }
00439 
00440 static void CmiDestoryLocks()
00441 {
00442   pthread_mutex_destroy(comm_mutex);
00443   pthread_mutex_destroy(CmiMemLock_lock);
00444   CmiMemLock_lock = 0;
00445   pthread_mutex_destroy(&barrier_mutex);
00446 }
00447 
00448 #endif
00449 
00450 #if !CMK_SHARED_VARS_UNAVAILABLE
00451 
00452 /* Wait for all worker threads */
00453 void  CmiNodeBarrier(void) {
00454   CmiNodeBarrierCount(CmiMyNodeSize());
00455 }
00456 
00457 /* Wait for all worker threads as well as comm. thread */
00458 /* unfortunately this could also be called in a seemingly non smp version
00459    net-win32, which actually is implemented as smp with comm. thread */
00460 void CmiNodeAllBarrier(void) {
00461 #if CMK_MULTICORE
00462   if (!Cmi_commthread)
00463   CmiNodeBarrierCount(CmiMyNodeSize());
00464   else
00465 #endif
00466   CmiNodeBarrierCount(CmiMyNodeSize()+1);
00467 }
00468 
00469 #endif
00470 
00471 /***********************************************************
00472  * SMP Idle Locking
00473  *   In an SMP system, idle processors need to sleep on a
00474  * lock so that if a message for them arrives, they can be
00475  * woken up.
00476  **********************************************************/
00477 
00478 #if CMK_SHARED_VARS_NT_THREADS
00479 
00480 static void CmiIdleLock_init(CmiIdleLock *l) {
00481   l->hasMessages=0;
00482   l->isSleeping=0;
00483   l->sem=CreateSemaphore(NULL,0,1, NULL);
00484 }
00485 
00486 static void CmiIdleLock_sleep(CmiIdleLock *l,int msTimeout) {
00487   if (l->hasMessages) return;
00488   l->isSleeping=1;
00489   MACHSTATE(4,"Processor going to sleep {")
00490   WaitForSingleObject(l->sem,msTimeout);
00491   MACHSTATE(4,"} Processor awake again")
00492   l->isSleeping=0;
00493 }
00494 
00495 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00496   l->hasMessages=1;
00497   if (l->isSleeping) { /*The PE is sleeping on this lock-- wake him*/  
00498     MACHSTATE(4,"Waking sleeping processor")
00499     ReleaseSemaphore(l->sem,1,NULL);
00500   }
00501 }
00502 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00503   l->hasMessages=0;
00504 }
00505 
00506 #elif CMK_SHARED_VARS_POSIX_THREADS_SMP
00507 
00508 static void CmiIdleLock_init(CmiIdleLock *l) {
00509   l->hasMessages=0;
00510   l->isSleeping=0;
00511   pthread_mutex_init(&l->mutex,NULL);
00512   pthread_cond_init(&l->cond,NULL);
00513 }
00514 
00515 static void getTimespec(int msFromNow,struct timespec *dest) {
00516   struct timeval cur;
00517   int secFromNow;
00518   /*Get the current time*/
00519   gettimeofday(&cur,NULL);
00520   dest->tv_sec=cur.tv_sec;
00521   dest->tv_nsec=cur.tv_usec*1000;
00522   /*Add in the wait time*/
00523   secFromNow=msFromNow/1000;
00524   msFromNow-=secFromNow*1000;
00525   dest->tv_sec+=secFromNow;
00526   dest->tv_nsec+=1000*1000*msFromNow;
00527   /*Wrap around if we overflowed the nsec field*/
00528   while (dest->tv_nsec>=1000000000ul) {
00529     dest->tv_nsec-=1000000000ul;
00530     dest->tv_sec++;
00531   }
00532 }
00533 
00534 static void CmiIdleLock_sleep(CmiIdleLock *l,int msTimeout) {
00535   struct timespec wakeup;
00536 
00537   if (l->hasMessages) return;
00538   l->isSleeping=1;
00539   MACHSTATE(4,"Processor going to sleep {")
00540   pthread_mutex_lock(&l->mutex);
00541   getTimespec(msTimeout,&wakeup);
00542   while (!l->hasMessages)
00543     if (ETIMEDOUT==pthread_cond_timedwait(&l->cond,&l->mutex,&wakeup))
00544       break;
00545   pthread_mutex_unlock(&l->mutex);
00546   MACHSTATE(4,"} Processor awake again")
00547   l->isSleeping=0;
00548 }
00549 
00550 static void CmiIdleLock_wakeup(CmiIdleLock *l) {
00551   l->hasMessages=1; 
00552   MACHSTATE(4,"Waking sleeping processor")
00553   /*The PE is sleeping on this condition variable-- wake him*/
00554   pthread_mutex_lock(&l->mutex);
00555   pthread_cond_signal(&l->cond);
00556   pthread_mutex_unlock(&l->mutex);
00557 }
00558 
00559 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00560   if (l->isSleeping) CmiIdleLock_wakeup(l);
00561   l->hasMessages=1;
00562 }
00563 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00564   l->hasMessages=0;
00565 }
00566 #else
00567 #define CmiIdleLock_sleep(x, y) /*empty*/
00568 
00569 static void CmiIdleLock_init(CmiIdleLock *l) {
00570   l->hasMessages=0;
00571 }
00572 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00573   l->hasMessages=1;
00574 }
00575 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00576   l->hasMessages=0;
00577 }
00578 #endif
00579 
00580 void CmiStateInit(int pe, int rank, CmiState state)
00581 {
00582   MACHSTATE(4,"StateInit")
00583   state->pe = pe;
00584   state->rank = rank;
00585   if (rank==CmiMyNodeSize()) return; /* Communications thread */
00586   state->recv = PCQueueCreate();
00587   state->localqueue = CdsFifo_Create();
00588   CmiIdleLock_init(&state->idle);
00589 }
00590 
00591 void CmiNodeStateInit(CmiNodeState *nodeState)
00592 {
00593 #if CMK_IMMEDIATE_MSG
00594   MACHSTATE(4,"NodeStateInit")
00595   nodeState->immSendLock = CmiCreateLock();
00596   nodeState->immRecvLock = CmiCreateLock();
00597   nodeState->immQ = PCQueueCreate();
00598   nodeState->delayedImmQ = PCQueueCreate();
00599 #endif
00600 #if CMK_NODE_QUEUE_AVAILABLE
00601   nodeState->CmiNodeRecvLock = CmiCreateLock();
00602   nodeState->NodeRecv = PCQueueCreate();
00603 #endif
00604 }
00605 

Generated on Sun Jun 29 13:29:06 2008 for Charm++ by  doxygen 1.5.1