Charm++: Charm Source Code Documentation

00001 
00065 /*
00066 for SMP versions:
00067 
00068 CmiStateInit
00069 CmiNodeStateInit
00070 CmiGetState
00071 CmiGetStateN
00072 CmiYield
00073 CmiStartThreads
00074 
00075 CmiIdleLock_init
00076 CmiIdleLock_sleep
00077 CmiIdleLock_addMessage
00078 CmiIdleLock_checkMessage
00079 */
00080 
00081 #include "machine-smp.h"
00082 
00083 void CmiStateInit(int pe, int rank, CmiState state);
00084 void CommunicationServerInit();
00085 
00086 static struct CmiStateStruct Cmi_default_state; /* State structure to return during startup */
00087 
00088 /************************ Win32 kernel SMP threads **************/
00089 
00090 #if CMK_SHARED_VARS_NT_THREADS
00091 
00092 CmiNodeLock CmiMemLock_lock;
00093 #ifdef CMK_NO_ASM_AVAILABLE
00094 CmiNodeLock cmiMemoryLock;
00095 #endif
00096 static HANDLE comm_mutex;
00097 #define CmiCommLockOrElse(x) /*empty*/
00098 #define CmiCommLock() (WaitForSingleObject(comm_mutex, INFINITE))
00099 #define CmiCommUnlock() (ReleaseMutex(comm_mutex))
00100 
00101 static DWORD Cmi_state_key = 0xFFFFFFFF;
00102 static CmiState     Cmi_state_vector = 0;
00103 
00104 #if 0
00105 #  define CmiGetState() ((CmiState)TlsGetValue(Cmi_state_key))
00106 #else
00107 CmiState CmiGetState()
00108 {
00109   CmiState result;
00110   result = (CmiState)TlsGetValue(Cmi_state_key);
00111   if(result == 0) {
00112         return &Cmi_default_state;
00113         /* PerrorExit("CmiGetState: TlsGetValue");*/
00114   }
00115   return result;
00116 }
00117 #endif
00118 
00119 CmiNodeLock CmiCreateLock(void)
00120 {
00121   HANDLE hMutex = CreateMutex(NULL, FALSE, NULL);
00122   return hMutex;
00123 }
00124 
00125 void CmiDestroyLock(CmiNodeLock lk)
00126 {
00127   CloseHandle(lk);
00128 }
00129 
00130 void CmiYield(void) 
00131 { 
00132   Sleep(0);
00133 }
00134 
00135 #define CmiGetStateN(n) (Cmi_state_vector+(n))
00136 
00137 /*
00138 static DWORD WINAPI comm_thread(LPVOID dummy)
00139 {  
00140   if (Cmi_charmrun_fd!=-1)
00141     while (1) CommunicationServerThread(5);
00142   return 0;
00143 }
00144 
00145 static DWORD WINAPI call_startfn(LPVOID vindex)
00146 {
00147   int index = (int)vindex;
00148  
00149   CmiState state = Cmi_state_vector + index;
00150   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc");
00151   if(TlsSetValue(Cmi_state_key, (LPVOID)state) == 0) PerrorExit("TlsSetValue");
00152 
00153   ConverseRunPE(0);
00154   return 0;
00155 }
00156 */
00157 
00158 static DWORD WINAPI call_startfn(LPVOID vindex)
00159 {
00160   int index = (int)vindex;
00161  
00162   CmiState state = Cmi_state_vector + index;
00163   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc");
00164   if(TlsSetValue(Cmi_state_key, (LPVOID)state) == 0) PerrorExit("TlsSetValue");
00165 
00166   ConverseRunPE(0);
00167 #if 0
00168   if (index<_Cmi_mynodesize)
00169           ConverseRunPE(0); /*Regular worker thread*/
00170   else { /*Communication thread*/
00171           CommunicationServerInit();
00172           if (Cmi_charmrun_fd!=-1)
00173                   while (1) CommunicationServerThread(5);
00174   } 
00175 #endif
00176   return 0;
00177 }
00178 
00179 
00180 /*Classic sense-reversing barrier algorithm.
00181 FIXME: This should be the barrier implementation for 
00182 all thread types.
00183 */
00184 static volatile HANDLE barrier_mutex;
00185 static volatile int    barrier_wait[2] = {0,0};
00186 static volatile int    barrier_which = 0;
00187 
00188 void CmiNodeBarrierCount(int nThreads) {
00189   int doWait = 1;
00190   int which;
00191 
00192   while (WaitForSingleObject(barrier_mutex, INFINITE)!=WAIT_OBJECT_0);
00193   which=barrier_which;
00194   barrier_wait[which]++;
00195   if (barrier_wait[which] == nThreads) {
00196     barrier_which = !which;
00197     barrier_wait[barrier_which] = 0;/*Reset new counter*/
00198     doWait = 0;
00199   }
00200   while (!ReleaseMutex(barrier_mutex));
00201 
00202   if (doWait)
00203       while(barrier_wait[which] != nThreads)
00204                   sleep(0);/*<- could also just spin here*/
00205 }
00206 
00207 static void CmiStartThreads(char **argv)
00208 {
00209   int     i,tocreate;
00210   DWORD   threadID;
00211   HANDLE  thr;
00212 
00213   CmiMemLock_lock=CmiCreateLock();
00214   comm_mutex = CmiCreateLock();
00215   barrier_mutex = CmiCreateLock();
00216 #ifdef CMK_NO_ASM_AVAILABLE
00217   cmiMemoryLock = CmiCreateLock();
00218   if (CmiMyNode()==0) printf("Charm++ warning> fences and atomic operations not available in native assembly\n");
00219 #endif
00220 
00221   Cmi_state_key = TlsAlloc();
00222   if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc main");
00223   
00224   Cmi_state_vector =
00225     (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct));
00226   
00227   for (i=0; i<_Cmi_mynodesize; i++)
00228     CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i));
00229   /*Create a fake state structure for the comm. thread*/
00230 /*  CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */
00231   CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize));
00232   
00233 #if CMK_MULTICORE || CMK_SMP_NO_COMMTHD
00234   if (!Cmi_commthread)
00235     tocreate = _Cmi_mynodesize-1;
00236   else
00237 #endif
00238   tocreate = _Cmi_mynodesize;
00239   for (i=1; i<=tocreate; i++) {
00240     if((thr = CreateThread(NULL, 0, call_startfn, (LPVOID)i, 0, &threadID)) 
00241        == NULL) PerrorExit("CreateThread");
00242     CloseHandle(thr);
00243   }
00244   
00245   if(TlsSetValue(Cmi_state_key, (LPVOID)Cmi_state_vector) == 0) 
00246     PerrorExit("TlsSetValue");
00247 }
00248 
00249 static void CmiDestoryLocks()
00250 {
00251   CloseHandle(comm_mutex);
00252   CloseHandle(CmiMemLock_lock);
00253   CmiMemLock_lock = 0;
00254   CloseHandle(barrier_mutex);
00255 #ifdef CMK_NO_ASM_AVAILABLE
00256   CloseHandle(cmiMemoryLock);
00257 #endif
00258 }
00259 
00260 /***************** Pthreads kernel SMP threads ******************/
00261 #elif CMK_SHARED_VARS_POSIX_THREADS_SMP
00262 
00263 CmiNodeLock CmiMemLock_lock;
00264 #ifdef CMK_NO_ASM_AVAILABLE
00265 CmiNodeLock cmiMemoryLock;
00266 #endif
00267 int _Cmi_noprocforcommthread=0;/*this variable marks if there is an extra processor for comm thread
00268 in smp*/
00269 
00270 #if CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD
00271 static __thread struct CmiStateStruct     Cmi_mystate;
00272 static CmiState     *Cmi_state_vector;
00273 
00274 CmiState CmiGetState() {
00275         return &Cmi_mystate;
00276 }
00277 #define CmiGetStateN(n) Cmi_state_vector[n]
00278 
00279 #else
00280 
00281 static pthread_key_t Cmi_state_key=(pthread_key_t)(-1);
00282 static CmiState     Cmi_state_vector;
00283 
00284 #if 0
00285 #define CmiGetState() ((CmiState)pthread_getspecific(Cmi_state_key))
00286 #else
00287 CmiState CmiGetState() {
00288         CmiState ret=(CmiState)pthread_getspecific(Cmi_state_key);
00289         if (ret==NULL || Cmi_state_key == (pthread_key_t)(-1)) {
00290                 return &Cmi_default_state;
00291         }
00292         return ret;
00293 }
00294 
00295 #endif
00296 #define CmiGetStateN(n) (Cmi_state_vector+(n))
00297 #endif
00298 
00299 
00300 #if CMK_HAS_SPINLOCK && CMK_USE_SPINLOCK
00301 CmiNodeLock CmiCreateLock()
00302 {
00303   CmiNodeLock lk = (CmiNodeLock)malloc(sizeof(pthread_spinlock_t));
00304   _MEMCHECK(lk);
00305   pthread_spin_init(lk, 0);
00306   return lk;
00307 }
00308 
00309 void CmiDestroyLock(CmiNodeLock lk)
00310 {
00311   pthread_spin_destroy(lk);
00312   free((void*)lk);
00313 }
00314 #else
00315 CmiNodeLock CmiCreateLock()
00316 {
00317   CmiNodeLock lk = (CmiNodeLock)malloc(sizeof(pthread_mutex_t));
00318   _MEMCHECK(lk);
00319   pthread_mutex_init(lk,(pthread_mutexattr_t *)0);
00320   return lk;
00321 }
00322 
00323 void CmiDestroyLock(CmiNodeLock lk)
00324 {
00325   pthread_mutex_destroy(lk);
00326   free(lk);
00327 }
00328 #endif
00329 
00330 void CmiYield(void) { sched_yield(); }
00331 
00332 int barrier = 0;
00333 pthread_cond_t barrier_cond = PTHREAD_COND_INITIALIZER;
00334 pthread_mutex_t barrier_mutex = PTHREAD_MUTEX_INITIALIZER;
00335 
00336 void CmiNodeBarrierCount(int nThreads)
00337 {
00338   static unsigned int volatile level = 0;
00339   unsigned int cur;
00340   pthread_mutex_lock(&barrier_mutex);
00341   cur = level;
00342   /* CmiPrintf("[%d] CmiNodeBarrierCount: %d of %d level:%d\n", CmiMyPe(), barrier, nThreads, level); */
00343   barrier++;
00344   if(barrier != nThreads) {
00345       /* occasionally it wakes up without having reach the count */
00346     while (cur == level)
00347       pthread_cond_wait(&barrier_cond, &barrier_mutex);
00348   }
00349   else{
00350     barrier = 0;
00351     level++;  /* !level;  */
00352     pthread_cond_broadcast(&barrier_cond);
00353   }
00354   pthread_mutex_unlock(&barrier_mutex);
00355 }
00356 
00357 static CmiNodeLock comm_mutex;
00358 
00359 #define CmiCommLockOrElse(x) /*empty*/
00360 
00361 #if 1
00362 /*Regular comm. lock*/
00363 #  define CmiCommLock() CmiLock(comm_mutex)
00364 #  define CmiCommUnlock() CmiUnlock(comm_mutex)
00365 #else
00366 /*Verbose debugging comm. lock*/
00367 static int comm_mutex_isLocked=0;
00368 void CmiCommLock(void) {
00369         if (comm_mutex_isLocked) 
00370                 CmiAbort("CommLock: already locked!\n");
00371         CmiLock(comm_mutex);
00372         comm_mutex_isLocked=1;
00373 }
00374 void CmiCommUnlock(void) {
00375         if (!comm_mutex_isLocked)
00376                 CmiAbort("CommUnlock: double unlock!\n");
00377         comm_mutex_isLocked=0;
00378         CmiUnlock(comm_mutex);
00379 }
00380 #endif
00381 
00382 /*
00383 static void comm_thread(void)
00384 {
00385   while (1) CommunicationServer(5);
00386 }
00387 
00388 static void *call_startfn(void *vindex)
00389 {
00390   int index = (int)vindex;
00391   CmiState state = Cmi_state_vector + index;
00392   pthread_setspecific(Cmi_state_key, state);
00393   ConverseRunPE(0);
00394   return 0;
00395 }
00396 */
00397 
00398 static void *call_startfn(void *vindex)
00399 {
00400   size_t index = (size_t)vindex;
00401 #if CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD
00402   if (index<_Cmi_mynodesize) 
00403     CmiStateInit(index+Cmi_nodestart, index, &Cmi_mystate);
00404   else
00405     CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,&Cmi_mystate);
00406   Cmi_state_vector[index] = &Cmi_mystate;
00407 #else
00408   CmiState state = Cmi_state_vector + index;
00409   pthread_setspecific(Cmi_state_key, state);
00410 #endif
00411 
00412   ConverseRunPE(0);
00413 #if 0
00414   if (index<_Cmi_mynodesize) 
00415           ConverseRunPE(0); /*Regular worker thread*/
00416   else 
00417   { /*Communication thread*/
00418           CommunicationServerInit();
00419           if (Cmi_charmrun_fd!=-1)
00420                   while (1) CommunicationServer(5,COM_SERVER_FROM_SMP);
00421   }
00422 #endif  
00423   return 0;
00424 }
00425 
00426 static void CmiStartThreads(char **argv)
00427 {
00428   pthread_t pid;
00429   size_t i;
00430   int ok, tocreate;
00431   pthread_attr_t attr;
00432 
00433   MACHSTATE(4,"CmiStartThreads")
00434   CmiMemLock_lock=CmiCreateLock();
00435   comm_mutex=CmiCreateLock();
00436   _smp_mutex = CmiCreateLock();
00437 #if defined(CMK_NO_ASM_AVAILABLE) && CMK_PCQUEUE_LOCK
00438   cmiMemoryLock = CmiCreateLock();
00439   if (CmiMyNode()==0) printf("Charm++ warning> fences and atomic operations not available in native assembly\n");
00440 #endif
00441 
00442 #if ! (CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD)
00443   pthread_key_create(&Cmi_state_key, 0);
00444   Cmi_state_vector =
00445     (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct));
00446   for (i=0; i<_Cmi_mynodesize; i++)
00447     CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i));
00448   /*Create a fake state structure for the comm. thread*/
00449 /*  CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */
00450   CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize));
00451 #else
00452     /* for main thread */
00453   Cmi_state_vector = (CmiState *)calloc(_Cmi_mynodesize+1, sizeof(CmiState));
00454 #if CMK_CONVERSE_MPI
00455       /* main thread is communication thread */
00456   CmiStateInit(_Cmi_mynode+CmiNumPes(), _Cmi_mynodesize, &Cmi_mystate);
00457   Cmi_state_vector[_Cmi_mynodesize] = &Cmi_mystate;
00458 #else
00459       /* main thread is of rank 0 */
00460   CmiStateInit(Cmi_nodestart, 0, &Cmi_mystate);
00461   Cmi_state_vector[0] = &Cmi_mystate;
00462 #endif
00463 #endif
00464 
00465 #if CMK_MULTICORE || CMK_SMP_NO_COMMTHD
00466   if (!Cmi_commthread)
00467     tocreate = _Cmi_mynodesize-1;
00468   else
00469 #endif
00470   tocreate = _Cmi_mynodesize;
00471 #if CMK_CONVERSE_MPI
00472   for (i=0; i<=tocreate-1; i++) {          /* skip comm thread */
00473 #else
00474   for (i=1; i<=tocreate; i++) {            /* skip rank 0 main thread */
00475 #endif
00476     pthread_attr_init(&attr);
00477     pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
00478     ok = pthread_create(&pid, &attr, call_startfn, (void *)i);
00479     if (ok<0) PerrorExit("pthread_create"); 
00480     pthread_attr_destroy(&attr);
00481   }
00482 #if ! (CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD)
00483 #if CMK_CONVERSE_MPI
00484   pthread_setspecific(Cmi_state_key, Cmi_state_vector+_Cmi_mynodesize);
00485 #else
00486   pthread_setspecific(Cmi_state_key, Cmi_state_vector);
00487 #endif
00488 #endif
00489 
00490   MACHSTATE(4,"CmiStartThreads done")
00491 }
00492 
00493 static void CmiDestoryLocks()
00494 {
00495   CmiDestroyLock(comm_mutex);
00496   CmiDestroyLock(CmiMemLock_lock);
00497   CmiMemLock_lock = 0;
00498   pthread_mutex_destroy(&barrier_mutex);
00499 #ifdef CMK_NO_ASM_AVAILABLE
00500   pthread_mutex_destroy(cmiMemoryLock);
00501 #endif
00502 }
00503 
00504 #endif
00505 
00506 #if !CMK_SHARED_VARS_UNAVAILABLE
00507 
00508 /* Wait for all worker threads */
00509 void  CmiNodeBarrier(void) {
00510   CmiNodeBarrierCount(CmiMyNodeSize());
00511 }
00512 
00513 /* Wait for all worker threads as well as comm. thread */
00514 /* unfortunately this could also be called in a seemingly non smp version
00515    net-win32, which actually is implemented as smp with comm. thread */
00516 void CmiNodeAllBarrier(void) {
00517 #if CMK_MULTICORE || CMK_SMP_NO_COMMTHD
00518   if (!Cmi_commthread)
00519   CmiNodeBarrierCount(CmiMyNodeSize());
00520   else
00521 #endif
00522   CmiNodeBarrierCount(CmiMyNodeSize()+1);
00523 }
00524 
00525 #endif
00526 
00527 /***********************************************************
00528  * SMP Idle Locking
00529  *   In an SMP system, idle processors need to sleep on a
00530  * lock so that if a message for them arrives, they can be
00531  * woken up.
00532  **********************************************************/
00533 
00534 #if CMK_SHARED_VARS_NT_THREADS
00535 
00536 static void CmiIdleLock_init(CmiIdleLock *l) {
00537   l->hasMessages=0;
00538   l->isSleeping=0;
00539   l->sem=CreateSemaphore(NULL,0,1, NULL);
00540 }
00541 
00542 static void CmiIdleLock_sleep(CmiIdleLock *l,int msTimeout) {
00543   if (l->hasMessages) return;
00544   l->isSleeping=1;
00545   MACHSTATE(4,"Processor going to sleep {")
00546   WaitForSingleObject(l->sem,msTimeout);
00547   MACHSTATE(4,"} Processor awake again")
00548   l->isSleeping=0;
00549 }
00550 
00551 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00552   l->hasMessages=1;
00553   if (l->isSleeping) { /*The PE is sleeping on this lock-- wake him*/  
00554     MACHSTATE(4,"Waking sleeping processor")
00555     ReleaseSemaphore(l->sem,1,NULL);
00556   }
00557 }
00558 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00559   l->hasMessages=0;
00560 }
00561 
00562 #elif CMK_SHARED_VARS_POSIX_THREADS_SMP
00563 
00564 static void CmiIdleLock_init(CmiIdleLock *l) {
00565   l->hasMessages=0;
00566   l->isSleeping=0;
00567   pthread_mutex_init(&l->mutex,NULL);
00568   pthread_cond_init(&l->cond,NULL);
00569 }
00570 
00571 static void getTimespec(int msFromNow,struct timespec *dest) {
00572   struct timeval cur;
00573   int secFromNow;
00574   /*Get the current time*/
00575   gettimeofday(&cur,NULL);
00576   dest->tv_sec=cur.tv_sec;
00577   dest->tv_nsec=cur.tv_usec*1000;
00578   /*Add in the wait time*/
00579   secFromNow=msFromNow/1000;
00580   msFromNow-=secFromNow*1000;
00581   dest->tv_sec+=secFromNow;
00582   dest->tv_nsec+=1000*1000*msFromNow;
00583   /*Wrap around if we overflowed the nsec field*/
00584   while (dest->tv_nsec>=1000000000ul) {
00585     dest->tv_nsec-=1000000000ul;
00586     dest->tv_sec++;
00587   }
00588 }
00589 
00590 static void CmiIdleLock_sleep(CmiIdleLock *l,int msTimeout) {
00591   struct timespec wakeup;
00592 
00593   if (l->hasMessages) return;
00594   l->isSleeping=1;
00595   MACHSTATE(4,"Processor going to sleep {")
00596   pthread_mutex_lock(&l->mutex);
00597   getTimespec(msTimeout,&wakeup);
00598   while (!l->hasMessages)
00599     if (ETIMEDOUT==pthread_cond_timedwait(&l->cond,&l->mutex,&wakeup))
00600       break;
00601   pthread_mutex_unlock(&l->mutex);
00602   MACHSTATE(4,"} Processor awake again")
00603   l->isSleeping=0;
00604 }
00605 
00606 static void CmiIdleLock_wakeup(CmiIdleLock *l) {
00607   l->hasMessages=1; 
00608   MACHSTATE(4,"Waking sleeping processor")
00609   /*The PE is sleeping on this condition variable-- wake him*/
00610   pthread_mutex_lock(&l->mutex);
00611   pthread_cond_signal(&l->cond);
00612   pthread_mutex_unlock(&l->mutex);
00613 }
00614 
00615 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00616   if (l->isSleeping) CmiIdleLock_wakeup(l);
00617   l->hasMessages=1;
00618 }
00619 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00620   l->hasMessages=0;
00621 }
00622 #else
00623 #define CmiIdleLock_sleep(x, y) /*empty*/
00624 
00625 static void CmiIdleLock_init(CmiIdleLock *l) {
00626   l->hasMessages=0;
00627 }
00628 static void CmiIdleLock_addMessage(CmiIdleLock *l) {
00629   l->hasMessages=1;
00630 }
00631 static void CmiIdleLock_checkMessage(CmiIdleLock *l) {
00632   l->hasMessages=0;
00633 }
00634 #endif
00635 
00636 void CmiStateInit(int pe, int rank, CmiState state)
00637 {
00638 #if CMK_SMP_MULTIQ
00639   int i;
00640 #endif
00641 
00642   MACHSTATE(4,"StateInit")
00643   state->pe = pe;
00644   state->rank = rank;
00645   if (rank==CmiMyNodeSize()) return; /* Communications thread */
00646 #if !CMK_SMP_MULTIQ
00647   state->recv = PCQueueCreate();
00648 #else
00649   for(i=0; i<MULTIQ_GRPSIZE; i++) state->recv[i]=PCQueueCreate();
00650   state->myGrpIdx = rank % MULTIQ_GRPSIZE;
00651   state->curPolledIdx = 0;
00652 #endif
00653   state->localqueue = CdsFifo_Create();
00654   CmiIdleLock_init(&state->idle);
00655 }
00656 
00657 void CmiNodeStateInit(CmiNodeState *nodeState)
00658 {
00659   MACHSTATE1(4,"NodeStateInit %p", nodeState)
00660 #if CMK_IMMEDIATE_MSG
00661   nodeState->immSendLock = CmiCreateLock();
00662   nodeState->immRecvLock = CmiCreateLock();
00663   nodeState->immQ = PCQueueCreate();
00664   nodeState->delayedImmQ = PCQueueCreate();
00665 #endif
00666 #if CMK_NODE_QUEUE_AVAILABLE
00667   nodeState->CmiNodeRecvLock = CmiCreateLock();
00668   nodeState->NodeRecv = PCQueueCreate();
00669 #endif
00670   MACHSTATE(4,"NodeStateInit done")
00671 }
00672
arch/util/machine-smp.c
