Charm++: Charm Source Code Documentation

00001 /*  Library to manage the program's global offset table.
00002  *
00003  *  The global offset table (GOT) is a static, linker-generated 
00004  *  table used to look up the locations of dynamic stuff
00005  *  like subroutines or global data.
00006  *
00007  *  It's only generated and used if you're on an ELF binary
00008  *  format machine, and compile with "-fpic", for "position 
00009  *  independent code"; otherwise the code includes direct 
00010  *  references to subroutines and data.
00011  *
00012  *  During execution of a single routine, the GOT is pointed to
00013  *  by %ebx.  Changing %ebx only affects global access within
00014  *  the current subroutine, so we have to change the single, 
00015  *  static copy of the GOT to switch between sets of global variables.
00016  *  This is slow, but because the GOT just contains pointers,
00017  *  it's not as slow as copying the data in and out of the GOT.
00018  *
00019  
00020 The ELF GOT layout is described in excruciating detail
00021 by the Sun documentation, at 
00022    Solaris 2.5 Software Developer AnswerBook >> 
00023       Linker and Libraries Guide >> 
00024         6 Object Files >> 
00025           File Format
00026 
00027 A more readable summary is at:  
00028   http://www.iecc.com/linker/linker08.html
00029 
00030  *
00031  *  Developed by Sameer Kumar (sameer@ks.uiuc.edu) 8/25/03
00032  *  Made functional by Orion Lawlor (olawlor@acm.org) 2003/9/16
00033  *  Made working on AMD64 by Gengbin Zheng 12/20/2005
00034  *
00035  *  FIXME2: Sanity check. I am assuming that if a symbol is in the
00036  *  relocation table it is in the global offset table. A sanity check
00037  *  would be to get the address from the symbol table and look for it
00038  *  in the GOT. Pointers to remote function calls may be an exception
00039  *  to this.
00040  */
00041 
00042 #include "converse.h"
00043 #include "cklists.h"
00044 #include <string.h>
00045 #include <stdio.h>
00046 #include <stdlib.h>
00047 #include <errno.h>
00048 #include <sys/types.h>
00049 #if CMK_HAS_REGEX_H
00050 #include <regex.h>
00051 #endif
00052 #include <vector>
00053 #include <algorithm>
00054 #include "converse.h"
00055 #include "pup.h"
00056 
00057 #if CMK_HAS_ELF_H
00058 #include <elf.h>
00059 
00060 #define DEBUG_GOT_MANAGER 0
00061 
00062 #define UNPROTECT_GOT     1
00063 
00064 #if UNPROTECT_GOT && CMK_HAS_MPROTECT
00065 #include <sys/mman.h>
00066 #if CMK_HAS_GETPAGESIZE
00067 #include <unistd.h>
00068 #endif
00069 #endif
00070 
00071 #ifdef __INTEL_COMPILER
00072 #define ALIGN_GOT(x)       (long)((~15)&((x)+15))
00073 #else
00074 #define ALIGN_GOT(x)       ALIGN8(x)
00075 #endif
00076 
00077 #if !CMK_SHARED_VARS_UNAVAILABLE
00078 #  error "Global-elfgot won't work properly under smp version: -swapglobals disabled"
00079 #endif
00080 
00081 CpvDeclare(int, CmiPICMethod);
00082 
00083 #if CMK_AMD64
00084 typedef Elf64_Addr    ELFXX_TYPE_Addr;
00085 typedef Elf64_Dyn     ELFXX_TYPE_Dyn;
00086 typedef Elf64_Rela    ELFXX_TYPE_Rel;
00087 typedef Elf64_Sym     ELFXX_TYPE_Sym;
00088 #define ELFXX_R_TYPE   ELF64_R_TYPE
00089 #define ELFXX_R_SYM    ELF64_R_SYM
00090 #define ELFXX_ST_TYPE  ELF64_ST_TYPE
00091 #define CMK_DT_REL     DT_RELA
00092 #define CMK_DT_RELSZ   DT_RELASZ
00093 #define is_elf_global(x)  ((x) == R_X86_64_GLOB_DAT)
00094 #elif CMK_IA64
00095 #error "NOT SUPPORTED"
00096 #else
00097 typedef Elf32_Addr    ELFXX_TYPE_Addr;
00098 typedef Elf32_Dyn     ELFXX_TYPE_Dyn;
00099 typedef Elf32_Rel     ELFXX_TYPE_Rel;
00100 typedef Elf32_Sym     ELFXX_TYPE_Sym;
00101 #define ELFXX_R_TYPE   ELF32_R_TYPE
00102 #define ELFXX_R_SYM    ELF32_R_SYM
00103 #define ELFXX_ST_TYPE  ELF32_ST_TYPE
00104 #define CMK_DT_REL     DT_REL
00105 #define CMK_DT_RELSZ   DT_RELSZ
00106 #define is_elf_global(x)  ((x) == R_386_GLOB_DAT)
00107 #endif
00108 
00109 extern ELFXX_TYPE_Dyn _DYNAMIC[];      //The Dynamic section table pointer
00110 
00111 
00117 CkVec<char *>  _blacklist;
00118 static int loaded = 0;
00119 
00120 static void readBlacklist()
00121 {
00122   if (loaded) return;
00123   const char *fname = "blacklist";
00124   FILE *bl = fopen(fname, "r");
00125   if (bl == NULL){
00126                 if (CmiMyPe() == 0) printf("WARNING: Running swapglobals without blacklist, globals from libraries might be getting un-necessarily swapped\n");
00127                 loaded = 1;
00128                 return;
00129   }
00130   printf("Loading blacklist from file \"%s\" ... \n", fname);
00131   while (!feof(bl)){
00132     char name[512];
00133     fscanf(bl, "%s\n", name);
00134      _blacklist.push_back(strdup(name));
00135   }
00136   fclose(bl);
00137   loaded = 1;
00138 }
00139 
00140 
00141 
00142 
00143 /****************** Global Variable Understanding *********************/
00147 class CtgGlobalList
00148 {
00149   int datalen; 
00150   struct CtgRec {
00151     ELFXX_TYPE_Addr *got; 
00152     int off; 
00153     CtgRec() {got=NULL;}
00154     CtgRec(ELFXX_TYPE_Addr *got_,int off_) :got(got_), off(off_) {}
00155   };
00156   CkVec<CtgRec> rec;
00157   int nRec;
00158 public:
00164   CtgGlobalList();
00165   
00167   inline int getSize(void) const {return datalen;}
00168   
00171   void read(void *datav) const;
00172   
00174   inline void install(void *datav) const {
00175     char *data=(char *)datav;
00176     for (int i=0;i<nRec;i++)
00177       *(rec[i].got)=(ELFXX_TYPE_Addr)(data+rec[i].off);
00178   }
00179   
00180 private:
00181   /* Return 1 if this is the name of a user global variable;
00182      0 if this is the name of some other (Charm or system)
00183      global variable. */
00184   int isUserSymbol(const char *name);
00185 };
00186 
00187 int match(const char *string, const char *pattern) {
00188 #if CMK_HAS_REGEX_H
00189   int status;
00190 
00191   regex_t re;
00192   if (regcomp(&re, pattern, REG_EXTENDED|REG_NOSUB) != 0) {
00193     CmiAbort("error compiling regex");
00194   }
00195   status = regexec(&re, string, (size_t) 0, NULL, 0);
00196   regfree(&re);
00197   if (status == 0) {
00198     return 1;
00199   } else if (status == REG_NOMATCH) {
00200     return 0;
00201   }
00202   perror("error in match\n");
00203   return 0;
00204 #else
00205   CmiPrinf("Warning: elfgot.C::match() is not implemented!\n");
00206   return 0;
00207 #endif
00208 }
00209 
00210 int CtgGlobalList::isUserSymbol(const char *name) {
00211     // return 1;
00212     if((strncmp("_", name, 1) == 0) || (strncmp("Cpv_", name, 4) == 0)
00213        || (strncmp("Csv_", name, 4) == 0) || (strncmp("Ctv_", name, 4) == 0)
00214        || (strncmp("Bnv_", name, 4) == 0) || (strncmp("Bpv_", name, 4) == 0)
00215        || (strncmp("ckout", name, 5) == 0) || (strncmp("stdout", name, 6) == 0)
00216        || (strncmp("ckerr", name, 5) == 0)
00217        || (strncmp("environ", name, 7) == 0)
00218        || (strncmp("stderr", name, 6) == 0) || (strncmp("stdin", name, 5) == 0)) {
00219 #ifdef CMK_GFORTRAN
00220         if (match(name, "__.*_MOD_.*")) return 1;
00221 #endif
00222         return 0;
00223     }
00224     
00228         for(int i=0;i<_blacklist.size();i++){
00229                 if(strlen(name) == strlen(_blacklist[i]) && strncmp(name,_blacklist[i],strlen(name)) == 0){
00230                         return 0;
00231                 }
00232         }
00233                 
00234     return 1;
00235 }
00236 
00237 void CtgGlobalList::read(void *datav) const {
00238     char *data=(char *)datav;
00239     for (int i=0;i<nRec;i++) {
00240       int size;
00241       if (i<nRec-1) 
00242         size=rec[i+1].off-rec[i].off;
00243       else /* i==nRec-1, last one: */ 
00244         size=datalen-rec[i].off;
00245       memcpy(data+rec[i].off, (void *)*rec[i].got, size);
00246     }
00247 }
00248 
00254 CtgGlobalList::CtgGlobalList() {
00255     datalen=0;
00256     nRec=0;
00257     
00258     int count;
00259     int relt_size = 0;
00260     int type, symindx;
00261     char *sym_name;
00262     ELFXX_TYPE_Rel *relt=NULL;       //Relocation table
00263     ELFXX_TYPE_Sym *symt=NULL;       //symbol table
00264     char *str_tab=NULL;         //String table
00265 
00266     // Find tables and sizes of tables from the dynamic segment table
00267     for(count = 0; _DYNAMIC[count].d_tag != 0; ++count) {
00268         switch(_DYNAMIC[count].d_tag) {
00269             case CMK_DT_REL:
00270                 relt = (ELFXX_TYPE_Rel *) _DYNAMIC[count].d_un.d_ptr;
00271                 break;
00272             case CMK_DT_RELSZ:
00273                 relt_size = _DYNAMIC[count].d_un.d_val/ sizeof(ELFXX_TYPE_Rel);
00274                 break;
00275             case DT_SYMTAB:
00276                 symt = (ELFXX_TYPE_Sym *) _DYNAMIC[count].d_un.d_ptr;
00277                 break;
00278             case DT_STRTAB:
00279                 str_tab = (char *)_DYNAMIC[count].d_un.d_ptr;
00280                 break;
00281         }
00282     }
00283 
00284     int padding = 0;
00285 
00286 #if UNPROTECT_GOT && CMK_HAS_MPROTECT
00287     size_t pagesize = CMK_MEMORY_PAGESIZE;
00288 #if CMK_HAS_GETPAGESIZE
00289     pagesize = getpagesize();
00290 #endif
00291 #endif
00292 
00293     // Figure out which relocation data entries refer to global data:
00294     for(count = 0; count < relt_size; count ++) {
00295         type = ELFXX_R_TYPE(relt[count].r_info);
00296         symindx = ELFXX_R_SYM(relt[count].r_info);
00297  
00298         if(!is_elf_global(type))
00299             continue; /* It's not global data */
00300 
00301         sym_name = str_tab + symt[symindx].st_name;
00302 
00303 #if DEBUG_GOT_MANAGER
00304         printf("relt[%d]= %s: %d bytes, %p sym, R_==%d\n", count, sym_name, 
00305                symt[symindx].st_size, (void *)symt[symindx].st_value, type);
00306 #endif
00307 
00308         if(ELFXX_ST_TYPE(symt[symindx].st_info) != STT_OBJECT &&
00309            ELFXX_ST_TYPE(symt[symindx].st_info) != STT_NOTYPE
00310 #if 0
00311 #ifdef __INTEL_COMPILER
00312           && ELFXX_ST_TYPE(symt[symindx].st_info) != STT_FUNC
00313 #endif
00314 #endif
00315                  ) /* ? */
00316             continue;
00317 
00318         if(strcmp(sym_name, "_DYNAMIC") == 0 ||
00319            strcmp(sym_name, "__gmon_start__") == 0 ||
00320            strcmp(sym_name, "_GLOBAL_OFFSET_TABLE_") == 0)
00321             continue; /* It's system data */
00322 
00323         if (!isUserSymbol(sym_name))
00324             continue;
00325 
00326         // It's got the right name-- it's a user global
00327         int size = symt[symindx].st_size;
00328         int gSize = ALIGN_GOT(size);
00329         padding += gSize - size;
00330         ELFXX_TYPE_Addr *gGot=(ELFXX_TYPE_Addr *)relt[count].r_offset;
00331 
00332 #if DEBUG_GOT_MANAGER
00333         printf("   -> %s is a user global, of size %d, at %p\n",
00334                sym_name, size, (void *)*gGot);
00335 #endif
00336         if ((void *)*gGot != (void *)symt[symindx].st_value)
00337             CmiAbort("CtgGlobalList: symbol table and GOT address mismatch!\n");
00338 
00339 #if UNPROTECT_GOT && CMK_HAS_MPROTECT
00340         static void *last = NULL;
00341         void *pg = (void*)(((size_t)gGot) & ~(pagesize-1));
00342         if (pg != last) {
00343             mprotect(pg, pagesize, PROT_READ | PROT_WRITE);
00344             last = pg;
00345         }
00346 #endif
00347 
00348         rec.push_back(CtgRec(gGot,datalen));
00349         datalen+=gSize;
00350     }
00351 
00352     nRec=rec.size();
00353 
00354 #if DEBUG_GOT_MANAGER   
00355     printf("relt has %d entries, %d of which are user globals\n\n", 
00356            relt_size, nRec);
00357     printf("Globals take %d bytes (padding bytes: %d)\n", datalen, padding);
00358 #endif
00359 }
00360 
00361 /****************** Global Variable Storage and Swapping *********************/
00362 CpvStaticDeclare(CtgGlobals,_curCtg);
00363 
00364 struct CtgGlobalStruct {
00365 public:
00366     /* This is set when our data is pointed to by the current GOT */
00367     int installed;
00368 
00369     /* Pointer to our global data segment. */
00370     void *data_seg;  
00371     int seg_size; /* size in bytes of data segment */
00372     
00373     void allocate(int size, CthThread tid) {
00374       seg_size=size;
00375         /* global data segment need to be isomalloc */
00376       if (CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
00377         data_seg=CmiIsomalloc(seg_size,tid);
00378       else
00379         data_seg=malloc(seg_size);
00380     }
00381     
00382     CtgGlobalStruct(void) {
00383       installed=0;
00384       data_seg=0;
00385     }
00386     ~CtgGlobalStruct() {
00387       if (data_seg) {
00388         if (CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
00389         {
00390 #if !CMK_USE_MEMPOOL_ISOMALLOC
00391           CmiIsomallocFree(data_seg);
00392 #endif
00393         }
00394         else
00395           free(data_seg);
00396         data_seg = NULL;
00397       }
00398     }
00399     
00400     void pup(PUP::er &p);
00401 };
00402 
00403 void CtgGlobalStruct::pup(PUP::er &p) {
00404     p | seg_size;
00405         /* global data segment need to be isomalloc pupped */
00406     if (CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
00407 #if CMK_USE_MEMPOOL_ISOMALLOC
00408       pup_bytes(&p, &data_seg, sizeof(void*));
00409 #else
00410       CmiIsomallocPup(&p, &data_seg);
00411 #endif
00412       else {
00413       if (p.isUnpacking()) allocate(seg_size, NULL);
00414       p((char *)data_seg, seg_size);
00415     }
00416 }
00417 
00419 static CtgGlobalList *_ctgList=NULL;
00421 static CtgGlobalStruct *_ctgListGlobals=NULL;
00422 
00424 void CtgInit(void) {
00425         CpvInitialize(int, CmiPICMethod);
00426         CpvAccess(CmiPICMethod) = 1;
00427         CpvInitialize(CtgGlobal,_curCtg);
00428         
00429         if (!_ctgList) 
00430         {
00431         /*
00432           First call on this node: parse out our globals:
00433         */
00434                 readBlacklist();
00435                 CtgGlobalList *l=new CtgGlobalList;
00436                 CtgGlobalStruct *g=new CtgGlobalStruct;
00437                 if (CmiMyNode()==0) {
00438                         CmiPrintf("Charm++> -swapglobals enabled\n");
00439                 }
00440                 
00441                 g->allocate(l->getSize(), NULL);
00442                 l->read(g->data_seg);
00443                 l->install(g->data_seg);
00444                 _ctgList=l;
00445                 _ctgListGlobals=g;
00446         }
00447         
00448         CpvAccess(_curCtg)=_ctgListGlobals;
00449 }
00450 
00452 CtgGlobals CtgCreate(CthThread tid) {
00453         CtgGlobalStruct *g=new CtgGlobalStruct;
00454         g->allocate(_ctgList->getSize(), tid);
00455         _ctgList->read(g->data_seg);
00456         return g;
00457 }
00459 CtgGlobals CtgPup(pup_er pv, CtgGlobals g) {
00460         PUP::er *p=(PUP::er *)pv;
00461         if (p->isUnpacking()) g=new CtgGlobalStruct;
00462         if (g->installed) 
00463                 CmiAbort("CtgPup called on currently installed globals!\n");
00464         g->pup(*p);
00465         if (g->seg_size!=_ctgList->getSize())
00466                 CmiAbort("CtgPup: global variable size changed during migration!\n");
00467         return g;
00468 }
00469 
00471 void CtgInstall(CtgGlobals g) {
00472         CtgGlobals *cur=&CpvAccess(_curCtg);
00473         CtgGlobals oldG=*cur;
00474         if (g==NULL) g=_ctgListGlobals;
00475         if (g == oldG) return;
00476         *cur=g;
00477         oldG->installed=0;
00478         _ctgList->install(g->data_seg);
00479         g->installed=1;
00480 }
00481 
00483 void CtgFree(CtgGlobals g) {
00484         if (g->installed) CmiAbort("CtgFree called on currently installed globals!\n");
00485         delete g;
00486 }
00487 
00488 CtgGlobals CtgCurrentGlobals(void){
00489         return CpvAccess(_curCtg);
00490 }
00491 
00492 void CtgInstall_var(CtgGlobals g, void *ptr) {}
00493 void CtgUninstall_var(CtgGlobals g, void *ptr) {}
00494 
00495 #else
00496 
00497 #include "global-nop.c"
00498 
00499 #endif
conv-core/global-elfgot.C
