Home Research Papers Posters Manuals Talks Download People Help Wiki Internal

arch/net/charmrun/charmrun.c

Go to the documentation of this file.

/*****************************************************************************
 * $Source: /cvsroot/charm/src/arch/net/charmrun/charmrun.c,v $
 * $Author: gzheng $
 * $Date: 2008-06-26 05:17:59 $
 * $Revision: 1.138 $
 *****************************************************************************/

#include "converse.h"

#include "../sockRoutines.h"
#include "../sockRoutines.c"
#include "../ccs-auth.h"
#include "../ccs-auth.c"
#include "../ccs-server.h"
#include "../ccs-server.c"

#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <setjmp.h>
#include <stdlib.h>
#include <signal.h>
#include <fcntl.h>
#include <time.h>
#include <assert.h>
#if CMK_BPROC
#include <sys/bproc.h>
#endif

#if defined(_WIN32) && !defined(__CYGWIN__)
/*Win32 has screwy names for the standard UNIX calls:*/
#define getcwd _getcwd
#define strdup _strdup
#define unlink _unlink
#define open _open
#define fdopen _fdopen
#define ftruncate _chsize
#include <winbase.h>
#include <direct.h>
#include <io.h>
#include <sys/timeb.h>
#include <process.h>
#define DIRSEP "\\"
#define SIGBUS -1  /*These signals don't exist in Win32*/
#define SIGKILL -1
#define SIGQUIT -1


#else /*UNIX*/
#include <pwd.h>   /*getcwd*/
#include <unistd.h>
#define DIRSEP "/"
#endif

#if CMK_RSH_NOT_NEEDED /*No RSH-- use daemon to start node-programs*/
#  define CMK_USE_RSH 0

#else /*Use RSH to start node-programs*/
#  define CMK_USE_RSH 1
#ifdef __MINGW_H
#  include <rpc.h>
#elif !defined(__CYGWIN__)
#  include <rpc/rpc.h>
#else
#  include <w32api/rpc.h>
#endif
#  if CMK_RSH_IS_A_COMMAND
#    define RSH_CMD "rsh"
#  endif

#  if CMK_RSH_USE_REMSH
#    define RSH_CMD "remsh"
#  endif
#endif

#include "daemon.h"

/*#define DEBUGF(x) printf x*/
#define DEBUGF(x) 

#ifndef MAXPATHLEN
#define MAXPATHLEN 1024
#endif

static double ftTimer;
                                                                                
double GetClock(void)
{
#if defined(_WIN32) && !defined(__CYGWIN__)
  struct _timeb tv;
  _ftime(&tv);
  return (tv.time * 1.0 + tv.millitm * 1.0E-3);
#else
  struct timeval tv; int ok;
  ok = gettimeofday(&tv, NULL);
  if (ok<0) { perror("gettimeofday"); exit(1); }
  return (tv.tv_sec * 1.0 + tv.tv_usec * 1.0E-6);
#endif
}


int probefile(path)
    char *path;
{
        FILE *f=fopen(path,"r");
        if (f==NULL) return 0;
        fclose(f);
        return 1;
}

char *mylogin(void)
{
#if defined(_WIN32) && !defined(__CYGWIN__)
        static char name[100]={'d','u','n','n','o',0};
        int len=100;
        GetUserName(name,&len);
        return name;
#else /*UNIX*/
  struct passwd *self;

  self = getpwuid(getuid());
  if (self==0) { return "unknown"; }
  return self->pw_name;
#endif
} 

/**************************************************************************
 *
 * ping_developers
 *
 * Sends a single UDP packet to the charm developers notifying them
 * that charm is in use.
 *
 **************************************************************************/

void ping_developers()
{
#ifdef NOTIFY
  char               info[1000];
  /*This is the resolved IP address of elegance.cs.uiuc.edu */
  skt_ip_t destination_ip=skt_lookup_ip("128.174.241.211");
  unsigned int destination_port=6571;
  struct sockaddr_in addr=skt_build_addr(destination_ip,destination_port);
  SOCKET             skt;
  
  skt = socket(AF_INET, SOCK_DGRAM, 0);
  if (skt == INVALID_SOCKET) return;

  sprintf(info,"%s",mylogin());
  
  sendto(skt, info, strlen(info), 0, (struct sockaddr *)&addr, sizeof(addr));
  skt_close(skt);
#endif /* NOTIFY */
}

/**************************************************************************
 *
 * Pathfix : alters a path according to a set of rewrite rules
 *
 *************************************************************************/

typedef struct pathfixlist {
  char *s1;
  char *s2;
  struct pathfixlist *next;
} *pathfixlist;

pathfixlist pathfix_append(char *s1, char *s2, pathfixlist l)
{
  pathfixlist pf = (pathfixlist)malloc(sizeof(struct pathfixlist));
  pf->s1 = s1;
  pf->s2 = s2;
  pf->next = l;
  return pf;
}

char *pathfix(char *path, pathfixlist fixes)
{
  char buffer[MAXPATHLEN]; pathfixlist l; 
  char buf2[MAXPATHLEN]; 
  char *offs; int mod, len;
  strcpy(buffer,path);
  mod = 1;
  while (mod) {
    mod = 0;
    for (l=fixes; l; l=l->next) {
      len = strlen(l->s1);
      offs = strstr(buffer, l->s1);
      if (offs) {
        offs[0]=0;
        sprintf(buf2,"%s%s%s",buffer,l->s2,offs+len);
        strcpy(buffer,buf2);
        mod = 1;
      }
    }
  }
  return strdup(buffer);
}

char *pathextfix(char *path, pathfixlist fixes, char *ext)
{
  char *newpath = pathfix(path, fixes);
  char *ret;
  if (ext == NULL) return newpath;
  ret = (char *)malloc(strlen(newpath)+strlen(ext)+2);
  strcpy(ret, newpath);
  strcat(ret, ext);
  return ret;
}

/****************************************************************************
 *
 * Miscellaneous minor routines.
 *
 ****************************************************************************/

int is_quote(char c)
{
  return (c=='\'' || c == '"');
}

void zap_newline(char *s)
{
  char *p;
  p = s + strlen(s)-1;
  if (*p == '\n') *p = '\0';
  /* in case of DOS ^m */
  p--;
  if (*p == '\15') *p = '\0';
}

/* get substring from lo to hi, remove quote chars */
char *substr(char *lo, char *hi)
{
  int len;
  char *res;
  if (is_quote(*lo)) lo++;
  if (is_quote(*(hi-1))) hi--;
  len = hi-lo;
  res = (char *)malloc(1+len);
  memcpy(res, lo, len);
  res[len]=0;
  return res;
}

int subeqs(char *lo, char *hi, char *str)
{
  int len = strlen(str);
  if (hi-lo != len) return 0;
  if (memcmp(lo, str, len)) return 0;
  return 1;
}

/* advance pointer over blank characters */
char *skipblanks(char *p)
{
  while ((*p==' ')||(*p=='\t')) p++;
  return p;
}

/* advance pointer over nonblank characters and a quoted string */
char *skipstuff(char *p)
{
  char quote = 0;
  if (*p && (*p=='\'' || *p=='"')) { quote=*p; p++; }
  if (quote != 0) {
    while (*p&&*p!=quote) p++;
    if (*p!=quote) {
      fprintf(stderr, "ERROR> Unmatched quote in nodelist file.\n");
      exit(1);
    }
    p++;
  }
  else
    while ((*p)&&(*p!=' ')&&(*p!='\t')) p++;
  return p;
}

#if CMK_USE_RSH
char *getenv_rsh()
{
  char *e;

  e = getenv("CONV_RSH");
  return e ? e : RSH_CMD;
}
#endif

#if !defined(_WIN32) || defined(__CYGWIN__)
char *getenv_display()
{
  static char result[100],ipBuf[200];
  char *e, *p;
  
  e = getenv("DISPLAY");
  if (e==0) return NULL;
  p = strrchr(e, ':');
  if (p==0) return NULL;
  if ((e[0]==':')||(strncmp(e,"unix:",5)==0)) {
    sprintf(result,"%s:%s",skt_print_ip(ipBuf,skt_my_ip()),p+1);
  }
  else strcpy(result, e);
  return result;
}
char *getenv_display_no_tamper()
{
  static char result[100],ipBuf[200];
  char *e, *p;
  
  e = getenv("DISPLAY");
  if (e==0) return NULL;
  p = strrchr(e, ':');
  if (p==0) return NULL;
  strcpy(result, e);
  return result;
}

#endif

/*****************************************************************************
 *                                                                           *
 * PPARAM - obtaining "program parameters" from the user.                    *
 *                                                                           *
 *****************************************************************************/

typedef struct ppdef
{
  union {
      int *i;
      double *r;
      char **s;
      int *f;
    } where;/*Where to store result*/
  const char *lname; /*Argument name on command line*/
  const char *doc;
  char  type; /*One of i, r, s, f.*/
  struct ppdef *next;
}
*ppdef;

static ppdef ppdefs;

static int     pparam_pos;
static char  **pparam_argv;
static char    pparam_optc='-';
char           pparam_error[100];

static ppdef pparam_find(lname)
    const char *lname;
{
  ppdef def;
  for (def=ppdefs; def; def=def->next)
    if (strcmp(def->lname, lname)==0)
      return def;
  return 0;
}

static ppdef pparam_cell(lname)
    const char *lname;
{
  ppdef def = pparam_find(lname);
  if (def) return def;
  def = (ppdef)malloc(sizeof(struct ppdef));
  def->lname = lname;
  def->type  = 's';
  def->doc   = "(undocumented)";
  def->next  = ppdefs;
  ppdefs = def;
  return def;
}



void pparam_int(int *where,int defValue,
                                   const char *arg,const char *doc)
{
  ppdef def = pparam_cell(arg);
  def->type  = 'i';
  def->where.i = where; *where=defValue;
  def->lname=arg;
  def->doc=doc;
}

void pparam_flag(int *where,int defValue,
                                   const char *arg,const char *doc)
{
  ppdef def = pparam_cell(arg);
  def->type  = 'f';
  def->where.f = where; *where=defValue;
  def->lname=arg;
  def->doc=doc;
}

void pparam_real(double *where,double defValue,
                                   const char *arg,const char *doc)
{
  ppdef def = pparam_cell(arg);
  def->type  = 'r';
  def->where.r = where; *where=defValue;
  def->lname=arg;
  def->doc=doc;
}
void pparam_str(char **where,char *defValue,
                                   const char *arg,const char *doc)
{
  ppdef def = pparam_cell(arg);
  def->type  = 's';
  def->where.s = where; *where=defValue;
  def->lname=arg;
  def->doc=doc;
}

static int pparam_setdef(def, value)
    ppdef def; char *value;
{
  char *p;
  switch(def->type)
    {
    case 'i' :
      *def->where.i = strtol(value, &p, 10);
      if (*p) return -1;
      return 0;
    case 'r' :
      *def->where.r = strtod(value, &p);
      if (*p) return -1;
      return 0;
    case 's' :
      *def->where.s = strdup(value);
      return 0;
    case 'f' :
      *def->where.f = strtol(value, &p, 10);
      if (*p) return -1;
      return 0;
    }
  return -1;
}

int pparam_set(lname, value)
    char *lname; char *value;
{
  ppdef def = pparam_cell(lname);
  return pparam_setdef(def, value);
}

char *pparam_getdef(def)
    ppdef def;
{
  static char result[100];
  switch(def->type)
    {
    case 'i': sprintf(result,"%d", *def->where.i); return result;
    case 'r': sprintf(result,"%f",*def->where.r); return result;
    case 's': return *def->where.s?*def->where.s:"";
    case 'f': sprintf(result,"%d", *def->where.f); return result;
    }
  return NULL;
}

void pparam_printdocs()
{
  ppdef def; int len, maxname, maxdoc;
  maxname = 0;
  maxdoc = 0;
  for (def=ppdefs; def; def=def->next)
    {
      len = strlen(def->lname);
      if (len>maxname) maxname=len;
      len = strlen(def->doc);
      if (len>maxdoc) maxdoc=len;
    }
  fprintf(stderr,"\n");
  fprintf(stderr,"Charmrun Command-line Parameters:\n");
  for (def=ppdefs; def; def=def->next)
    {
      fprintf(stderr,"  %c%c%-*s ",pparam_optc,pparam_optc,maxname,def->lname);
      fprintf(stderr,"  %-*s [%s]\n",maxdoc,def->doc,pparam_getdef(def));
    }
  fprintf(stderr,"\n");
}

void pparam_delarg(i)
    int i;
{
  int j;
  for (j=i; pparam_argv[j]; j++)
    pparam_argv[j]=pparam_argv[j+1];
}

int pparam_countargs(argv)
    char **argv;
{
  int argc;
  for (argc=0; argv[argc]; argc++);
  return argc;
}

int pparam_parseopt()
{
  int ok; ppdef def=NULL;
  char *opt = pparam_argv[pparam_pos];
  /* handle ++ by skipping to end */
  if ((opt[1]=='+')&&(opt[2]==0))
    {
      pparam_delarg(pparam_pos);
      while (pparam_argv[pparam_pos]) pparam_pos++;
      return 0;
    }
  /* handle + by itself - an error */
  if (opt[1]==0) 
    {
      sprintf(pparam_error,"Illegal option +\n");
      return -1;
    }
  /* look up option definition */
  if (opt[1]=='+') def = pparam_find(opt+2);
  else
    {
      char name[2];
      name[0]=opt[1];
      name[1]=0;
      def = pparam_find(name);
    }
  if (def==NULL)
  {
    if (opt[1]=='+')
    {
       sprintf(pparam_error,"Option %s not recognized.",opt);
       return -1;
    } else {
           /*Unrecognized + option-- skip it.*/
           pparam_pos++;
           return 0;
        }
  }
  /* handle flag-options */
  if ((def->type=='f')&&(opt[1]!='+')&&(opt[2]))
    {
      sprintf(pparam_error,"Option %s should not include a value",opt);
      return -1;
    }
  if (def->type=='f')
    {
      *def->where.f = 1;
      pparam_delarg(pparam_pos);
      return 0;
    }
  /* handle non-flag options */
  if ((opt[1]=='+')||(opt[2]==0))
    {
      pparam_delarg(pparam_pos);
      opt = pparam_argv[pparam_pos];
    }
  else opt+=2;
  if ((opt == 0)||(opt[0] == 0))
    {
      sprintf(pparam_error,"%s must be followed by a value.",opt);
      return -1;
    }
  ok = pparam_setdef(def, opt);
  pparam_delarg(pparam_pos);
  if (ok<0)
    {
      sprintf(pparam_error,"Illegal value for %s",opt);
      return -1;
    }
  return 0;
}

int pparam_parsecmd(optchr, argv)
    char optchr; char **argv;
{
  pparam_error[0]=0;
  pparam_argv = argv;
  pparam_optc = optchr;
  pparam_pos  = 0;
  while(1)
    {
      char *opt = pparam_argv[pparam_pos];
      if (opt==0) break;
      if (opt[0]!=optchr) pparam_pos++;
      else if (pparam_parseopt()<0) return -1;
    }
  return 0;
}

/****************************************************************************
 * 
 * ARG
 *
 * The following module computes a whole bunch of miscellaneous values, which
 * are all constant throughout the program.  Naturally, this includes the
 * value of the command-line arguments.
 *
 *****************************************************************************/


#define MAX_NODES 1000
#define MAX_LINE_LENGTH 1000

char **arg_argv;
int    arg_argc;

int   arg_requested_pes;
int   arg_timeout;
int   arg_verbose;
char *arg_nodelist;
char *arg_nodegroup;
char *arg_runscript; /* script to run the node-program with */
char *arg_charmrunip;
#if CONVERSE_VERSION_VMI
char *arg_vmispecfile;
#endif

int   arg_debug;
int   arg_debug_no_pause;
int   arg_charmdebug;

int   arg_local;        /* start node programs directly by exec on localhost */
int   arg_batch_spawn;  /* control starting node programs, several at a time */

int   arg_help;         /* print help message */
int   arg_ppn;          /* pes per node */
int   arg_usehostname;

#if CMK_USE_RSH
int   arg_maxrsh;
char *arg_shell;
int   arg_in_xterm;
char *arg_debugger;
char *arg_xterm;
char *arg_display;
int arg_ssh_display;
char *arg_mylogin;
#endif

char *arg_nodeprog_a;
char *arg_nodeprog_r;
char *arg_currdir_a;
char *arg_currdir_r;

int   arg_server;
int   arg_server_port=0;
char *arg_server_auth=NULL;

#if CMK_BPROC
int   arg_startpe;
int   arg_endpe;
int   arg_singlemaster;
int   arg_skipmaster;
#endif

void arg_init(int argc, char **argv)
{
  static char buf[1024];
  
  int i, local_def=0;
#if CMK_CHARMRUN_LOCAL
  local_def=1; /*++local is the default*/
#endif
  
  pparam_int(&arg_requested_pes, 1, "p",             "number of processes to create");
  pparam_int(&arg_timeout,      60, "timeout",       "seconds to wait per host connection");
  pparam_flag(&arg_verbose,      0, "verbose",       "Print diagnostic messages");
  pparam_str(&arg_nodelist,      0, "nodelist",      "file containing list of nodes");
  pparam_str(&arg_nodegroup,"main", "nodegroup",     "which group of nodes to use");

#if CMK_CCS_AVAILABLE
  pparam_flag(&arg_server,       0, "server",        "Enable client-server (CCS) mode");
  pparam_int(&arg_server_port,   0, "server-port",   "Port to listen for CCS requests");
  pparam_str(&arg_server_auth,   0, "server-auth",   "CCS Authentication file");
#endif
  pparam_flag(&arg_local,       local_def, "local", "Start node programs locally without daemon");
  pparam_int(&arg_batch_spawn,   0, "batch", "Rsh several node programs at a time, avoiding overloading charmrun pe");
  pparam_flag(&arg_usehostname,  0, "usehostname", "Send nodes our symbolic hostname instead of IP address");
  pparam_str(&arg_charmrunip,    0, "useip",      "Use IP address provided for charmrun IP");
#if CMK_USE_RSH
  pparam_flag(&arg_debug,         0, "debug",         "Run each node under gdb in an xterm window");
  pparam_flag(&arg_debug_no_pause,0, "debug-no-pause","Like debug, except doesn't pause at beginning");

  /* When the ++charmdebug flag is used, charmrun listens from its stdin for
     commands, and forwards them to the gdb info program (a child), or to the
     processor gdbs. The stderr is redirected to the stdout, so the two streams
     are mixed together. The channel for stderr is reused to forward the replies
     of gdb back to the java debugger. */
#if !defined(_WIN32)
  pparam_flag(&arg_charmdebug,    0, "charmdebug",    "Used only when charmrun is started by charmdebug");
#endif

  pparam_int(&arg_maxrsh,        16, "maxrsh",        "Maximum number of rsh's to run at a time");
  pparam_str(&arg_shell,          0, "remote-shell",  "which remote shell to use");
  pparam_str(&arg_debugger,       0, "debugger",      "which debugger to use");
  pparam_str(&arg_display,        0, "display",       "X Display for xterm");
  pparam_flag(&arg_ssh_display,   0, "ssh-display",   "use own X Display for each ssh session");
  pparam_flag(&arg_in_xterm,      0, "in-xterm",      "Run each node in an xterm window");
  pparam_str(&arg_xterm,          0, "xterm",         "which xterm to use");
#endif
#ifdef CMK_BPROC
  /* options for Scyld */
  pparam_int(&arg_startpe,   0, "startpe",   "first pe to start job(SCYLD)");
  pparam_int(&arg_endpe,  1000000, "endpe",   "last pe to start job(SCYLD)");
  pparam_flag(&arg_singlemaster, 0, "singlemaster", "Only assign one process to master node(SCYLD)");
  pparam_flag(&arg_skipmaster, 0, "skipmaster", "Donot assign any process to master node(SCYLD)");
  if (arg_skipmaster && arg_singlemaster) {
    printf("Charmrun> 'singlemaster' is ignored due to 'skipmaster'. \n");
    arg_singlemaster = 0;
  }
  pparam_flag(&arg_debug,         0, "debug",         "turn on more verbose debug print");
#endif
#ifdef CONVERSE_VERSION_VMI
  pparam_str (&arg_vmispecfile, 0, "specfile", "device specfile to load (VMI)");
#endif
  pparam_str(&arg_runscript,    0, "runscript", "script to run node-program with");
  pparam_flag(&arg_help,        0, "help", "print help messages");
  pparam_int(&arg_ppn,          0, "ppn",             "number of pes per node");

  if (pparam_parsecmd('+', argv) < 0) {
    fprintf(stderr,"ERROR> syntax: %s\n",pparam_error);
    pparam_printdocs();
    exit(1);
  }
  
  /* Check for (but do *not* remove) the "-?", "-h", or "--help" flags */
  for (i=0;argv[i];i++) {
        if (0==strcmp(argv[i],"-?") ||
            0==strcmp(argv[i],"-h") ||
            0==strcmp(argv[i],"--help")) 
                arg_help=1;
  }
  if (arg_help) {
    pparam_printdocs();
    /*exit(0);*/
  }

  arg_argv = argv+1; /*Skip over charmrun (0) here and program name (1) later*/
  arg_argc = pparam_countargs(arg_argv);
  if (arg_argc<1) {
    fprintf(stderr,"ERROR> You must specify a node-program.\n");
    pparam_printdocs();
    exit(1);
  }
  arg_argv++; arg_argc--;

  if (arg_server_port || arg_server_auth) arg_server=1;

  if (arg_debug || arg_debug_no_pause) {
        arg_verbose=1;
        /*Pass ++debug along to program (used by machine.c)*/
        arg_argv[arg_argc++]="++debug";
  }

#ifdef CMK_BPROC
  if (arg_local) {
    fprintf(stderr,"Warning> ++local cannot be used in bproc version, ignored!\n");
    arg_local = 0;
  }
#endif

#if CMK_USE_RSH
  /* Find the current value of the CONV_RSH variable */
  if(!arg_shell) arg_shell = getenv_rsh();

  /* Find the current value of the DISPLAY variable */
  if(!arg_display)
    arg_display = getenv_display_no_tamper();
  if ((arg_debug || arg_debug_no_pause || arg_in_xterm) && (arg_display==0)) {
    fprintf(stderr,"ERROR> DISPLAY must be set to use debugging mode\n");
    exit(1);
  }
  if (arg_debug || arg_debug_no_pause) 
    arg_timeout=8*60*60; /* Wait 8 hours for ++debug */

  /* default debugger is gdb */
  if(!arg_debugger)
    arg_debugger = "gdb" ;
  /* default xterm is xterm */
  if(!arg_xterm)
    arg_xterm = "xterm" ;

  arg_mylogin = mylogin();
#endif

#if CONVERSE_VERSION_VMI
  if (!arg_vmispecfile) {
    arg_vmispecfile = getenv ("VMI_SPECFILE");
  }

  if (!arg_vmispecfile) {
    fprintf (stderr, "ERROR> ++specfile not specified and VMI_SPECFILE not given in environment\n");
    exit (1);
  }
#endif

  /* find the current directory, absolute version */
  getcwd(buf, 1023);
  arg_currdir_a = strdup(buf);
    
  /* find the node-program, absolute version */
  arg_nodeprog_r = argv[1];

  if (arg_nodeprog_r[0]=='-' || arg_nodeprog_r[0]=='+') 
  { /*If it starts with - or +, it ain't a node program.
      Chances are, the user screwed up and passed some unknown flag to charmrun*/
     printf("Charmrun does not recognize the flag '%s'.\n",arg_nodeprog_r);
     if (arg_nodeprog_r[0]=='+')
       printf("Charm++'s flags need to be placed *after* the program name.\n");
     pparam_printdocs();
     exit(1);
  }


#if defined(_WIN32) && !defined(__CYGWIN__)
  if (argv[1][1]==':' || argv[1][0]=='\\' && argv[1][1]=='\\') { /*E.g.: "C:\foo\bar.exe*/
#else
  if (argv[1][0]=='/') { /*E.g.: "\foo\bar"*/
#endif
          /*Absolute path to node-program*/
    arg_nodeprog_a = argv[1];
  } else {
    sprintf(buf,"%s%s%s",arg_currdir_a,DIRSEP,arg_nodeprog_r);
    arg_nodeprog_a = strdup(buf);
  }
}

/****************************************************************************
 *                                                                           
 * NODETAB:  The nodes file and nodes table.
 *
 ****************************************************************************/

static int portOk = 1;
static const char *nodetab_tempName=NULL;
char *nodetab_file_find()
{
  char buffer[MAXPATHLEN];

  /* Find a nodes-file as specified by ++nodelist */
  if (arg_nodelist) {
    char *path = arg_nodelist;
    if (probefile(path)) return strdup(path);
    fprintf(stderr,"ERROR> No such nodelist file %s\n",path);
    exit(1);
  }
  /* Find a nodes-file as specified by getenv("NODELIST") */
  if (getenv("NODELIST")) {
    char *path = getenv("NODELIST");        
    if (path && probefile(path)) return strdup(path);
    fprintf(stderr,"ERROR> Cannot find nodelist file %s\n",path);
    exit(1);
  }
  /* Find a nodes-file by looking under 'nodelist' in the current directory */
  if (probefile("./nodelist")) return strdup("./nodelist");
#if defined(_WIN32) && !defined(__CYGWIN__)
  tmpnam(buffer);
  nodetab_tempName=strdup(buffer);
#else /*UNIX*/
  if (getenv("HOME")) {
    sprintf(buffer,"%s/.nodelist",getenv("HOME"));
  }
#endif
  if (!probefile(buffer)) 
  {
    /*Create a simple nodelist in the user's home*/
    FILE *f=fopen(buffer,"w");
    if (f==NULL) {
      fprintf(stderr,"ERROR> Cannot create a 'nodelist' file.\n");
      exit(1);
    }
    fprintf(f,"group main\nhost localhost\n");
    fclose(f);
  }
  return strdup(buffer);
}

typedef struct nodetab_host {
  char    *name;  /*Host DNS name*/
  skt_ip_t ip; /*IP address of host*/
  pathfixlist pathfixes;
  char    *ext;  /*FIXME: What the heck is this?  OSL 9/8/00*/
  int      cpus;  /* # of physical CPUs*/
  int      rank;  /*Rank of this CPU*/
  double   speed; /*Relative speed of each CPU*/
  int      nice;  /* process priority */
  /*These fields are set during node-startup*/
  int     dataport;/*UDP port number*/
  SOCKET  ctrlfd;/*Connection to control port*/
#if CMK_USE_RSH
  char    *shell;  /*Rsh to use*/
  char    *debugger ; /*Debugger to use*/
  char    *xterm ;  /*Xterm to use*/
  char    *login;  /*User login name to use*/
  char    *passwd;  /*User login password*/
  char    *setup;  /*Commands to execute on login*/
#endif

#if CMK_USE_IBVERBS
        ChInfiAddr *qpData;
#endif


} nodetab_host;

nodetab_host **nodetab_table;
int           nodetab_max;
int           nodetab_size;
int          *nodetab_rank0_table;
int           nodetab_rank0_size;

void nodetab_reset(nodetab_host *h)
{
  h->name="SET_H->NAME";
  h->ip=_skt_invalid_ip;
  h->pathfixes = 0;
  h->ext = NULL;
  h->speed = 1.0;
  h->cpus = 1;
  h->rank = 0;
  h->nice=-100;
  h->dataport=-1;
  h->ctrlfd=-1;
#if CMK_USE_RSH
  h->shell = arg_shell;
  h->debugger = arg_debugger;
  h->xterm = arg_xterm;
  h->login = arg_mylogin;
  h->passwd = "*";
  h->setup = "*";
#endif
}

void nodetab_add(nodetab_host *h)
{
  if (h->rank == 0)
    nodetab_rank0_table[nodetab_rank0_size++] = nodetab_size;
  nodetab_table[nodetab_size] = (nodetab_host *) malloc(sizeof(nodetab_host));

  if (arg_verbose) {
    char ips[200];
    skt_print_ip(ips,h->ip);
    printf("Charmrun> adding client %d: \"%s\", IP:%s\n", nodetab_size, h->name, ips);
  }

  *nodetab_table[nodetab_size++] = *h;
}

void nodetab_makehost(char *name,nodetab_host *h)
{
  h->name=strdup(name);
  h->ip = skt_innode_lookup_ip(name);
  if (skt_ip_match(h->ip,_skt_invalid_ip)) {
#ifdef CMK_BPROC
    /* only the master node is used */
    if (!(1 == arg_requested_pes && atoi(name)==-1))
#endif
    {
    fprintf(stderr,"ERROR> Cannot obtain IP address of %s\n", name);
    exit(1);
    }
  }
  if (nodetab_size == nodetab_max) return;
  nodetab_add(h);
}

char *nodetab_args(char *args,nodetab_host *h)
{
  if (arg_ppn>0) h->cpus = arg_ppn;
  while(*args != 0) {
    char *b1 = skipblanks(args), *e1 = skipstuff(b1);
    char *b2 = skipblanks(e1), *e2 = skipstuff(b2);
    while (*b1=='+') b1++;/*Skip over "++" on parameters*/
#if CMK_USE_RSH
    if (subeqs(b1,e1,"login")) h->login = substr(b2,e2);
    else if (subeqs(b1,e1,"passwd")) h->passwd = substr(b2,e2);
    else if (subeqs(b1,e1,"setup")) h->setup = strdup(b2);
    else if (subeqs(b1,e1,"shell")) h->shell = substr(b2,e2);
    else if (subeqs(b1,e1,"debugger")) h->debugger = substr(b2,e2);
    else if (subeqs(b1,e1,"xterm")) h->xterm = substr(b2,e2);
    else 
#endif
    if (subeqs(b1,e1,"speed")) h->speed = atof(b2);
    else if (subeqs(b1,e1,"cpus")) {
      if (arg_ppn==0) h->cpus = atol(b2);       /* ignore if there is ++ppn */
    }
    else if (subeqs(b1,e1,"pathfix")) {
      char *b3 = skipblanks(e2), *e3 = skipstuff(b3);
      args = skipblanks(e3);
      h->pathfixes=pathfix_append(substr(b2,e2),substr(b3,e3),h->pathfixes);
      e2 = e3;       /* for the skipblanks at the end */
    } 
    else if (subeqs(b1,e1,"ext"))  h->ext = substr(b2,e2);
    else if (subeqs(b1,e1,"nice"))  h->nice = atoi(b2);
    else return args;
    args = skipblanks(e2);
  }
#if CMK_SHARED_VARS_UNAVAILABLE
  if (h->cpus != 1) {
    fprintf(stderr,"Warning> Invalid cpus %d in nodelist ignored.\n", h->cpus);
    h->cpus = 1;
  }
#endif
  return args;
}

/*  setup nodetab as localhost only */
void nodetab_init_for_local()
{
  int tablesize, i, done=0;
  nodetab_host group;

  tablesize = arg_requested_pes;
  nodetab_table=(nodetab_host**)malloc(tablesize*sizeof(nodetab_host*));
  nodetab_rank0_table=(int*)malloc(tablesize*sizeof(int));
  nodetab_max=tablesize;

  nodetab_reset(&group);
  if (arg_ppn==0) arg_ppn=1;
#if CMK_SHARED_VARS_UNAVAILABLE
  if (arg_ppn > 1) {
    fprintf(stderr,"Warning> Invalid ppn %d in nodelist ignored.\n", arg_ppn);
    arg_ppn=1;
  }
#endif
  group.cpus = arg_ppn;
  i = 0;
  while (!done) {
    char *hostname = "127.0.0.1";
    for (group.rank = 0;