Home Research Papers Posters Manuals Talks Download People Help Internal

libs/ck-libs/liveViz3d/lv3d0_server.C

Go to the documentation of this file.

/*
  Implementation of the simplest server portion 
  of the liveViz3d library.
  
  Orion Sky Lawlor, olawlor@acm.org, 2003/9/13
*/
#include "lv3d0_server.h"
#include "pup_toNetwork.h"
#include <vector>
#include <map>
#include <algorithm>
#include "stats.h"
#include "LBDatabase.h" /* for set_avail_vector */

#include "lv3d0.decl.h" //For LV3D0_ViewMsg
void _registerlv3d1(void);
void _registerliveViz(void);
// #include "lv3d1.decl.h" //For LV3D1 registration (for .def file)

static /* readonly */ CProxy_LV3D0_Manager mgrProxy;
static /* readonly */ int LV3D_dosave_views=0; 
static /* readonly */ int LV3D_disable_ship_prio=0; 
static /* readonly */ int LV3D_disable_ship_replace=0; 
static /* readonly */ int LV3D_disable_ship_throttle=0; 
static /* readonly */ int LV3D_disable_ship=0;  
#define masterProcessor 0

class LV3D0_ViewMsg : public CMessage_LV3D0_ViewMsg {
public:
// (these fields are all copied from our viewable)
    CkViewableID id;
    
    int prio;
    
    int pixels;
    
    int clientID;
    
    int view_size;
    unsigned char *view;
    
    static LV3D0_ViewMsg *new_(CkView *view);
    static void delete_(LV3D0_ViewMsg *m);
};

LV3D0_ViewMsg *LV3D0_ViewMsg::new_(CkView *vp) {
    PUP_toNetwork_sizer ps; ps|vp;
    int view_size=ps.size();
    
    LV3D0_ViewMsg *m=new (view_size,0) LV3D0_ViewMsg;
    m->id=vp->id;
    m->prio=vp->prio;
    m->pixels=vp->pixels;
    m->view_size=view_size;
    PUP_toNetwork_pack pp(m->view); pp|vp;
    return m;
}
void LV3D0_ViewMsg::delete_(LV3D0_ViewMsg *m) {
    delete m;
}


/********** LV3D0_ClientManager: outgoing images ********/

class CkViewPrioHolder {
public:
    LV3D0_ViewMsg *v;
    
    CkViewPrioHolder(LV3D0_ViewMsg *v_) :v(v_) {}
    
    // Comparison operator, used to keep a priority queue.
    //  Returns true if we should be processed before this guy.
    bool operator<(const CkViewPrioHolder &h) const {
        if (v->prio<h.v->prio) return true;
        if (v->prio>h.v->prio) return false;
        /* else priorities are equal: compare ID's */
        int i;
        for (i=0;i<4;i++) {
            if (v->id.id[i]<h.v->id.id[i]) return true;
            if (v->id.id[i]>h.v->id.id[i]) return false;
        }
        // Woa-- they're *identical*.  Just pick smaller one.
        return v < h.v;
    }
};

class CkViewPrioSorter {

    /*
      We generally want to process views in priority order;
      but we never want to send an earlier view to a client
      when a later view is available.
      
      This means we need to keep views in a priority queue
      (for extraction), but also provide a way to look up 
      the view (to prevent duplicates).
    */
    
    // This indexes unsent views by ID:
    CkHashtableT<CkViewableID,CkViewPrioHolder> id2view;

    // This indexes unsent views by priority:
    //   The "key" is all that's needed here-- the "obj" field is useless.
    typedef std::map<CkViewPrioHolder,char> prio2view_t;
    prio2view_t prio2view;
    
public:
    CkViewPrioSorter() 
        :id2view(129,0.2)
    {
    }
    ~CkViewPrioSorter() {
        for (iterator it=begin();it!=end();++it)
            delete (*it).first.v;
    }
    
    bool isEmpty(void) const {return prio2view.size()==0;}
    
    void add(LV3D0_ViewMsg *v)
    {
        if (LV3D_disable_ship_prio) v->prio=0;
        CkViewPrioHolder old=id2view.get(v->id);
        if (old.v!=0 && !LV3D_disable_ship_replace) 
        { /* An old entry from this viewable exists: remove it.
             This is so we always send off the most up-to-date view. */
            // printf("[OUT] Replacing old view for viewable %d\n",v->id.id[0]);
            if (v->prio>old.v->prio)  // keep old priority if it's higher
                v->prio=old.v->prio;
            prio2view.erase(prio2view.find(old));
            delete old.v;
        }
        else {
            // printf("[OUT] Queueing new view for viewable %d\n",v->id.id[0]);
        }
        id2view.put(v->id)=CkViewPrioHolder(v);
        prio2view.insert(std::make_pair(CkViewPrioHolder(v),(char)1));
    }
    
    typedef prio2view_t::iterator iterator;
    iterator begin(void) {return prio2view.begin();}
    iterator end(void) {return prio2view.end();}
    
    LV3D0_ViewMsg *extract(const iterator &doomed) {
        LV3D0_ViewMsg *v=(*doomed).first.v;
        prio2view.erase(doomed);
        id2view.remove(v->id);
        return v;
    }
};

class LV3D0_ClientManager : public CkViewPrioSorter {
public:
    virtual ~LV3D0_ClientManager() {}
    virtual void add(LV3D0_ViewMsg *v) =0;
    virtual void getViews(void) {}
    virtual void whenEmptyCallback(const CkCallback &cb) {}
};

/******************** Shipping views to Master ***************/
int LV3D0_toMaster_bytesPer=100*1024;
int LV3D0_toMaster_bytesMax=2*LV3D0_toMaster_bytesPer;

static void toMaster_fillBucket(void *ptr,double timer);

static stats::op_t op_master_count=stats::count_op("master.count","Number of final master impostors","count");
static stats::op_t op_master_bytes=stats::count_op("master.bytes","Number of final master bytes","bytes");
static stats::op_t op_master_pixels=stats::count_op("master.pixels","Number of final master pixels","pixels");

class LV3D0_ClientManager_toMaster : public LV3D0_ClientManager {
    int bucket_bytes;
    int cidx;
public:
    LV3D0_ClientManager_toMaster() {
        bucket_bytes=LV3D0_toMaster_bytesMax;
        cidx=CcdCallOnConditionKeep(CcdPERIODIC_10ms,toMaster_fillBucket,this);
    }
    ~LV3D0_ClientManager_toMaster() {
        CcdCancelCallOnConditionKeep(CcdPERIODIC_10ms,cidx);
    }
    
    virtual void add(LV3D0_ViewMsg *v) {
        CkViewPrioSorter::add(v);
        progress();
    }
    void progress(void) {
        // printf("progress: bucket=%d\n",bucket_bytes);
        if (CmiLongSendQueue(CmiNodeOf(masterProcessor),LV3D0_toMaster_bytesMax))
            return; /* too many outstanding messages already! */
        while ((!isEmpty()) && (bucket_bytes>0 || LV3D_disable_ship_throttle)) 
        {
            LV3D0_ViewMsg *v=extract(begin());
            bucket_bytes-=v->view_size;
            stats::get()->add(1.0,op_master_count);
            stats::get()->add(v->view_size,op_master_bytes);
            stats::get()->add(v->pixels,op_master_pixels);
            mgrProxy[masterProcessor].addView(v);
        }
    }
    void fillBucket(void) {
        bucket_bytes+=LV3D0_toMaster_bytesPer;
        if (bucket_bytes>LV3D0_toMaster_bytesMax) 
            bucket_bytes=LV3D0_toMaster_bytesMax;
        progress();
    }
};
static void toMaster_fillBucket(void *ptr,double timer) {
    LV3D0_ClientManager_toMaster *p=(LV3D0_ClientManager_toMaster *)ptr;
    p->fillBucket();
}

/************** Shipping views to Client ***************/
int LV3D0_toClient_bytesPer=100*1024;

static stats::op_t op_client_pack=stats::time_op("client.pack","Time spent packing final client impostors");
static stats::op_t op_client_count=stats::count_op("client.count","Number of final client impostors","count");
static stats::op_t op_client_bytes=stats::count_op("client.bytes","Number of final client bytes","bytes");
static stats::op_t op_client_pixels=stats::count_op("client.pixels","Final client impostor pixels","pixels");

class LV3D0_ClientManager_toClient : public LV3D0_ClientManager
{
    typedef CkViewPrioSorter super;
    bool hasDelayed;
    CcsDelayedReply delayedReply;
    
    CkVec<CkCallback> emptyCallbacks;
    void checkEmpty(void) {
        if (isEmpty()) { /* no views left-- send all empty callbacks */
            for (int i=0;i<emptyCallbacks.size();i++)
                emptyCallbacks[i].send();
            emptyCallbacks.resize(0);
        }
    }
    
    void sendReply(CcsDelayedReply repl) {
        stats::op_sentry stats_sentry(op_client_pack);
        
    // Figure out how many views we can afford to send at once:
        int len=4; /* leave room for the "length" field */
        int n=0;
        iterator it=begin();
        while (it!=end() && (len<LV3D0_toClient_bytesPer || LV3D_disable_ship_throttle)) {
            LV3D0_ViewMsg *v=(*it++).first.v;
            len+=v->view_size;
            n++;
        }
    
    // Pack the views into a network buffer:
        // CmiPrintf("[OUT] Sending off %d new views (%d bytes)\n",n,len);
        
        // FIXME: special case this when n==1.
        char *retMsg=new char[len];
        PUP_toNetwork_pack pp(retMsg);
        pp|n;
        for (int i=0;i<n;i++) {
            LV3D0_ViewMsg *v=extract(begin());
            pp(v->view,v->view_size);
            stats::get()->add(v->pixels,op_client_pixels);
            delete v;
        }
        
        if (len!=pp.size()) {
            CkError("Sizing pup was %d bytes; packing pup was %d!\n",
                len,pp.size());
            CkAbort("Pup size mismatch (logic error) in LV3D0_!\n");
        }

        stats::get()->add(1.0,op_client_count);
        stats::get()->add(len,op_client_bytes);
        CcsSendDelayedReply(repl,len,retMsg);
        delete[] retMsg;
        
        // CmiPrintf("[OUT] Done sending off %d views\n",n);
        checkEmpty();
    }
    
public:
    LV3D0_ClientManager_toClient() 
    {
        hasDelayed=false;
    }
    
    virtual void add(LV3D0_ViewMsg *v) {
        super::add(v);
        
        if (hasDelayed) { //There's already somebody waiting
            hasDelayed=false;
            sendReply(delayedReply);
        }
    }
    
    virtual void getViews(void) {
        if (isEmpty()) { //Nothing to send yet-- wait for it
            // printf("[OUT] Views requested, but none available\n");
            hasDelayed=true;
            delayedReply=CcsDelayReply();
        }
        else 
        {
            sendReply(CcsDelayReply());
        }   
    }
    
    virtual void whenEmptyCallback(const CkCallback &cb) {
        emptyCallbacks.push_back(cb);
        checkEmpty();
    }
};

class LV3D0_Manager : public CBase_LV3D0_Manager {
    CkHashtableT<CkHashtableAdaptorT<int>,LV3D0_ClientManager *> clientTable;
    
    int nextClientID;
public:
    LV3D0_Manager(void);
    
    int newClient(void);
    
    LV3D0_ClientManager *getClient(int clientID);
    
    void getViews(int clientID) {
        getClient(clientID)->getViews();
    }
    
    inline void addView(LV3D0_ViewMsg *m) {
        getClient(m->clientID)->add(m);
    }
};


LV3D0_Manager::LV3D0_Manager(void)
{
    mgrProxy=thisgroup;
    nextClientID=1;
}

int LV3D0_Manager::newClient(void)
{
    return nextClientID++;
}

LV3D0_ClientManager *LV3D0_Manager::getClient(int clientID)
{
    LV3D0_ClientManager *m=clientTable.get(clientID);
    if (m==NULL) {
        if (CkMyPe()==masterProcessor) 
            m=new LV3D0_ClientManager_toClient;
        else
            m=new LV3D0_ClientManager_toMaster;
        clientTable.put(clientID)=m;
    }
    return m;
}

static stats::op_t op_deposit_views=stats::count_op("deposit.views","CkView count","CkViews");
static stats::op_t op_deposit_bytes=stats::count_op("deposit.bytes","CkView sizes","bytes");
static stats::op_t op_deposit_pixels=stats::count_op("deposit.pixels","CkView pixels","pixels");

/***** Saving Views to File ********/
CkpvStaticDeclare(FILE *,LV3D_save_views);
static double LV3D_save_viewStart=0; 
static char *LV3D_copy_view_src=0, *LV3D_copy_view_dest=0;
static stats::op_t op_save=stats::time_op("save.time","Time spent saving views to disk");

static void LV3D_save_init(void) {
    if (LV3D_copy_view_src==0) return;
    if (CkpvAccess(LV3D_save_views)) { /* file already open: close and re-open */
        fclose(CkpvAccess(LV3D_save_views));
    }
    
    char fName[1024];
    sprintf(fName,LV3D_copy_view_src,CkMyPe());
    FILE *f=fopen(fName,"wb");
    if (f==NULL) CmiAbort("Couldn't create save view file!\n");
    CkpvAccess(LV3D_save_views)=f;
    CkPrintf("Created views file %s\n",fName);
}

static void LV3D_save_start(void)
{
    if (!LV3D_dosave_views) return;
    LV3D_save_init();
    LV3D_save_viewStart=CkWallTimer(); //< HACK!
}

struct savedViewRecord {
public:
    double t; /* Time view was rendered */
    int view_size; /* size, in bytes, of view buffer */
    void pup(PUP::er &p) {
        p|t;
        p|view_size;
    }
};

static int LV3D_save_view(LV3D0_ViewMsg *v) {
    if (!LV3D_dosave_views) return 0;
    if (!CkpvAccess(LV3D_save_views)) return 0;
    stats::op_sentry stats_sentry(op_save);
    savedViewRecord rec;
    rec.t=CkWallTimer()-LV3D_save_viewStart;
    rec.view_size=v->view_size;
    enum {bufLen=sizeof(rec)};
    char buf[bufLen];
    PUP_toNetwork_pack p(buf); p|rec;
    FILE *f=CkpvAccess(LV3D_save_views);
    if (1!=fwrite(buf,p.size(),1,f)) CmiAbort("Can't write header to saved view file!\n");
    if (1!=fwrite(v->view,v->view_size,1,f)) CmiAbort("Can't write view to saved view file!\n");
    delete v;
    return 1;
}

static void LV3D_save_finish(void) {
    if (!LV3D_dosave_views) return;
    if (!CkpvAccess(LV3D_save_views)) return;
    fclose(CkpvAccess(LV3D_save_views));
    CkpvAccess(LV3D_save_views)=0;
    if (LV3D_copy_view_dest) { /* Copy view file to dest directory */
        char fSrc[1024], fDest[1024], cmd[2048];
        sprintf(fSrc,LV3D_copy_view_src,CkMyPe());
        sprintf(fDest,LV3D_copy_view_dest,CkMyPe());
        sprintf(cmd,"cp '%s' '%s' && rm '%s'", fSrc,fDest, fSrc);
        CkPrintf("Copying views file from %s to %s\n",fSrc,fDest);
        system(cmd);
        CkPrintf("Views file copied.\n");
    }
}


void LV3D0_Deposit(CkView *v,int clientID) {
    stats::stats *s=stats::get();
    s->add(1.0,op_deposit_views);
    s->add(v->pixels,op_deposit_pixels);
    LV3D0_ViewMsg *vm=LV3D0_ViewMsg::new_(v);
    if (LV3D_save_view(vm)) return;
    s->add(vm->view_size,op_deposit_bytes);
    if (LV3D_disable_ship) {delete vm; return;}
    vm->clientID=clientID;
    mgrProxy.ckLocalBranch()->addView(vm);
}


/**************** Performance collection ***************/

static stats::op_t op_pes=stats::count_op("cmi.pes","Processors","pes");
static stats::op_t op_time=stats::time_op("cmi.time","Elapsed wall-clock time");
static stats::op_t op_unknown=stats::time_op("cmi.unknown","Unaccounted-for time");
static stats::op_t op_idle=stats::time_op("cmi.idle","Time spent waiting for data");


static CcsDelayedReply statsReply;

static void printStats(void *rednMsg) {
    CkReductionMsg *m=(CkReductionMsg *)rednMsg;
    
    // FIXME: make stats::print return a std::string, instead
    //  of this horrible "print to a file and read it back" business.
    
    char tmpFileName[100];
    sprintf(tmpFileName,"/tmp/stats.%d.%d",CkMyPe(),(int)getpid());
    FILE *f=fopen(tmpFileName,"w");
    int len=0; void *buf=0;
    if (f!=NULL) {
        /* write stats to temp file */
        const stats::stats *s=(const stats::stats *)m->getData();
        s->print(f,"total",1.0);
        s->print(f,"per_second",1.0/s->get(op_time));
        s->print(f,"per_pe-second",1.0/(CkNumPes()*s->get(op_time)));
        fclose(f);
        
        /* Read stats back */
        f=fopen(tmpFileName,"r");
        fseek(f,0,SEEK_END);
        len=ftell(f);
        buf=malloc(len);
        fseek(f,0,SEEK_SET);
        fread(buf,1,len,f);
        fclose(f);
        
        /* print to screen */
        write(1,buf,len);
        printf("\n");
    
    }
    unlink(tmpFileName);
    CcsSendDelayedReply(statsReply,len,buf);
    free(buf);
    delete m;
}

static void perfmanager_stats_idle(void *ptr,double timer)
{
    stats::swap(op_idle);
}

class LV3D_PerfManager : public CBase_LV3D_PerfManager {
    double startTime;
public:
    LV3D_PerfManager(void) {
        zero();
        CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,perfmanager_stats_idle,0);
        CcdCallOnConditionKeep(CcdPROCESSOR_END_IDLE,perfmanager_stats_idle,0);
        
        /* Don't let the load balancer move stuff onto node 0: */
        char *bitmap=new char[CkNumPes()];
        for (int i=0;i<CkNumPes();i++)
            bitmap[i]=(i!=masterProcessor);
        set_avail_vector(bitmap);
    }
    void zero(void) { 
        stats::stats *s=stats::get();
        stats::swap(op_unknown); // so unknown gets zerod out
        s->zero();
        s->add(1.0,op_pes);
        startTime=stats::time();
        stats::swap(op_unknown);
        LV3D_save_start();
    }
    void collect(void) { 
        stats::stats *s=stats::get();
        if (CkMyPe()==0) s->set(stats::time()-startTime,op_time);
        else s->set(0,op_time);
        stats::swap(op_unknown);
        contribute(sizeof(double)*stats::op_len,&s->t[0],CkReduction::sum_double,
            CkCallback(printStats));
        LV3D_save_finish();
        zero();
    }
    void traceOn(void) {
        traceBegin();
    }
    void startBalance(void) {
        LBClearLoads();
        LBTurnInstrumentOn();
    }
    void doneBalance(void) {
        LBTurnInstrumentOff();
    }
    void throttle(int throttleOn) { LV3D_disable_ship_throttle=!throttleOn; }
};


/*************** CCS Interface ***************/
//This only ever gets set on processor 0 (FIXME: checkpointing)
static LV3D_Universe *theUniverse=0;
static LV3D_ServerMgr *theMgr=0;
static CProxy_LV3D_PerfManager perfMgr;

LV3D_ServerMgr::~LV3D_ServerMgr() {}

extern "C" void LV3D0_setup(char *msg) {
    CmiFree(msg);
    int clientID=mgrProxy.ckLocalBranch()->newClient();
    PUP_toNetwork_sizer sp;
    sp|clientID;
    sp|theUniverse;
    unsigned char *buf=new unsigned char[sp.size()];
    PUP_toNetwork_pack pp(buf);
    pp|clientID;
    pp|theUniverse;
    CcsSendReply(sp.size(),buf);
    theMgr->newClient(clientID);
    CmiPrintf("Registered (client %d)\n",clientID);
    delete[] buf;
    perfMgr.zero(); // New client implicitly zeros out stats.
}


static stats::op_t op_view_count=stats::count_op("client.views","New viewpoints","Views");

extern "C" void LV3D0_newViewpoint(char *msg) {
    PUP_toNetwork_unpack p(&msg[CmiMsgHeaderSizeBytes]);
    LV3D_ViewpointMsg *m=new LV3D_ViewpointMsg;
    p|m->clientID;
    p|m->frameID;
    m->viewpoint.pup(p);
    // CmiPrintf("New user viewpoint (client %d, frame %d)\n",clientID,frameID);
    theMgr->newViewpoint(m);
    stats::get()->add(1.0,op_view_count);
}

extern "C" void LV3D0_getViews(char *msg) {
    PUP_toNetwork_unpack p(&msg[CmiMsgHeaderSizeBytes]);
    int clientID=0; p|clientID; CmiFree(msg);
    mgrProxy.ckLocalBranch()->getViews(clientID);
}

/*
"lv3d_qd" CCS handler:
    Sent by the client to wait until all views have been
updated and sent back.  No input or output data: control flow only.
*/
struct lv3d_qdState {
    int clientID;
    
    CcsDelayedReply reply;
};

static void qdDoneFn(void *param,void *msg);
static void emptyDoneFn(void *param,void *msg);

extern "C" void LV3D0_qd(char *msg) /* stage 1 */
{
    lv3d_qdState *s=new lv3d_qdState;
    PUP_toNetwork_unpack p(&msg[CmiMsgHeaderSizeBytes]);
    p|s->clientID;
    s->reply=CcsDelayReply();
    CmiFree(msg);
    CkCallback cb(qdDoneFn,s);
    CkStartQD(cb); /* finish CCS reply after quiescence */
}
static void qdDoneFn(void *param,void *msg)  /* stage 2 */
{
    lv3d_qdState *s=(lv3d_qdState *)param;
    mgrProxy.ckLocalBranch()->getClient(s->clientID)->whenEmptyCallback(
        CkCallback(emptyDoneFn,s));
}
static void emptyDoneFn(void *param,void *msg) /* stage 3 */
{
    lv3d_qdState *s=(lv3d_qdState *)param;
    CcsSendDelayedReply(s->reply,0,0);
    delete s;
}

extern "C" void LV3D0_flush(char *msg) {
    int clientID=0;
    PUP_toNetwork_unpack p(&msg[CmiMsgHeaderSizeBytes]);
    p|clientID;
    CmiFree(msg);
    theMgr->newClient(clientID);
}

/*
"lv3d_startbal" CCS handler:
    Begin a load balancing phase.  Normally followed by 
    some rendering, then an endbalance.
*/
extern "C" void LV3D0_startbalance(char *msg) 
{
    CkPrintf("CCS call to LV3D0_startbalance\n");
    perfMgr.startBalance();
    CmiFree(msg);
}
/*
"lv3d_endbal" CCS handler:
    Go to sync.  Exactly one end must follow 
    each startbalance call.
*/
extern "C" void LV3D0_endbalance(char *msg) 
{
    CkPrintf("CCS call to LV3D0_endbalance\n");
    theMgr->doBalance();
    perfMgr.doneBalance();
    LV3D0_qd(msg); /* wait for quiescence */
}

/*
"lv3d_quit" CCS handler:
    Shut down the program.
*/
extern "C" void LV3D0_quit(char *msg) 
{
    CkPrintf("Exiting: CCS call to LV3D0_quit\n");
    CcsSendReply(0,0);
    CmiFree(msg);
    CkExit();
}
extern "C" void LV3D0_zero(char *msg) 
{
    CkPrintf("Zeroing statistics\n");
    perfMgr.zero();
    CmiFree(msg);
}
extern "C" void LV3D0_stats(char *msg) 
{
    CkPrintf("Printing statistics\n");
    statsReply=CcsDelayReply();
    perfMgr.collect();
    CmiFree(msg);
}
extern "C" void LV3D0_trace(char *msg) 
{
    CkPrintf("Tracing turned on\n");
    perfMgr.traceOn();
    CmiFree(msg);
}
extern "C" void LV3D0_throttle0(char *msg) { perfMgr.throttle(0); CmiFree(msg); }
extern "C" void LV3D0_throttle1(char *msg) { perfMgr.throttle(1); CmiFree(msg); }

void LV3D0_Init(LV3D_Universe *clientUniverse,LV3D_ServerMgr *mgr)
{
    if (clientUniverse==0)
        clientUniverse=new LV3D_Universe();
    theUniverse=clientUniverse;
    theMgr=mgr;
    CcsRegisterHandler("lv3d_setup",(CmiHandler)LV3D0_setup);
    CcsRegisterHandler("lv3d_flush",(CmiHandler)LV3D0_flush);
    CcsRegisterHandler("lv3d_newViewpoint",(CmiHandler)LV3D0_newViewpoint);
    CcsRegisterHandler("lv3d_getViews",(CmiHandler)LV3D0_getViews);
    CcsRegisterHandler("lv3d_qd",(CmiHandler)LV3D0_qd);
    CcsRegisterHandler("lv3d_startbal",(CmiHandler)LV3D0_startbalance);
    CcsRegisterHandler("lv3d_endbal",(CmiHandler)LV3D0_endbalance);
    CcsRegisterHandler("lv3d_quit",(CmiHandler)LV3D0_quit);
    CcsRegisterHandler("lv3d_zero",(CmiHandler)LV3D0_zero);
    CcsRegisterHandler("lv3d_stats",(CmiHandler)LV3D0_stats);
    CcsRegisterHandler("lv3d_trace",(CmiHandler)LV3D0_trace);
    CcsRegisterHandler("lv3d_throttle0",(CmiHandler)LV3D0_throttle0);
    CcsRegisterHandler("lv3d_throttle1",(CmiHandler)LV3D0_throttle1);
    CProxy_LV3D0_Manager::ckNew();
    perfMgr=CProxy_LV3D_PerfManager::ckNew();
}

void LV3D0_ProcInit(void) {
    CkpvInitialize(FILE *,LV3D_save_views);
    CkpvAccess(LV3D_save_views)=0;
    CmiGetArgStringDesc(CkGetArgv(),"+LV3D_save_views",&LV3D_copy_view_src,"Save rendered views to a file with this pattern.  Use like '/tmp/views.%d.pe'");
    CmiGetArgStringDesc(CkGetArgv(),"+LV3D_copy_views",&LV3D_copy_view_dest,"Copy view files to this pattern.  Use like 'views.%d.pe'");
    LV3D0_toMaster_bytesPer=LV3D0_toMaster_bytesPer/CkNumPes();
    LV3D0_toMaster_bytesMax=LV3D0_toMaster_bytesMax/CkNumPes();
}
void LV3D0_NodeInit(void) {
    CkViewNodeInit();
}


#include "lv3d0.def.h"
#include "liveViz3d.def.h"

---