ck-core/ckmessagelogging.C File Reference

Go to the source code of this file.

Data Structures

class ElementPacker

class ElementDistributor

class ElementCaller

Functions

const char * idx2str (const CkArrayIndex &ind)

const char * idx2str (const ArrayElement *el)

void getGlobalStep (CkGroupID gID)

bool fault_aware (CkObjID &recver)

void sendCheckpointData (int mode)

Sends the checkpoint to its buddy.

void createObjIDList (void *data, ChareMlogData *mlogData)

bool isLocal (int destPE)

Determines if the message is local or not.

bool isTeamLocal (int destPE)

Determines if the message is group local or not.

void readKillFile ()

CpvDeclare (Chare *, _currentObj)

CpvDeclare (CkQ< LocalMessageLog > *, _localMessageLog)

CpvDeclare (CkQ< TicketRequest * > *, _delayedTicketRequests)

CpvDeclare (StoredCheckpoint *, _storedCheckpointData)

CpvDeclare (CkQ< MlogEntry * > *, _delayedLocalTicketRequests)

CpvDeclare (Queue, _outOfOrderMessageQueue)

CpvDeclare (CkQ< LocalMessageLog > *, _bufferedLocalMessageLogs)

CpvDeclare (char **, _bufferedTicketRequests)

CpvDeclare (int *, _numBufferedTicketRequests)

CpvDeclare (char *, _bufferTicketReply)

void setTeamRecovery (void *data, ChareMlogData *mlogData)

Turns on the flag for team recovery that selectively restores particular metadata information.

void unsetTeamRecovery (void *data, ChareMlogData *mlogData)

Turns off the flag for team recovery.

void _messageLoggingInit ()

Initialize message logging data structures and register handlers.

void killLocal (void *_dummy, double curWallTime)

void sendTicketGroupRequest (envelope *env, int destPE, int _infoIdx)

void sendTicketNodeGroupRequest (envelope *env, int destNode, int _infoIdx)

void sendTicketArrayRequest (envelope *env, int destPE, int _infoIdx)

void generateCommonTicketRequest (CkObjID &recver, envelope *_env, int destPE, int _infoIdx)

A method to generate the actual ticket requests for groups, nodegroups or arrays.

void sendTicketRequest (CkObjID &sender, CkObjID &recver, int destPE, MlogEntry *entry, MCount SN, MCount TN, int resend)

Method that does the actual send by creating a ticket request filling it up and sending it.

void sendBufferedTicketRequests (int destPE)

Send the ticket requests buffered for processor PE.

void checkBufferedTicketRequests (void *_destPE, double curWallTime)

void ticketLogLocalMessage (MlogEntry *entry)

Gets a ticket for a local message and then sends a copy to the buddy.

void sendLocalMessageCopy (MlogEntry *entry)

Sends the metadata of a local message to its buddy.

void sendBufferedLocalMessageCopy ()

void checkBufferedLocalMessageCopy (void *_dummy, double curWallTime)

bool _processTicketRequest (TicketRequest *ticketRequest, TicketReply *reply)

Process the ticket request.

void _ticketRequestHandler (TicketRequest *ticketRequest)

If there are any delayed requests, process them first before processing this request.

void _bufferedTicketRequestHandler (BufferedTicketRequestHeader *recvdHeader)

Handler used for dealing with a bunch of ticket requests from one processor.

void _ticketHandler (TicketReply *ticketReply)

This function handles the ticket received after a request.

void _bufferedTicketHandler (BufferedTicketRequestHeader *recvdHeader)

Message to handle the bunch of tickets that we get from one processor.

void _localMessageCopyHandler (LocalMessageLog *msgLog)

Stores the metadata of a local message from its buddy.

void _bufferedLocalMessageCopyHandler (BufferedLocalLogHeader *recvdHeader, int freeHeader)

void _localMessageAckHandler (LocalMessageLogAck *ack)

void _bufferedLocalMessageAckHandler (BufferedLocalLogHeader *recvdHeader)

int preProcessReceivedMessage (envelope *env, Chare **objPointer, MlogEntry **logEntryPointer)

void postProcessReceivedMessage (Chare *obj, CkObjID &sender, MCount SN, MlogEntry *entry)

Updates a few variables once a message has been processed.

void generalCldEnqueue (int destPE, envelope *env, int _infoIdx)

void retryTicketRequestTimer (void *_dummy, double _time)

void retryTicketRequest (void *_dummy, double curWallTime)

void _pingHandler (CkPingMsg *msg)

void buildProcessedTicketLog (void *data, ChareMlogData *mlogData)

void clearUpMigratedRetainedLists (int PE)

void checkpointAlarm (void *_dummy, double curWallTime)

void _checkpointRequestHandler (CheckpointRequest *request)

void startMlogCheckpoint (void *_dummy, double curWallTime)

Starts the checkpoint phase after migration.

void pupArrayElementsSkip (PUP::er &p, CmiBool create, MigrationRecord *listToSkip, int listsize)

Pups all the array elements in this processor.

void writeCheckpointToDisk (int size, char *chkpt)

void _storeCheckpointHandler (char *msg)

void sendRemoveLogRequests ()

void _checkpointAckHandler (CheckPointAck *ackMsg)

void removeProcessedLogs (void *_data, ChareMlogData *mlogData)

void _removeProcessedLogHandler (char *requestMsg)

Removes messages in the log according to the received ticket numbers.

void CkMlogRestart (const char *dummy, CkArgMsg *dummyMsg)

Function for restarting the crashed processor.

void _restartHandler (RestartRequest *restartMsg)

Function to restart this processor.

void _getRestartCheckpointHandler (RestartRequest *restartMsg)

Gets the stored checkpoint but calls another function in the sender.

void _recvRestartCheckpointHandler (char *_restartData)

Receives the checkpoint coming from its buddy.

void CkMlogRestartDouble (void *, double)

void CkMlogRestartLocal ()

void readCheckpointFromDisk (int size, char *buf)

void _getCheckpointHandler (RestartRequest *restartMsg)

Gets the stored checkpoint for its buddy processor.

void _verifyAckRequestHandler (VerifyAckMsg *verifyRequest)

void _verifyAckHandler (VerifyAckMsg *verifyReply)

void _recvCheckpointHandler (char *_restartData)

Receives the checkpoint data from its buddy, restores the state of all the objects and asks everyone else to update its home.

void _updateHomeAckHandler (RestartRequest *updateHomeAck)

Receives the updateHome ACKs from all other processors.

void initializeRestart (void *data, ChareMlogData *mlogData)

Initializes variables and flags for restarting procedure.

void updateHomePE (void *data, ChareMlogData *mlogData)

Updates the homePe of chare array elements.

void _updateHomeRequestHandler (RestartRequest *updateRequest)

Updates the homePe for all chares in this processor.

void fillTicketForChare (void *data, ChareMlogData *mlogData)

Fills up the ticket vector for each chare.

void resendMessageForChare (void *data, ChareMlogData *mlogData)

void _resendMessagesHandler (char *msg)

Resends the messages since the last checkpoint to the list of objects included in the request.

void sortVec (CkVec< MCount > *TNvec)

int searchVec (CkVec< MCount > *TNVec, MCount searchTN)

void _resendReplyHandler (char *msg)

Receives the tickets assigned to message to other objects.

void _receivedTNDataHandler (ReceivedTNData *msg)

Receives a list of TNs coming from the home PE of a migrated object (parallel restart).

void processReceivedTN (Chare *obj, int listSize, MCount *listTNs)

Processes the received list of tickets from a particular PE.

void distributeRestartedObjects ()

Distributes objects to accelerate recovery after a failure.

void _distributedLocationHandler (char *receivedMsg)

Handler to update information about an object just received.

void sendDummyMigration (int restartPE, CkGroupID lbID, CkGroupID locMgrID, CkArrayIndex &idx, int locationPE)

this method is used to send messages to a restarted processor to tell it that a particular expected object is not going to get to it

void sendDummyMigrationCounts (int *dummyCounts)

this method is used by a restarted processor to tell other processors that they are not going to receive these many objects.

void _dummyMigrationHandler (DummyMigrationMsg *msg)

this handler is used to process a dummy migration msg.

void forAllCharesDo (MlogFn fnPointer, void *data)

Map function pointed by fnPointer over all the chares living in this processor.

void initMlogLBStep (CkGroupID gid)

This is the first time Converse is called after AtSync method has been called by every local object.

void startLoadBalancingMlog (void(*_fnPtr)(void *), void *_centralLb)

Load Balancing.

void finishedCheckpointLoadBalancing ()

void sendMlogLocation (int targetPE, envelope *env)

void _receiveMigrationNoticeHandler (MigrationNotice *msg)

void _receiveMigrationNoticeAckHandler (MigrationNoticeAck *msg)

void _receiveMlogLocationHandler (void *buf)

void resumeFromSyncRestart (void *data, ChareMlogData *mlogData)

void checkAndSendCheckpointBarrierAcks (CheckpointBarrierMsg *msg)

void _checkpointBarrierHandler (CheckpointBarrierMsg *msg)

Processor 0 receives a contribution from every other processor after checkpoint.

void _checkpointBarrierAckHandler (CheckpointBarrierMsg *msg)

void informLocationHome (CkGroupID locMgrID, CkArrayIndex idx, int homePE, int currentPE)

method that informs an array elements home processor of its current location It is a converse method to bypass the charm++ message logging framework

void _receiveLocationHandler (CurrentLocationMsg *data)

void _getGlobalStepHandler (LBStepMsg *msg)

void _recvGlobalStepHandler (LBStepMsg *msg)

Receives the global step handler from PE 0.

void _messageLoggingExit ()

Function to wrap up performance information.

void sortRestoredLocalMsgLog (void *_dummy, ChareMlogData *mlogData)

int getCheckPointPE ()

Getting the pe number of the current processor's buddy.

envelope * copyEnvelope (envelope *env)

Variables

int _restartFlag = 0

float MLOGFT_totalLogSize = 0.0

float MLOGFT_totalMessages = 0.0

float MLOGFT_totalObjects = 0.0

int countHashRefs = 0

int countHashCollisions = 0

char * checkpointDirectory = "."

int unAckedCheckpoint = 0

int countLocal = 0

int countBuffered = 0

int countPiggy = 0

int countClearBufferedLocalCalls = 0

int countUpdateHomeAcks = 0

int teamSize

int chkptPeriod

bool parallelRestart

char * killFile

int killFlag = 0

int restartingMlogFlag = 0

double killTime = 0.0

int checkpointCount = 0

static double adjustChkptPeriod = 0.0

static double nextCheckpointTime = 0.0

double lastBufferedLocalMessageCopyTime

int _maxBufferedMessages

int _maxBufferedTicketRequests

int BUFFER_TIME = 2

int _ticketRequestHandlerIdx

int _ticketHandlerIdx

int _localMessageCopyHandlerIdx

int _localMessageAckHandlerIdx

int _pingHandlerIdx

int _bufferedLocalMessageCopyHandlerIdx

int _bufferedLocalMessageAckHandlerIdx

int _bufferedTicketRequestHandlerIdx

int _bufferedTicketHandlerIdx

char objString [100]

int _checkpointRequestHandlerIdx

int _storeCheckpointHandlerIdx

int _checkpointAckHandlerIdx

int _getCheckpointHandlerIdx

int _recvCheckpointHandlerIdx

int _removeProcessedLogHandlerIdx

int _verifyAckRequestHandlerIdx

int _verifyAckHandlerIdx

int _dummyMigrationHandlerIdx

int _getGlobalStepHandlerIdx

int _recvGlobalStepHandlerIdx

int _updateHomeRequestHandlerIdx

int _updateHomeAckHandlerIdx

int _resendMessagesHandlerIdx

int _resendReplyHandlerIdx

int _receivedTNDataHandlerIdx

int _distributedLocationHandlerIdx

int _restartHandlerIdx

int _getRestartCheckpointHandlerIdx

int _recvRestartCheckpointHandlerIdx

int verifyAckTotal

int verifyAckCount

int verifyAckedRequests = 0

RestartRequest * storedRequest

int _falseRestart = 0

int _lockNewTicket = 0

For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.

int onGoingLoadBalancing = 0

For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.

void * centralLb

void(* resumeLbFnPtr )(void *)

int _receiveMlogLocationHandlerIdx

int _receiveMigrationNoticeHandlerIdx

int _receiveMigrationNoticeAckHandlerIdx

int _checkpointBarrierHandlerIdx

int _checkpointBarrierAckHandlerIdx

CkVec< MigrationRecord > migratedNoticeList

CkVec< RetainedMigratedObject * > retainedObjectList

int donotCountMigration = 0

int countLBMigratedAway = 0

int countLBToMigrate = 0

int migrationDoneCalled = 0

int checkpointBarrierCount = 0

int globalResumeCount = 0

CkGroupID globalLBID

int restartDecisionNumber = -1

double lastCompletedAlarm = 0

double lastRestart = 0

int _receiveLocationHandlerIdx

int calledRetryTicketRequest = 0

This method is used to retry the ticket requests that had been queued up earlier.

CkVec< TProcessedLog > processedTicketLog

double totalSearchRestoredTime = 0

double totalSearchRestoredCount = 0

Function Documentation

const char* idx2str ( const CkArrayIndex & ind )

const char* idx2str ( const ArrayElement * el )

void getGlobalStep ( CkGroupID gID )

bool fault_aware ( CkObjID & recver )

void sendCheckpointData ( int mode )

Sends the checkpoint to its buddy.

The mode distinguishes between the two cases: MLOG_RESTARTED: sending the checkpoint to a team member that did not crash but is restarting. MLOG_CRASHED: sending the checkpoint to the processor that crashed.

void createObjIDList	(	void *	data,
		ChareMlogData *	mlogData
	)

bool isLocal ( int destPE ) [inline]

Determines if the message is local or not.

A message is local if: 1) Both the destination and origin are the same PE.

bool isTeamLocal ( int destPE ) [inline]

Determines if the message is group local or not.

A message is group local if: 1) They belong to the same group in the group-based message logging.

void readKillFile ( )

CpvDeclare	(	Chare *	,
		_currentObj
	)

CpvDeclare	(	CkQ< LocalMessageLog > *	,
		_localMessageLog
	)

CpvDeclare	(	CkQ< TicketRequest * > *	,
		_delayedTicketRequests
	)

CpvDeclare	(	StoredCheckpoint *	,
		_storedCheckpointData
	)

CpvDeclare	(	CkQ< MlogEntry * > *	,
		_delayedLocalTicketRequests
	)

CpvDeclare	(	Queue	,
		_outOfOrderMessageQueue
	)

CpvDeclare	(	CkQ< LocalMessageLog > *	,
		_bufferedLocalMessageLogs
	)

CpvDeclare	(	char **	,
		_bufferedTicketRequests
	)

CpvDeclare	(	int *	,
		_numBufferedTicketRequests
	)

CpvDeclare	(	char *	,
		_bufferTicketReply
	)

void setTeamRecovery	(	void *	data,
		ChareMlogData *	mlogData
	)

Turns on the flag for team recovery that selectively restores particular metadata information.

void unsetTeamRecovery	(	void *	data,
		ChareMlogData *	mlogData
	)

Turns off the flag for team recovery.

void _messageLoggingInit ( )

Initialize message logging data structures and register handlers.

Definition at line 191 of file ckmessagelogging.C.

References _bufferedLocalMessageAckHandler(), _bufferedLocalMessageCopyHandler(), _bufferedTicketHandler(), _bufferedTicketRequestHandler(), _checkpointAckHandler(), _checkpointBarrierAckHandler(), _checkpointBarrierHandler(), _checkpointRequestHandler(), _distributedLocationHandler(), _dummyMigrationHandler(), _getCheckpointHandler(), _getGlobalStepHandler(), _getRestartCheckpointHandler(), _localMessageAckHandler(), _localMessageCopyHandler(), _pingHandler(), _receivedDetDataHandler(), _receivedTNDataHandler(), _receiveLocationHandler(), _receiveMigrationNoticeAckHandler(), _receiveMigrationNoticeHandler(), _receiveMlogLocationHandler(), _recvCheckpointHandler(), _recvGlobalStepHandler(), _recvRestartCheckpointHandler(), _removeDeterminantsHandler(), _removeProcessedLogHandler(), _resendMessagesHandler(), _resendReplyHandler(), _restartHandler(), _sendDetsHandler(), _sendDetsReplyHandler(), _storeCheckpointHandler(), _storeDeterminantsHandler(), _ticketHandler(), _ticketRequestHandler(), _updateHomeAckHandler(), _updateHomeRequestHandler(), _verifyAckHandler(), _verifyAckRequestHandler(), CcdCallFnAfter(), checkBufferedLocalMessageCopy(), Converse::CkMyPe(), Converse::CkNumPes(), CmiAlloc(), CmiMyPe(), CmiWallTimer(), CqsCreate(), printf(), retryTicketRequest(), totalMsgsSize, and traceRegisterUserEvent().

void killLocal	(	void *	_dummy,
		double	curWallTime
	)

void sendTicketGroupRequest	(	envelope *	env,
		int	destPE,
		int	_infoIdx
	)

Definition at line 355 of file ckmessagelogging.C.

References CkCopyMsg(), Converse::CkMyPe(), CmiMyPe(), CkObjID::data, EnvToUsr(), generateCommonTicketRequest(), envelope::getGroupNum(), _ObjectID::group, _ObjectID::id, _ObjectID::onPE, printf(), envelope::sender, sendTicketGroupRequest(), envelope::SN, envelope::TN, CkObjID::type, TypeGroup, TypeInvalid, and UsrToEnv().

Referenced by _sendMsgBranch(), and sendTicketGroupRequest().

void sendTicketNodeGroupRequest	(	envelope *	env,
		int	destNode,
		int	_infoIdx
	)

Definition at line 383 of file ckmessagelogging.C.

References CkCopyMsg(), Converse::CkMyPe(), CkObjID::data, EnvToUsr(), generateCommonTicketRequest(), envelope::getGroupNum(), _ObjectID::group, _ObjectID::id, _ObjectID::onPE, printf(), envelope::sender, sendTicketNodeGroupRequest(), envelope::SN, envelope::TN, CkObjID::type, TypeInvalid, TypeNodeGroup, and UsrToEnv().

Referenced by _sendMsgNodeBranch(), and sendTicketNodeGroupRequest().

void sendTicketArrayRequest	(	envelope *	env,
		int	destPE,
		int	_infoIdx
	)

Definition at line 407 of file ckmessagelogging.C.

References _ObjectID::array, Converse::CkMyPe(), CkObjID::data, generateCommonTicketRequest(), envelope::getsetArrayIndex(), envelope::getsetArrayMgr(), _ObjectID::s_array::id, _ObjectID::s_array::idx, printf(), CkObjID::toString(), CkObjID::type, and TypeArray.

Referenced by CkArrayManagerDeliver().

void generateCommonTicketRequest	(	CkObjID &	recver,
		envelope *	_env,
		int	destPE,
		int	_infoIdx
	)

A method to generate the actual ticket requests for groups, nodegroups or arrays.

Definition at line 425 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), copyEnvelope(), generalCldEnqueue(), SNToTicket::get(), CkHashtableT< KEY, OBJ >::get(), CkObjID::getObject(), isLocal(), isTeamLocal(), Chare::mlogData, ChareMlogData::nextSN(), printf(), envelope::recver, envelope::sender, sendTicketRequest(), envelope::SN, ChareMlogData::teamTable, ticketLogLocalMessage(), Ticket::TN, CkObjID::toString(), CkObjID::type, and TypeInvalid.

Referenced by sendTicketArrayRequest(), sendTicketGroupRequest(), and sendTicketNodeGroupRequest().

void sendTicketRequest	(	CkObjID &	sender,
		CkObjID &	recver,
		int	destPE,
		MlogEntry *	entry,
		MCount	SN,
		MCount	TN,
		int	resend
	)

Method that does the actual send by creating a ticket request filling it up and sending it.

Definition at line 528 of file ckmessagelogging.C.

References ChareMlogData::addLogEntry(), CcdCallFnAfter(), checkBufferedTicketRequests(), Converse::CkMyPe(), CmiAlloc(), CmiMemoryCheck(), CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), MlogEntry::destPE, MlogEntry::env, envelope::freeMsg, CkObjID::getObject(), envelope::getTotalsize(), isTeamLocal(), TicketRequest::logEntry, Chare::mlogData, MLOGFT_totalLogSize, MLOGFT_totalMessages, printf(), TicketRequest::recver, envelope::recver, sendBufferedTicketRequests(), TicketRequest::sender, envelope::sender, TicketRequest::senderPE, TicketRequest::SN, envelope::SN, TicketRequest::TN, and CkObjID::toString().

Referenced by generateCommonTicketRequest(), and resendMessageForChare().

void sendBufferedTicketRequests ( int destPE )

Send the ticket requests buffered for processor PE.

Definition at line 598 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiMemoryCheck(), Converse::CmiSyncSend(), BufferedLocalLogHeader::numberLogs, and printf().

Referenced by checkBufferedTicketRequests(), and sendTicketRequest().

void checkBufferedTicketRequests	(	void *	_destPE,
		double	curWallTime
	)

Definition at line 617 of file ckmessagelogging.C.

References CmiMemoryCheck(), and sendBufferedTicketRequests().

Referenced by sendTicketRequest().

void ticketLogLocalMessage ( MlogEntry * entry )

Gets a ticket for a local message and then sends a copy to the buddy.

This method is always in the main thread(not interrupt).. so it should never find itself locked out of a newTicket.

Definition at line 632 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), CmiMemoryCheck(), CmiMyPe(), MlogEntry::destPE, MlogEntry::env, CkObjID::getObject(), ChareMlogData::mapTable, Chare::mlogData, ChareMlogData::next_ticket(), CkHashtable::numObjects(), printf(), envelope::recver, ChareMlogData::searchRestoredLocalQ(), envelope::sender, sendLocalMessageCopy(), envelope::SN, envelope::TN, Ticket::TN, CkObjID::toString(), and MlogEntry::unackedLocal.

Referenced by generateCommonTicketRequest(), and retryTicketRequest().

void sendLocalMessageCopy ( MlogEntry * entry )

Sends the metadata of a local message to its buddy.

Definition at line 689 of file ckmessagelogging.C.

References CcdCallFnAfter(), checkBufferedLocalMessageCopy(), Converse::CkMyPe(), CmiMemoryCheck(), Converse::CmiSyncSend(), LocalMessageLog::entry, MlogEntry::env, getCheckPointPE(), length, printf(), envelope::recver, LocalMessageLog::recver, sendBufferedLocalMessageCopy(), envelope::sender, LocalMessageLog::sender, LocalMessageLog::senderPE, envelope::SN, LocalMessageLog::SN, envelope::TN, LocalMessageLog::TN, and CkObjID::toString().

Referenced by _ticketHandler(), resendMessageForChare(), and ticketLogLocalMessage().

void sendBufferedLocalMessageCopy ( )

Definition at line 723 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiAlloc(), CmiMemoryCheck(), Converse::CmiSyncSendAndFree(), countBuffered, getCheckPointPE(), BufferedLocalLogHeader::numberLogs, printf(), and ptr.

Referenced by checkBufferedLocalMessageCopy(), and sendLocalMessageCopy().

void checkBufferedLocalMessageCopy	(	void *	_dummy,
		double	curWallTime
	)

Definition at line 752 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiAbort(), CmiMemoryCheck(), length, printf(), and sendBufferedLocalMessageCopy().

Referenced by _messageLoggingInit(), and sendLocalMessageCopy().

bool _processTicketRequest	(	TicketRequest *	ticketRequest,
		TicketReply *	reply
	)			`[inline]`

Process the ticket request.

If it is processed and a reply is being sent by this processor return true else return false. If a reply buffer is specified put the reply into that else send the reply

Definition at line 847 of file ckmessagelogging.C.

References _ObjectID::array, CkArrayID::ckLocalBranch(), Converse::CkMyPe(), CmiAlloc(), CmiMemoryCheck(), Converse::CmiSyncSend(), CmiWallTimer(), CkObjID::data, CkArray::getLocMgr(), CkObjID::getObject(), CkObjID::guessPE(), CkLocMgr::homePe(), _ObjectID::s_array::id, _ObjectID::s_array::idx, ChareMlogData::mapTable, Chare::mlogData, ChareMlogData::next_ticket(), CkHashtable::numObjects(), printf(), TicketRequest::recver, TicketReply::recverPE, TicketReply::request, ChareMlogData::searchRestoredLocalQ(), TicketRequest::sender, TicketRequest::senderPE, TicketRequest::SN, Ticket::state, TicketReply::ticket, Ticket::TN, TicketRequest::TN, CkObjID::toString(), ChareMlogData::tProcessed, CkObjID::type, TypeArray, and ChareMlogData::verifyTicket().

Referenced by _bufferedTicketRequestHandler(), _ticketRequestHandler(), and retryTicketRequest().

void _ticketRequestHandler ( TicketRequest * ticketRequest ) [inline]

If there are any delayed requests, process them first before processing this request.

Definition at line 778 of file ckmessagelogging.C.

References _processTicketRequest(), _startTime, Converse::CkMyPe(), CmiFree(), length, printf(), and retryTicketRequest().

void _bufferedTicketRequestHandler ( BufferedTicketRequestHeader * recvdHeader )

Handler used for dealing with a bunch of ticket requests from one processor.

The replies are also bunched together Does not use _ticketRequestHandler

Definition at line 792 of file ckmessagelogging.C.

References _processTicketRequest(), _startTime, Converse::CkMyPe(), CmiFree(), CmiMemoryCheck(), Converse::CmiSyncSend(), length, msg, BufferedLocalLogHeader::numberLogs, printf(), ptr, and retryTicketRequest().

Referenced by _messageLoggingInit().

void _ticketHandler ( TicketReply * ticketReply ) [inline]

This function handles the ticket received after a request.

Definition at line 965 of file ckmessagelogging.C.

References MlogEntry::_infoIdx, _startTime, Converse::CkMyPe(), CmiAbort(), CmiFree(), CmiMemoryCheck(), Converse::CmiSyncSend(), Converse::CmiSyncSendAndFree(), CmiWallTimer(), MlogEntry::env, generalCldEnqueue(), CkHashtableT< KEY, OBJ >::get(), ChareMlogData::getMlog(), CkObjID::getObject(), envelope::getTotalsize(), CkObjID::guessPE(), isTeamLocal(), CkQ< T >::length(), TicketRequest::logEntry, Chare::mlogData, printf(), SNToTicket::put(), CkHashtableTslow< KEY, OBJ >::put(), envelope::recver, TicketRequest::recver, TicketReply::recverPE, TicketReply::request, envelope::sender, TicketRequest::sender, sendLocalMessageCopy(), envelope::setSrcPe(), envelope::SN, TicketRequest::SN, Ticket::state, ChareMlogData::teamTable, TicketReply::ticket, envelope::TN, Ticket::TN, CkObjID::toString(), and CkObjID::updatePosition().

Referenced by _bufferedTicketHandler(), and _messageLoggingInit().

void _bufferedTicketHandler ( BufferedTicketRequestHeader * recvdHeader )

Message to handle the bunch of tickets that we get from one processor.

We send the tickets to be handled one at a time

Definition at line 1066 of file ckmessagelogging.C.

References _startTime, _ticketHandler(), CmiFree(), CmiMemoryCheck(), CmiWallTimer(), msg, and BufferedLocalLogHeader::numberLogs.

Referenced by _messageLoggingInit().

void _localMessageCopyHandler ( LocalMessageLog * msgLog )

Stores the metadata of a local message from its buddy.

Definition at line 1091 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), Converse::CmiSyncSend(), CmiWallTimer(), LocalMessageLog::entry, LocalMessageLogAck::entry, printf(), LocalMessageLog::recver, LocalMessageLog::sender, LocalMessageLog::senderPE, LocalMessageLog::TN, and CkObjID::toString().

Referenced by _messageLoggingInit().

void _bufferedLocalMessageCopyHandler	(	BufferedLocalLogHeader *	recvdHeader,
		int	freeHeader
	)

Definition at line 1110 of file ckmessagelogging.C.

References _startTime, CmiAlloc(), CmiFree(), CmiMemoryCheck(), CmiMyPe(), Converse::CmiSyncSendAndFree(), LocalMessageLog::entry, LocalMessageLogAck::entry, getCheckPointPE(), msg, BufferedLocalLogHeader::numberLogs, printf(), ptr, and LocalMessageLog::senderPE.

Referenced by _bufferedLocalMessageAckHandler(), and _messageLoggingInit().

void _localMessageAckHandler ( LocalMessageLogAck * ack )

Definition at line 1182 of file ckmessagelogging.C.

References MlogEntry::_infoIdx, _skipCldEnqueue(), _startTime, CkExit(), Converse::CkMyPe(), CmiFree(), CmiMemoryCheck(), CmiMyPe(), MlogEntry::destPE, LocalMessageLogAck::entry, MlogEntry::env, envelope::localMlogEntry, printf(), envelope::recver, envelope::sender, envelope::SN, envelope::TN, CkObjID::toString(), and MlogEntry::unackedLocal.

Referenced by _bufferedLocalMessageAckHandler(), and _messageLoggingInit().

void _bufferedLocalMessageAckHandler ( BufferedLocalLogHeader * recvdHeader )

Definition at line 1237 of file ckmessagelogging.C.

References _bufferedLocalMessageCopyHandler(), _localMessageAckHandler(), _startTime, CmiFree(), CmiMemoryCheck(), CmiMyPe(), msg, BufferedLocalLogHeader::numberLogs, and printf().

Referenced by _messageLoggingInit().

int preProcessReceivedMessage	(	envelope *	env,
		Chare **	objPointer,
		MlogEntry **	logEntryPointer
	)

void postProcessReceivedMessage	(	Chare *	obj,
		CkObjID &	sender,
		MCount	SN,
		MlogEntry *	entry
	)

Updates a few variables once a message has been processed.

Definition at line 1350 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiMemoryCheck(), MlogEntry::env, CkObjID::guessPE(), Chare::mlogData, and ChareMlogData::tProcessed.

void generalCldEnqueue	(	int	destPE,
		envelope *	env,
		int	_infoIdx
	)

Definition at line 1370 of file ckmessagelogging.C.

References _noCldNodeEnqueue(), _skipCldEnqueue(), envelope::recver, CkObjID::type, and TypeNodeGroup.

void retryTicketRequestTimer	(	void *	_dummy,
		double	_time
	)

Definition at line 1391 of file ckmessagelogging.C.

References retryTicketRequest().

Referenced by retryTicketRequest().

void retryTicketRequest	(	void *	_dummy,
		double	curWallTime
	)

Definition at line 1396 of file ckmessagelogging.C.

References _processTicketRequest(), CcdCallFnAfter(), Converse::CkMyPe(), CmiFree(), CmiMemoryCheck(), CmiWallTimer(), CqsLength(), length, printf(), TicketRequest::recver, retryTicketRequestTimer(), TicketRequest::sender, TicketRequest::SN, ticketLogLocalMessage(), and CkObjID::toString().

Referenced by _bufferedTicketRequestHandler(), _messageLoggingInit(), _ticketRequestHandler(), and retryTicketRequestTimer().

void _pingHandler ( CkPingMsg * msg )

Definition at line 1438 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiFree(), RestartRequest::PE, and printf().

void buildProcessedTicketLog	(	void *	data,
		ChareMlogData *	mlogData
	)

void clearUpMigratedRetainedLists ( int PE )

void checkpointAlarm	(	void *	_dummy,
		double	curWallTime
	)

Definition at line 1453 of file ckmessagelogging.C.

References CcdCallFnAfter(), checkpointAlarm(), Converse::CkMyPe(), Converse::CmiSyncBroadcastAll(), diff, RestartRequest::PE, and printf().

void _checkpointRequestHandler ( CheckpointRequest * request )

Definition at line 1470 of file ckmessagelogging.C.

References CmiWallTimer(), and startMlogCheckpoint().

void startMlogCheckpoint	(	void *	_dummy,
		double	curWallTime
	)

Starts the checkpoint phase after migration.

Definition at line 1474 of file ckmessagelogging.C.

References _startTime, buildProcessedTicketLog(), CcdCallFnAfter(), checkpointAlarm(), Converse::CkMyPe(), CkPupGroupData(), CkPupNodeGroupData(), CkPupROData(), CmiAlloc(), CmiMemoryCheck(), CmiMyPe(), Converse::CmiSyncSendAndFree(), CmiTimer(), CmiTrue, CmiWallTimer(), CheckPointDataMsg::dataSize, dataSize, forAllCharesDo(), getCheckPointPE(), msg, CheckPointDataMsg::PE, printf(), pupArrayElementsSkip(), CkVec< T >::removeAll(), PUP::sizer::size(), and traceUserBracketEvent().

void pupArrayElementsSkip	(	PUP::er &	p,
		CmiBool	create,
		MigrationRecord *	listToSkip,
		int	listsize
	)

Pups all the array elements in this processor.

Definition at line 1563 of file ckmessagelogging.C.

References CkCountArrayElements(), Converse::CkMyPe(), CmiMyPe(), CkLocMgr::homePe(), _ckGroupID::idx, idx, MigrationRecord::idx, idx2str(), informLocationHome(), PUP::er::isUnpacking(), CkLocMgr::numLocalElements(), CkArrayIndex::print(), printf(), and CkLocMgr::resume().

void writeCheckpointToDisk	(	int	size,
		char *	chkpt
	)

Definition at line 1619 of file ckmessagelogging.C.

References checkpointDirectory, Converse::CkMyPe(), and write().

void _storeCheckpointHandler ( char * msg )

Definition at line 1637 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), CmiFree(), CmiMyPe(), Converse::CmiSyncSend(), CheckPointDataMsg::dataSize, CheckPointDataMsg::PE, printf(), CkVec< T >::remove(), CkVec< T >::size(), traceUserBracketEvent(), and writeCheckpointToDisk().

void sendRemoveLogRequests ( )

Definition at line 1693 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), Converse::CkNumPes(), clearUpMigratedRetainedLists(), CmiAlloc(), CmiFree(), CmiMemoryCheck(), CmiMyPe(), Converse::CmiSyncSend(), CkVec< T >::getVec(), ResendRequest::numberObjects, ResendRequest::PE, CkVec< T >::size(), and traceUserBracketEvent().

void _checkpointAckHandler ( CheckPointAck * ackMsg )

Definition at line 1723 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiFree(), CmiMemoryCheck(), CheckPointDataMsg::dataSize, finishedCheckpointLoadBalancing(), CheckPointDataMsg::PE, printf(), and sendRemoveLogRequests().

void removeProcessedLogs	(	void *	_data,
		ChareMlogData *	mlogData
	)

void _removeProcessedLogHandler ( char * requestMsg )

Removes messages in the log according to the received ticket numbers.

Definition at line 1774 of file ckmessagelogging.C.

References Converse::CkMyPe(), clearUpMigratedRetainedLists(), CmiAbort(), CmiFree(), CmiMemoryCheck(), CkQ< T >::enq(), fault_aware(), forAllCharesDo(), getCheckPointPE(), CkQ< T >::length(), list, ResendRequest::numberObjects, ResendRequest::PE, populateDeterminantTable(), printf(), TProcessedLog::recver, LocalMessageLog::recver, removeProcessedLogs(), LocalMessageLog::TN, TProcessedLog::tProcessed, and traceUserBracketEvent().

void CkMlogRestart	(	const char *	dummy,
		CkArgMsg *	dummyMsg
	)

Function for restarting the crashed processor.

It sets the restart flag and contacts the buddy processor to get the latest checkpoint.

Definition at line 1862 of file ckmessagelogging.C.

References Converse::CkMyPe(), Converse::CkNumPes(), Converse::CmiSyncSend(), CmiWallTimer(), getCheckPointPE(), msg, and RestartRequest::PE.

void _restartHandler ( RestartRequest * restartMsg )

Function to restart this processor.

The handler is invoked by a member of its same team in message logging.

Definition at line 1892 of file ckmessagelogging.C.

References Converse::CkMyPe(), Converse::CmiSyncSend(), CmiWallTimer(), getCheckPointPE(), msg, and RestartRequest::PE.

void _getRestartCheckpointHandler ( RestartRequest * restartMsg )

Gets the stored checkpoint but calls another function in the sender.

Definition at line 1923 of file ckmessagelogging.C.

References CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), VerifyAckMsg::fromPE, idx, idx2str(), VerifyAckMsg::index, VerifyAckMsg::migRecord, msg, StoredCheckpoint::PE, RestartRequest::PE, sendCheckpointData(), and CkVec< T >::size().

void _recvRestartCheckpointHandler ( char * _restartData )

Receives the checkpoint coming from its buddy.

This is the case of restart for one team member that did not crash.

Definition at line 1962 of file ckmessagelogging.C.

References _resendMessagesHandler(), ChareMlogData::addToRestoredLocalQ(), RestartProcessorData::checkPointSize, Converse::CkMyPe(), Converse::CkNumPes(), CkPupGroupData(), CkPupNodeGroupData(), CkPupROData(), CmiAlloc(), CmiFalse, CmiFree(), CmiMyPe(), Converse::CmiSyncSend(), CmiWallTimer(), createObjIDList(), forAllCharesDo(), CkObjID::getObject(), initializeRestart(), RestartProcessorData::lbGroupID, Chare::mlogData, ResendRequest::numberObjects, RestartProcessorData::numLocalMessages, RestartProcessorData::numMigratedAwayElements, ResendRequest::PE, RestartProcessorData::PE, printf(), pupArrayElementsSkip(), CkVec< T >::push_back(), ChareMlogData::receivedTNs, CentralLB::ReceiveDummyMigration(), LocalMessageLog::recver, RestartProcessorData::restartWallTime, LocalMessageLog::sender, setTeamRecovery(), PUP::mem::size(), sleep(), LocalMessageLog::SN, sortRestoredLocalMsgLog(), LocalMessageLog::TN, CkObjID::toString(), and unsetTeamRecovery().

void CkMlogRestartDouble	(	void *	,
		double
	)

Definition at line 2086 of file ckmessagelogging.C.

References CkMlogRestart().

void CkMlogRestartLocal ( )

Definition at line 2091 of file ckmessagelogging.C.

References CkMlogRestart().

void readCheckpointFromDisk	(	int	size,
		char *	buf
	)

Definition at line 2096 of file ckmessagelogging.C.

References checkpointDirectory, and Converse::CkMyPe().

void _getCheckpointHandler ( RestartRequest * restartMsg )

Gets the stored checkpoint for its buddy processor.

Definition at line 2114 of file ckmessagelogging.C.

References CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), VerifyAckMsg::fromPE, idx, idx2str(), VerifyAckMsg::index, VerifyAckMsg::migRecord, msg, StoredCheckpoint::PE, RestartRequest::PE, sendCheckpointData(), and CkVec< T >::size().

void _verifyAckRequestHandler ( VerifyAckMsg * verifyRequest )

void _verifyAckHandler ( VerifyAckMsg * verifyReply )

Definition at line 2179 of file ckmessagelogging.C.

References CmiMyPe(), CmiPrintf(), idx, idx2str(), VerifyAckMsg::index, VerifyAckMsg::migRecord, and sendCheckpointData().

void _recvCheckpointHandler ( char * _restartData )

Receives the checkpoint data from its buddy, restores the state of all the objects and asks everyone else to update its home.

Definition at line 2288 of file ckmessagelogging.C.

References _initDone(), ChareMlogData::addToRestoredLocalQ(), RestartProcessorData::checkPointSize, Converse::CkMyPe(), Converse::CkNumPes(), CkPupGroupData(), CkPupNodeGroupData(), CkPupROData(), CmiAlloc(), CmiFree(), CmiMyPe(), Converse::CmiSyncSend(), CmiTrue, CmiWallTimer(), createObjIDList(), forAllCharesDo(), getGlobalStep(), CkObjID::getObject(), initializeRestart(), RestartProcessorData::lbGroupID, Chare::mlogData, ResendRequest::numberObjects, RestartProcessorData::numLocalMessages, RestartProcessorData::numMigratedAwayElements, RestartRequest::PE, ResendRequest::PE, RestartProcessorData::PE, printf(), pupArrayElementsSkip(), CkVec< T >::push_back(), ChareMlogData::receivedTNs, LocalMessageLog::recver, RestartProcessorData::restartWallTime, LocalMessageLog::sender, PUP::mem::size(), LocalMessageLog::SN, sortRestoredLocalMsgLog(), LocalMessageLog::TN, and CkObjID::toString().

void _updateHomeAckHandler ( RestartRequest * updateHomeAck )

Receives the updateHome ACKs from all other processors.

Once everybody has replied, it sends a request to resend the logged messages.

Definition at line 2367 of file ckmessagelogging.C.

References _resendMessagesHandler(), Converse::CkMyPe(), Converse::CkNumPes(), CmiAlloc(), CmiFree(), Converse::CmiSyncSend(), CmiWallTimer(), createObjIDList(), distributeRestartedObjects(), fastRecovery, forAllCharesDo(), ResendRequest::numberObjects, parallelRestart, ResendRequest::PE, printf(), CentralLB::ReceiveDummyMigration(), and sleep().

void initializeRestart	(	void *	data,
		ChareMlogData *	mlogData
	)

Initializes variables and flags for restarting procedure.

Definition at line 2420 of file ckmessagelogging.C.

References CkHashtable::empty(), ChareMlogData::mapTable, ChareMlogData::receivedTNs, CkVec< T >::removeAll(), ChareMlogData::resendReplyRecvd, ChareMlogData::restartFlag, and ChareMlogData::restoredLocalMsgLog.

void updateHomePE	(	void *	data,
		ChareMlogData *	mlogData
	)

Updates the homePe of chare array elements.

Definition at line 2431 of file ckmessagelogging.C.

References _ObjectID::array, CkArrayID::ckLocalBranch(), Converse::CkMyPe(), CkObjID::data, CkLocMgr::getGroupID(), CkArray::getLocMgr(), CkLocMgr::homePe(), _ObjectID::s_array::id, _ObjectID::s_array::idx, informLocationHome(), ChareMlogData::objID, RestartRequest::PE, CkArrayIndex::print(), printf(), CkObjID::type, and TypeArray.

void _updateHomeRequestHandler ( RestartRequest * updateRequest )

Updates the homePe for all chares in this processor.

Definition at line 2455 of file ckmessagelogging.C.

References CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), Converse::CmiSyncSendAndFree(), forAllCharesDo(), getCheckPointPE(), MigrationNotice::migRecord, RestartRequest::PE, MigrationNotice::record, CkVec< T >::size(), startMlogCheckpoint(), and updateHomePE().

void fillTicketForChare	(	void *	data,
		ChareMlogData *	mlogData
	)

Fills up the ticket vector for each chare.

Definition at line 2484 of file ckmessagelogging.C.

References SNToTicket::get(), SNToTicket::getFinishSN(), SNToTicket::getStartSN(), CkHashtable::iterator(), ResendData::listObjects, ResendData::maxTickets, CkHashtableIterator::next(), ResendData::numberObjects, ResendData::PE, CkVec< T >::push_back(), ChareMlogData::teamTable, ResendData::ticketVecs, and Ticket::TN.

void resendMessageForChare	(	void *	data,
		ChareMlogData *	mlogData
	)

void _resendMessagesHandler ( char * msg )

Resends the messages since the last checkpoint to the list of objects included in the request.

Resends messages since last checkpoint to the list of objects included in the request.

Definition at line 2628 of file ckmessagelogging.C.

References _receiveMlogLocationHandler(), _ObjectID::array, CkArrayID::ckLocalBranch(), Converse::CkMyPe(), CmiAlloc(), CmiFree(), CmiMemoryCheck(), CmiMyPe(), CmiPrintf(), Converse::CmiSyncSendAndFree(), CmiWallTimer(), CkObjID::data, CkLocMgr::elementRec(), fillTicketForChare(), forAllCharesDo(), CkLocMgr::getGroupID(), CkArray::getLocMgr(), CkVec< T >::getVec(), CkLocMgr::homePe(), _ObjectID::s_array::id, _ObjectID::s_array::idx, idx2str(), informLocationHome(), isTeamLocal(), ResendData::listObjects, CkLocRec::local, ResendData::maxTickets, CkLocMgr::migratableList(), ResendRequest::numberObjects, ResendData::numberObjects, ResendRequest::PE, ResendData::PE, printf(), TProcessedLog::recver, resendMessageForChare(), sendDummyMigration(), CkVec< T >::size(), ResendData::ticketVecs, ChareMlogData::toResumeOrNot, CkLocRec::type(), CkObjID::type, and TypeArray.

void sortVec ( CkVec< MCount > * TNvec )

int searchVec	(	CkVec< MCount > *	TNVec,
		MCount	searchTN
	)

void _resendReplyHandler ( char * msg )

Receives the tickets assigned to message to other objects.

need to rewrite this method to deal with parallel restart

Definition at line 2763 of file ckmessagelogging.C.

References CmiAlloc(), CmiMyPe(), Converse::CmiSyncSendAndFree(), ResendRequest::numberObjects, ReceivedTNData::numTNs, ResendRequest::PE, printf(), processReceivedTN(), and ReceivedTNData::recver.

Referenced by _messageLoggingInit().

void _receivedTNDataHandler ( ReceivedTNData * msg )

Receives a list of TNs coming from the home PE of a migrated object (parallel restart).

Definition at line 2802 of file ckmessagelogging.C.

References CmiFree(), CmiMyPe(), Converse::CmiSyncSendAndFree(), CkObjID::getObject(), CkObjID::guessPE(), Chare::mlogData, ReceivedTNData::numTNs, ChareMlogData::objID, printf(), processReceivedTN(), ReceivedTNData::recver, and CkObjID::toString().

void processReceivedTN	(	Chare *	obj,
		int	listSize,
		MCount *	listTNs
	)

Processes the received list of tickets from a particular PE.

Definition at line 2819 of file ckmessagelogging.C.

References Converse::CkMyPe(), Converse::CkNumPes(), CmiMyPe(), CmiWallTimer(), ChareMlogData::currentHoles, maxVec(), Chare::mlogData, ChareMlogData::numberHoles, ChareMlogData::objID, objString, printf(), CkVec< T >::push_back(), ChareMlogData::receivedTNs, ChareMlogData::resendReplyRecvd, ChareMlogData::restartFlag, searchVec(), CkVec< T >::size(), sortVec(), ChareMlogData::tCount, ChareMlogData::ticketHoles, CkObjID::toString(), and ChareMlogData::tProcessed.

void distributeRestartedObjects ( )

Distributes objects to accelerate recovery after a failure.

Definition at line 3022 of file ckmessagelogging.C.

References Converse::CkMyPe().

void _distributedLocationHandler ( char * receivedMsg )

Handler to update information about an object just received.

Definition at line 3029 of file ckmessagelogging.C.

References Converse::CkMyPe(), CmiTrue, CkLocMgr::elementRec(), CkLocMgr::homePe(), idx, informLocationHome(), CkLocRec::local, CkLocMgr::migratableList(), CkArrayIndex::print(), printf(), CkLocMgr::resume(), CkVec< T >::size(), ChareMlogData::toResumeOrNot, and CkLocRec::type().

void sendDummyMigration	(	int	restartPE,
		CkGroupID	lbID,
		CkGroupID	locMgrID,
		CkArrayIndex &	idx,
		int	locationPE
	)

this method is used to send messages to a restarted processor to tell it that a particular expected object is not going to get to it

Definition at line 3063 of file ckmessagelogging.C.

References Converse::CmiSyncSend(), DummyMigrationMsg::flag, DummyMigrationMsg::idx, DummyMigrationMsg::lbID, DummyMigrationMsg::locationPE, and DummyMigrationMsg::mgrID.

void _dummyMigrationHandler ( DummyMigrationMsg * msg )

this handler is used to process a dummy migration msg.

it looks up the load balancer and calls migrated for it

Definition at line 3096 of file ckmessagelogging.C.

References CmiFree(), CmiMyPe(), CmiPrintf(), DummyMigrationMsg::count, DummyMigrationMsg::flag, DummyMigrationMsg::idx, _ckGroupID::idx, idx2str(), DummyMigrationMsg::lbID, DummyMigrationMsg::locationPE, DummyMigrationMsg::mgrID, and CentralLB::Migrated().

void forAllCharesDo	(	MlogFn	fnPointer,
		void *	data
	)

Map function pointed by fnPointer over all the chares living in this processor.

Definition at line 3147 of file ckmessagelogging.C.

References caller, and Chare::mlogData.

void startLoadBalancingMlog	(	void()(void )	_fnPtr,
		void *	_centralLb
	)

Load Balancing.

Definition at line 3180 of file ckmessagelogging.C.

References centralLb, CmiMyPe(), CmiWallTimer(), printf(), resumeLbFnPtr, and startMlogCheckpoint().

void finishedCheckpointLoadBalancing ( )

Definition at line 3193 of file ckmessagelogging.C.

References CmiMyPe(), Converse::CmiSyncSend(), and printf().

void sendMlogLocation	(	int	targetPE,
		envelope *	env
	)

void _receiveMigrationNoticeHandler ( MigrationNotice * msg )

Definition at line 3257 of file ckmessagelogging.C.

References MigrationRecord::ackFrom, MigrationRecord::ackTo, Converse::CmiSyncSend(), getCheckPointPE(), MigrationNotice::migRecord, CkVec< T >::push_back(), MigrationNotice::record, and MigrationNoticeAck::record.

void _receiveMigrationNoticeAckHandler ( MigrationNoticeAck * msg )

Definition at line 3267 of file ckmessagelogging.C.

References RetainedMigratedObject::acked, CmiMyPe(), Converse::CmiSyncSend(), CmiWallTimer(), MigrationRecord::gID, CkLocMgr::homePe(), MigrationRecord::idx, informLocationHome(), RetainedMigratedObject::migRecord, RetainedMigratedObject::msg, printf(), MigrationNoticeAck::record, RetainedMigratedObject::size, startMlogCheckpoint(), and MigrationRecord::toPE.

void _receiveMlogLocationHandler ( void * buf )

Definition at line 3287 of file ckmessagelogging.C.

References Converse::CkMyPe(), CkUnpackMessage(), EnvToUsr(), envelope::getTotalsize(), CkArrayElementMigrateMessage::gid, _ckGroupID::idx, CkLocMgr::immigrate(), and printf().

void resumeFromSyncRestart	(	void *	data,
		ChareMlogData *	mlogData
	)

Definition at line 3301 of file ckmessagelogging.C.

void checkAndSendCheckpointBarrierAcks ( CheckpointBarrierMsg * msg ) [inline]

Definition at line 3312 of file ckmessagelogging.C.

References Converse::CmiSyncSend().

void _checkpointBarrierHandler ( CheckpointBarrierMsg * msg )

Processor 0 receives a contribution from every other processor after checkpoint.

Definition at line 3321 of file ckmessagelogging.C.

References checkAndSendCheckpointBarrierAcks(), CheckpointBarrierMsg::checkpointCount, CmiAbort(), CmiFree(), CmiMyPe(), CmiPrintf(), CheckpointBarrierMsg::fromPE, and printf().

void _checkpointBarrierAckHandler ( CheckpointBarrierMsg * msg )

Definition at line 3338 of file ckmessagelogging.C.

References centralLb, Converse::CkMyPe(), CmiFree(), CmiMyPe(), CmiPrintf(), and sendRemoveLogRequests().

void informLocationHome	(	CkGroupID	locMgrID,
		CkArrayIndex	idx,
		int	homePE,
		int	currentPE
	)

method that informs an array elements home processor of its current location It is a converse method to bypass the charm++ message logging framework

Definition at line 3351 of file ckmessagelogging.C.

References _startTime, Converse::CkMyPe(), CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), CmiWallTimer(), CurrentLocationMsg::fromPE, _ckGroupID::idx, CurrentLocationMsg::idx, idx2str(), CurrentLocationMsg::locationPE, CurrentLocationMsg::mgrID, and traceUserBracketEvent().

void _receiveLocationHandler ( CurrentLocationMsg * data )

void _getGlobalStepHandler ( LBStepMsg * msg )

Definition at line 3407 of file ckmessagelogging.C.

References CmiMyPe(), CmiPrintf(), Converse::CmiSyncSend(), LBStepMsg::fromPE, _ckGroupID::idx, LBStepMsg::lbID, BaseLB::step(), and LBStepMsg::step.

void _recvGlobalStepHandler ( LBStepMsg * msg )

Receives the global step handler from PE 0.

Definition at line 3416 of file ckmessagelogging.C.

References _sendDetsReplyHandler(), _updateHomeAckHandler(), Converse::CkMyPe(), CmiAlloc(), CmiFree(), CmiMyPe(), CmiPrintf(), ResendRequest::numberObjects, ResendRequest::PE, and LBStepMsg::step.

void sortRestoredLocalMsgLog	(	void *	_dummy,
		ChareMlogData *	mlogData
	)

Definition at line 3786 of file ckmessagelogging.C.

References ChareMlogData::sortRestoredLocalMsgLog().

Referenced by _recvCheckpointHandler(), and _recvRestartCheckpointHandler().

int getCheckPointPE ( )

Getting the pe number of the current processor's buddy.

In the team-based approach each processor might checkpoint in the next team, but currently teams are only meant to reduce memory overhead.

Definition at line 4024 of file ckmessagelogging.C.

References CmiMyPe().

envelope* copyEnvelope ( envelope * env )

Definition at line 4033 of file ckmessagelogging.C.

References CmiAlloc(), and envelope::getTotalsize().

Variable Documentation

float MLOGFT_totalLogSize = 0.0

Definition at line 44 of file ckmessagelogging.C.

float MLOGFT_totalMessages = 0.0

Definition at line 45 of file ckmessagelogging.C.

float MLOGFT_totalObjects = 0.0

Definition at line 46 of file ckmessagelogging.C.

int countHashRefs = 0

Definition at line 49 of file ckmessagelogging.C.

int countHashCollisions = 0

Definition at line 50 of file ckmessagelogging.C.

char* checkpointDirectory = "."

Definition at line 53 of file ckmessagelogging.C.

int unAckedCheckpoint = 0

Definition at line 54 of file ckmessagelogging.C.

int countLocal = 0

Definition at line 56 of file ckmessagelogging.C.

int countBuffered = 0

Definition at line 56 of file ckmessagelogging.C.

int countPiggy = 0

Definition at line 57 of file ckmessagelogging.C.

int countClearBufferedLocalCalls = 0

Definition at line 58 of file ckmessagelogging.C.

int countUpdateHomeAcks = 0

Definition at line 60 of file ckmessagelogging.C.

bool parallelRestart

Referenced by _updateHomeAckHandler().

int restartingMlogFlag = 0

Definition at line 68 of file ckmessagelogging.C.

double killTime = 0.0

Definition at line 70 of file ckmessagelogging.C.

int checkpointCount = 0

Definition at line 71 of file ckmessagelogging.C.

double adjustChkptPeriod = 0.0 [static]

Definition at line 88 of file ckmessagelogging.C.

double nextCheckpointTime = 0.0 [static]

Definition at line 89 of file ckmessagelogging.C.

double lastBufferedLocalMessageCopyTime

Definition at line 91 of file ckmessagelogging.C.

int _maxBufferedMessages

Definition at line 93 of file ckmessagelogging.C.

Referenced by _parseCommandLineOpts().

int _maxBufferedTicketRequests

Definition at line 94 of file ckmessagelogging.C.

Referenced by _parseCommandLineOpts().

int _ticketRequestHandlerIdx

Definition at line 98 of file ckmessagelogging.C.

int _ticketHandlerIdx

Definition at line 99 of file ckmessagelogging.C.

int _localMessageCopyHandlerIdx

Definition at line 100 of file ckmessagelogging.C.

int _localMessageAckHandlerIdx

Definition at line 101 of file ckmessagelogging.C.

int _pingHandlerIdx

Definition at line 102 of file ckmessagelogging.C.

int _bufferedLocalMessageCopyHandlerIdx

Definition at line 103 of file ckmessagelogging.C.

int _bufferedLocalMessageAckHandlerIdx

Definition at line 104 of file ckmessagelogging.C.

int _bufferedTicketRequestHandlerIdx

Definition at line 105 of file ckmessagelogging.C.

int _bufferedTicketHandlerIdx

Definition at line 106 of file ckmessagelogging.C.

char objString[100]

Definition at line 109 of file ckmessagelogging.C.

int _checkpointRequestHandlerIdx

Definition at line 110 of file ckmessagelogging.C.

int _storeCheckpointHandlerIdx

Definition at line 111 of file ckmessagelogging.C.

int _checkpointAckHandlerIdx

Definition at line 112 of file ckmessagelogging.C.

int _getCheckpointHandlerIdx

Definition at line 113 of file ckmessagelogging.C.

int _recvCheckpointHandlerIdx

Definition at line 114 of file ckmessagelogging.C.

int _removeProcessedLogHandlerIdx

Definition at line 115 of file ckmessagelogging.C.

int _verifyAckRequestHandlerIdx

Definition at line 117 of file ckmessagelogging.C.

int _verifyAckHandlerIdx

Definition at line 118 of file ckmessagelogging.C.

int _dummyMigrationHandlerIdx

Definition at line 119 of file ckmessagelogging.C.

int _getGlobalStepHandlerIdx

Definition at line 122 of file ckmessagelogging.C.

int _recvGlobalStepHandlerIdx

Definition at line 123 of file ckmessagelogging.C.

int _updateHomeRequestHandlerIdx

Definition at line 125 of file ckmessagelogging.C.

int _updateHomeAckHandlerIdx

Definition at line 126 of file ckmessagelogging.C.

int _resendMessagesHandlerIdx

Definition at line 127 of file ckmessagelogging.C.

int _resendReplyHandlerIdx

Definition at line 128 of file ckmessagelogging.C.

int _receivedTNDataHandlerIdx

Definition at line 129 of file ckmessagelogging.C.

int _distributedLocationHandlerIdx

Definition at line 130 of file ckmessagelogging.C.

int _restartHandlerIdx

Definition at line 133 of file ckmessagelogging.C.

int _getRestartCheckpointHandlerIdx

Definition at line 134 of file ckmessagelogging.C.

int _recvRestartCheckpointHandlerIdx

Definition at line 135 of file ckmessagelogging.C.

int verifyAckTotal

Definition at line 139 of file ckmessagelogging.C.

int verifyAckCount

Definition at line 140 of file ckmessagelogging.C.

int verifyAckedRequests = 0

Definition at line 142 of file ckmessagelogging.C.

RestartRequest* storedRequest

Definition at line 144 of file ckmessagelogging.C.

int _falseRestart = 0

Definition at line 148 of file ckmessagelogging.C.

int _lockNewTicket = 0

For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.

Definition at line 156 of file ckmessagelogging.C.

int onGoingLoadBalancing = 0

For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.

Definition at line 160 of file ckmessagelogging.C.

void* centralLb

Definition at line 161 of file ckmessagelogging.C.

void(* resumeLbFnPtr)(void *)

int _receiveMlogLocationHandlerIdx

Definition at line 163 of file ckmessagelogging.C.

int _receiveMigrationNoticeHandlerIdx

Definition at line 164 of file ckmessagelogging.C.

int _receiveMigrationNoticeAckHandlerIdx

Definition at line 165 of file ckmessagelogging.C.

int _checkpointBarrierHandlerIdx

Definition at line 166 of file ckmessagelogging.C.

int _checkpointBarrierAckHandlerIdx

Definition at line 167 of file ckmessagelogging.C.

CkVec<MigrationRecord> migratedNoticeList

Definition at line 169 of file ckmessagelogging.C.

CkVec<RetainedMigratedObject *> retainedObjectList

Definition at line 170 of file ckmessagelogging.C.

int countLBMigratedAway = 0

Definition at line 172 of file ckmessagelogging.C.

int countLBToMigrate = 0

Definition at line 173 of file ckmessagelogging.C.

int migrationDoneCalled = 0

Definition at line 174 of file ckmessagelogging.C.

int checkpointBarrierCount = 0

Definition at line 175 of file ckmessagelogging.C.

CkGroupID globalLBID

Definition at line 177 of file ckmessagelogging.C.

int restartDecisionNumber = -1

Definition at line 178 of file ckmessagelogging.C.

double lastCompletedAlarm = 0

Definition at line 181 of file ckmessagelogging.C.

double lastRestart = 0

Definition at line 182 of file ckmessagelogging.C.

int _receiveLocationHandlerIdx

Definition at line 186 of file ckmessagelogging.C.

int calledRetryTicketRequest = 0

This method is used to retry the ticket requests that had been queued up earlier.

Definition at line 1389 of file ckmessagelogging.C.

CkVec<TProcessedLog> processedTicketLog

Definition at line 1449 of file ckmessagelogging.C.

double totalSearchRestoredTime = 0

Definition at line 3753 of file ckmessagelogging.C.

double totalSearchRestoredCount = 0

Definition at line 3754 of file ckmessagelogging.C.


Data Structures
class	ElementPacker
class	ElementDistributor
class	ElementCaller
Functions
const char *	idx2str (const CkArrayIndex &ind)
const char *	idx2str (const ArrayElement *el)
void	getGlobalStep (CkGroupID gID)
bool	fault_aware (CkObjID &recver)
void	sendCheckpointData (int mode)
	Sends the checkpoint to its buddy.
void	createObjIDList (void data, ChareMlogData mlogData)
bool	isLocal (int destPE)
	Determines if the message is local or not.
bool	isTeamLocal (int destPE)
	Determines if the message is group local or not.
void	readKillFile ()
	CpvDeclare (Chare *, _currentObj)
	CpvDeclare (CkQ< LocalMessageLog > *, _localMessageLog)
	CpvDeclare (CkQ< TicketRequest * > *, _delayedTicketRequests)
	CpvDeclare (StoredCheckpoint *, _storedCheckpointData)
	CpvDeclare (CkQ< MlogEntry * > *, _delayedLocalTicketRequests)
	CpvDeclare (Queue, _outOfOrderMessageQueue)
	CpvDeclare (CkQ< LocalMessageLog > *, _bufferedLocalMessageLogs)
	CpvDeclare (char **, _bufferedTicketRequests)
	CpvDeclare (int *, _numBufferedTicketRequests)
	CpvDeclare (char *, _bufferTicketReply)
void	setTeamRecovery (void data, ChareMlogData mlogData)
	Turns on the flag for team recovery that selectively restores particular metadata information.
void	unsetTeamRecovery (void data, ChareMlogData mlogData)
	Turns off the flag for team recovery.
void	_messageLoggingInit ()
	Initialize message logging data structures and register handlers.
void	killLocal (void *_dummy, double curWallTime)
void	sendTicketGroupRequest (envelope *env, int destPE, int _infoIdx)
void	sendTicketNodeGroupRequest (envelope *env, int destNode, int _infoIdx)
void	sendTicketArrayRequest (envelope *env, int destPE, int _infoIdx)
void	generateCommonTicketRequest (CkObjID &recver, envelope *_env, int destPE, int _infoIdx)
	A method to generate the actual ticket requests for groups, nodegroups or arrays.
void	sendTicketRequest (CkObjID &sender, CkObjID &recver, int destPE, MlogEntry *entry, MCount SN, MCount TN, int resend)
	Method that does the actual send by creating a ticket request filling it up and sending it.
void	sendBufferedTicketRequests (int destPE)
	Send the ticket requests buffered for processor PE.
void	checkBufferedTicketRequests (void *_destPE, double curWallTime)
void	ticketLogLocalMessage (MlogEntry *entry)
	Gets a ticket for a local message and then sends a copy to the buddy.
void	sendLocalMessageCopy (MlogEntry *entry)
	Sends the metadata of a local message to its buddy.
void	sendBufferedLocalMessageCopy ()
void	checkBufferedLocalMessageCopy (void *_dummy, double curWallTime)
bool	_processTicketRequest (TicketRequest ticketRequest, TicketReply reply)
	Process the ticket request.
void	_ticketRequestHandler (TicketRequest *ticketRequest)
	If there are any delayed requests, process them first before processing this request.
void	_bufferedTicketRequestHandler (BufferedTicketRequestHeader *recvdHeader)
	Handler used for dealing with a bunch of ticket requests from one processor.
void	_ticketHandler (TicketReply *ticketReply)
	This function handles the ticket received after a request.
void	_bufferedTicketHandler (BufferedTicketRequestHeader *recvdHeader)
	Message to handle the bunch of tickets that we get from one processor.
void	_localMessageCopyHandler (LocalMessageLog *msgLog)
	Stores the metadata of a local message from its buddy.
void	_bufferedLocalMessageCopyHandler (BufferedLocalLogHeader *recvdHeader, int freeHeader)
void	_localMessageAckHandler (LocalMessageLogAck *ack)
void	_bufferedLocalMessageAckHandler (BufferedLocalLogHeader *recvdHeader)
int	preProcessReceivedMessage (envelope env, Chare objPointer, MlogEntry *logEntryPointer)
void	postProcessReceivedMessage (Chare obj, CkObjID &sender, MCount SN, MlogEntry entry)
	Updates a few variables once a message has been processed.
void	generalCldEnqueue (int destPE, envelope *env, int _infoIdx)
void	retryTicketRequestTimer (void *_dummy, double _time)
void	retryTicketRequest (void *_dummy, double curWallTime)
void	_pingHandler (CkPingMsg *msg)
void	buildProcessedTicketLog (void data, ChareMlogData mlogData)
void	clearUpMigratedRetainedLists (int PE)
void	checkpointAlarm (void *_dummy, double curWallTime)
void	_checkpointRequestHandler (CheckpointRequest *request)
void	startMlogCheckpoint (void *_dummy, double curWallTime)
	Starts the checkpoint phase after migration.
void	pupArrayElementsSkip (PUP::er &p, CmiBool create, MigrationRecord *listToSkip, int listsize)
	Pups all the array elements in this processor.
void	writeCheckpointToDisk (int size, char *chkpt)
void	_storeCheckpointHandler (char *msg)
void	sendRemoveLogRequests ()
void	_checkpointAckHandler (CheckPointAck *ackMsg)
void	removeProcessedLogs (void _data, ChareMlogData mlogData)
void	_removeProcessedLogHandler (char *requestMsg)
	Removes messages in the log according to the received ticket numbers.
void	CkMlogRestart (const char dummy, CkArgMsg dummyMsg)
	Function for restarting the crashed processor.
void	_restartHandler (RestartRequest *restartMsg)
	Function to restart this processor.
void	_getRestartCheckpointHandler (RestartRequest *restartMsg)
	Gets the stored checkpoint but calls another function in the sender.
void	_recvRestartCheckpointHandler (char *_restartData)
	Receives the checkpoint coming from its buddy.
void	CkMlogRestartDouble (void *, double)
void	CkMlogRestartLocal ()
void	readCheckpointFromDisk (int size, char *buf)
void	_getCheckpointHandler (RestartRequest *restartMsg)
	Gets the stored checkpoint for its buddy processor.
void	_verifyAckRequestHandler (VerifyAckMsg *verifyRequest)
void	_verifyAckHandler (VerifyAckMsg *verifyReply)
void	_recvCheckpointHandler (char *_restartData)
	Receives the checkpoint data from its buddy, restores the state of all the objects and asks everyone else to update its home.
void	_updateHomeAckHandler (RestartRequest *updateHomeAck)
	Receives the updateHome ACKs from all other processors.
void	initializeRestart (void data, ChareMlogData mlogData)
	Initializes variables and flags for restarting procedure.
void	updateHomePE (void data, ChareMlogData mlogData)
	Updates the homePe of chare array elements.
void	_updateHomeRequestHandler (RestartRequest *updateRequest)
	Updates the homePe for all chares in this processor.
void	fillTicketForChare (void data, ChareMlogData mlogData)
	Fills up the ticket vector for each chare.
void	resendMessageForChare (void data, ChareMlogData mlogData)
void	_resendMessagesHandler (char *msg)
	Resends the messages since the last checkpoint to the list of objects included in the request.
void	sortVec (CkVec< MCount > *TNvec)
int	searchVec (CkVec< MCount > *TNVec, MCount searchTN)
void	_resendReplyHandler (char *msg)
	Receives the tickets assigned to message to other objects.
void	_receivedTNDataHandler (ReceivedTNData *msg)
	Receives a list of TNs coming from the home PE of a migrated object (parallel restart).
void	processReceivedTN (Chare obj, int listSize, MCount listTNs)
	Processes the received list of tickets from a particular PE.
void	distributeRestartedObjects ()
	Distributes objects to accelerate recovery after a failure.
void	_distributedLocationHandler (char *receivedMsg)
	Handler to update information about an object just received.
void	sendDummyMigration (int restartPE, CkGroupID lbID, CkGroupID locMgrID, CkArrayIndex &idx, int locationPE)
	this method is used to send messages to a restarted processor to tell it that a particular expected object is not going to get to it
void	sendDummyMigrationCounts (int *dummyCounts)
	this method is used by a restarted processor to tell other processors that they are not going to receive these many objects.
void	_dummyMigrationHandler (DummyMigrationMsg *msg)
	this handler is used to process a dummy migration msg.
void	forAllCharesDo (MlogFn fnPointer, void *data)
	Map function pointed by fnPointer over all the chares living in this processor.
void	initMlogLBStep (CkGroupID gid)
	This is the first time Converse is called after AtSync method has been called by every local object.
void	startLoadBalancingMlog (void(_fnPtr)(void ), void *_centralLb)
	Load Balancing.
void	finishedCheckpointLoadBalancing ()
void	sendMlogLocation (int targetPE, envelope *env)
void	_receiveMigrationNoticeHandler (MigrationNotice *msg)
void	_receiveMigrationNoticeAckHandler (MigrationNoticeAck *msg)
void	_receiveMlogLocationHandler (void *buf)
void	resumeFromSyncRestart (void data, ChareMlogData mlogData)
void	checkAndSendCheckpointBarrierAcks (CheckpointBarrierMsg *msg)
void	_checkpointBarrierHandler (CheckpointBarrierMsg *msg)
	Processor 0 receives a contribution from every other processor after checkpoint.
void	_checkpointBarrierAckHandler (CheckpointBarrierMsg *msg)
void	informLocationHome (CkGroupID locMgrID, CkArrayIndex idx, int homePE, int currentPE)
	method that informs an array elements home processor of its current location It is a converse method to bypass the charm++ message logging framework
void	_receiveLocationHandler (CurrentLocationMsg *data)
void	_getGlobalStepHandler (LBStepMsg *msg)
void	_recvGlobalStepHandler (LBStepMsg *msg)
	Receives the global step handler from PE 0.
void	_messageLoggingExit ()
	Function to wrap up performance information.
void	sortRestoredLocalMsgLog (void _dummy, ChareMlogData mlogData)
int	getCheckPointPE ()
	Getting the pe number of the current processor's buddy.
envelope *	copyEnvelope (envelope *env)
Variables
int	_restartFlag = 0
float	MLOGFT_totalLogSize = 0.0
float	MLOGFT_totalMessages = 0.0
float	MLOGFT_totalObjects = 0.0
int	countHashRefs = 0
int	countHashCollisions = 0
char *	checkpointDirectory = "."
int	unAckedCheckpoint = 0
int	countLocal = 0
int	countBuffered = 0
int	countPiggy = 0
int	countClearBufferedLocalCalls = 0
int	countUpdateHomeAcks = 0
int	teamSize
int	chkptPeriod
bool	parallelRestart
char *	killFile
int	killFlag = 0
int	restartingMlogFlag = 0
double	killTime = 0.0
int	checkpointCount = 0
static double	adjustChkptPeriod = 0.0
static double	nextCheckpointTime = 0.0
double	lastBufferedLocalMessageCopyTime
int	_maxBufferedMessages
int	_maxBufferedTicketRequests
int	BUFFER_TIME = 2
int	_ticketRequestHandlerIdx
int	_ticketHandlerIdx
int	_localMessageCopyHandlerIdx
int	_localMessageAckHandlerIdx
int	_pingHandlerIdx
int	_bufferedLocalMessageCopyHandlerIdx
int	_bufferedLocalMessageAckHandlerIdx
int	_bufferedTicketRequestHandlerIdx
int	_bufferedTicketHandlerIdx
char	objString [100]
int	_checkpointRequestHandlerIdx
int	_storeCheckpointHandlerIdx
int	_checkpointAckHandlerIdx
int	_getCheckpointHandlerIdx
int	_recvCheckpointHandlerIdx
int	_removeProcessedLogHandlerIdx
int	_verifyAckRequestHandlerIdx
int	_verifyAckHandlerIdx
int	_dummyMigrationHandlerIdx
int	_getGlobalStepHandlerIdx
int	_recvGlobalStepHandlerIdx
int	_updateHomeRequestHandlerIdx
int	_updateHomeAckHandlerIdx
int	_resendMessagesHandlerIdx
int	_resendReplyHandlerIdx
int	_receivedTNDataHandlerIdx
int	_distributedLocationHandlerIdx
int	_restartHandlerIdx
int	_getRestartCheckpointHandlerIdx
int	_recvRestartCheckpointHandlerIdx
int	verifyAckTotal
int	verifyAckCount
int	verifyAckedRequests = 0
RestartRequest *	storedRequest
int	_falseRestart = 0
int	_lockNewTicket = 0
	For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.
int	onGoingLoadBalancing = 0
	For testing on clusters we might carry out restarts on a porcessor without actually starting it 1 -> false restart 0 -> restart after an actual crash.
void *	centralLb
void(*	resumeLbFnPtr )(void *)
int	_receiveMlogLocationHandlerIdx
int	_receiveMigrationNoticeHandlerIdx
int	_receiveMigrationNoticeAckHandlerIdx
int	_checkpointBarrierHandlerIdx
int	_checkpointBarrierAckHandlerIdx
CkVec< MigrationRecord >	migratedNoticeList
CkVec< RetainedMigratedObject * >	retainedObjectList
int	donotCountMigration = 0
int	countLBMigratedAway = 0
int	countLBToMigrate = 0
int	migrationDoneCalled = 0
int	checkpointBarrierCount = 0
int	globalResumeCount = 0
CkGroupID	globalLBID
int	restartDecisionNumber = -1
double	lastCompletedAlarm = 0
double	lastRestart = 0
int	_receiveLocationHandlerIdx
int	calledRetryTicketRequest = 0
	This method is used to retry the ticket requests that had been queued up earlier.
CkVec< TProcessedLog >	processedTicketLog
double	totalSearchRestoredTime = 0
double	totalSearchRestoredCount = 0