Examples with GroupCheckpoint edu.snu.mist.formats.avro.GroupCheckpoint used on opensource projects

Example 1 with GroupCheckpoint

use of edu.snu.mist.formats.avro.GroupCheckpoint in project mist by snuspl.

the class DefaultGroupCheckpointStore method loadSavedGroupState.

@Override
public GroupCheckpoint loadSavedGroupState(final String groupId) throws IOException {
    // Load the file.
    final File storedFile = getGroupCheckpointFile(groupId);
    final DataFileReader<GroupCheckpoint> dataFileReader = new DataFileReader<>(storedFile, groupCheckpointDatumReader);
    GroupCheckpoint mgc = null;
    mgc = dataFileReader.next(mgc);
    if (mgc != null) {
        LOG.log(Level.INFO, "Checkpoint file found. groupId is " + groupId);
    } else {
        LOG.log(Level.WARNING, "Checkpoint file not found or error during loading. groupId is " + groupId);
    }
    return mgc;
}

Example 2 with GroupCheckpoint

use of edu.snu.mist.formats.avro.GroupCheckpoint in project mist by snuspl.

the class DefaultGroupCheckpointStore method checkpointGroupStates.

@Override
public CheckpointResult checkpointGroupStates(final Tuple<String, Group> tuple) {
    final String groupId = tuple.getKey();
    final Group group = tuple.getValue();
    final GroupCheckpoint checkpoint = group.checkpoint();
    try {
        // Write the file.
        final File storedFile = getGroupCheckpointFile(groupId);
        if (storedFile.exists()) {
            storedFile.delete();
            LOG.log(Level.INFO, "Checkpoint deleted for groupId: {0}", groupId);
        }
        final DataFileWriter<GroupCheckpoint> dataFileWriter = new DataFileWriter<>(groupCheckpointDatumWriter);
        dataFileWriter.create(checkpoint.getSchema(), storedFile);
        dataFileWriter.append(checkpoint);
        dataFileWriter.close();
        LOG.log(Level.INFO, "Checkpoint completed for groupId: {0}", groupId);
    } catch (final Exception e) {
        e.printStackTrace();
        return CheckpointResult.newBuilder().setIsSuccess(false).setMsg("Unsuccessful in checkpointing group " + tuple.getKey()).setPathToCheckpoint("").build();
    }
    // Delete all the unnecessary states within the stateMaps of stateful operators.
    for (final ExecutionDag ed : group.getExecutionDags().values()) {
        for (final ExecutionVertex ev : ed.getDag().getVertices()) {
            if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
                final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
                if (op instanceof StateHandler) {
                    final StateHandler stateHandler = (StateHandler) op;
                    stateHandler.removeOldStates(checkpoint.getCheckpointTimestamp());
                }
            }
        }
    }
    return CheckpointResult.newBuilder().setIsSuccess(true).setMsg("Successfully checkpointed group " + tuple.getKey()).setPathToCheckpoint(getGroupCheckpointFile(groupId).toString()).build();
}