Search in sources :

Example 1 with DefaultPhysicalOperatorImpl

use of edu.snu.mist.core.task.DefaultPhysicalOperatorImpl in project mist by snuspl.

the class DefaultGroupCheckpointStore method checkpointGroupStates.

@Override
public CheckpointResult checkpointGroupStates(final Tuple<String, Group> tuple) {
    final String groupId = tuple.getKey();
    final Group group = tuple.getValue();
    final GroupCheckpoint checkpoint = group.checkpoint();
    try {
        // Write the file.
        final File storedFile = getGroupCheckpointFile(groupId);
        if (storedFile.exists()) {
            storedFile.delete();
            LOG.log(Level.INFO, "Checkpoint deleted for groupId: {0}", groupId);
        }
        final DataFileWriter<GroupCheckpoint> dataFileWriter = new DataFileWriter<>(groupCheckpointDatumWriter);
        dataFileWriter.create(checkpoint.getSchema(), storedFile);
        dataFileWriter.append(checkpoint);
        dataFileWriter.close();
        LOG.log(Level.INFO, "Checkpoint completed for groupId: {0}", groupId);
    } catch (final Exception e) {
        e.printStackTrace();
        return CheckpointResult.newBuilder().setIsSuccess(false).setMsg("Unsuccessful in checkpointing group " + tuple.getKey()).setPathToCheckpoint("").build();
    }
    // Delete all the unnecessary states within the stateMaps of stateful operators.
    for (final ExecutionDag ed : group.getExecutionDags().values()) {
        for (final ExecutionVertex ev : ed.getDag().getVertices()) {
            if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
                final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
                if (op instanceof StateHandler) {
                    final StateHandler stateHandler = (StateHandler) op;
                    stateHandler.removeOldStates(checkpoint.getCheckpointTimestamp());
                }
            }
        }
    }
    return CheckpointResult.newBuilder().setIsSuccess(true).setMsg("Successfully checkpointed group " + tuple.getKey()).setPathToCheckpoint(getGroupCheckpointFile(groupId).toString()).build();
}
Also used : GroupCheckpoint(edu.snu.mist.formats.avro.GroupCheckpoint) Operator(edu.snu.mist.core.operators.Operator) Group(edu.snu.mist.core.task.groupaware.Group) StateHandler(edu.snu.mist.core.operators.StateHandler) DataFileWriter(org.apache.avro.file.DataFileWriter) DefaultPhysicalOperatorImpl(edu.snu.mist.core.task.DefaultPhysicalOperatorImpl) ExecutionDag(edu.snu.mist.core.task.ExecutionDag) File(java.io.File) IOException(java.io.IOException) ExecutionVertex(edu.snu.mist.core.task.ExecutionVertex)

Aggregations

Operator (edu.snu.mist.core.operators.Operator)1 StateHandler (edu.snu.mist.core.operators.StateHandler)1 DefaultPhysicalOperatorImpl (edu.snu.mist.core.task.DefaultPhysicalOperatorImpl)1 ExecutionDag (edu.snu.mist.core.task.ExecutionDag)1 ExecutionVertex (edu.snu.mist.core.task.ExecutionVertex)1 Group (edu.snu.mist.core.task.groupaware.Group)1 GroupCheckpoint (edu.snu.mist.formats.avro.GroupCheckpoint)1 File (java.io.File)1 IOException (java.io.IOException)1 DataFileWriter (org.apache.avro.file.DataFileWriter)1