Examples with Operator edu.snu.mist.core.operators.Operator used on opensource projects

Example 1 with Operator

use of edu.snu.mist.core.operators.Operator in project mist by snuspl.

the class DefaultGroupImpl method getQueryCheckpoint.

/**
 * Get the checkpoint data for the query.
 */
private QueryCheckpoint getQueryCheckpoint(final DAG<ConfigVertex, MISTEdge> configDag, final GroupMinimumLatestWatermarkTimeStamp groupTimestamp) {
    // Find the minimum of the available checkpoint timestamps for the group.
    // Replaying will start from this timestamp, if this ConfigDag is used for recovery.
    // This is initiated as Long.MAX_VALUE, as this means that there are no stateful operators within this dag,
    // and therefore requires no replay.
    long latestWatermarkTimestamp = Long.MAX_VALUE;
    for (final ConfigVertex cv : configDag.getVertices()) {
        final ExecutionVertex ev = configExecutionVertexMap.get(cv);
        if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
            final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
            if (op instanceof StateHandler) {
                final StateHandler stateHandler = (StateHandler) op;
                latestWatermarkTimestamp = stateHandler.getLatestTimestampBeforeCheckpoint();
                groupTimestamp.compareAndSetIfSmaller(latestWatermarkTimestamp);
            }
        }
    }
    final Map<String, StateWithTimestamp> stateWithTimestampMap = new HashMap<>();
    for (final ConfigVertex cv : configDag.getVertices()) {
        final ExecutionVertex ev = configExecutionVertexMap.get(cv);
        Map<String, Object> state = null;
        long checkpointTimestamp = 0L;
        if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
            final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
            if (op instanceof StateHandler) {
                final StateHandler stateHandler = (StateHandler) op;
                checkpointTimestamp = stateHandler.getMaxAvailableTimestamp(groupTimestamp.getValue());
                state = StateSerializer.serializeStateMap(stateHandler.getOperatorState(checkpointTimestamp));
            }
        }
        stateWithTimestampMap.put(cv.getId(), StateWithTimestamp.newBuilder().setVertexState(state).setCheckpointTimestamp(checkpointTimestamp).build());
    }
    return QueryCheckpoint.newBuilder().setQueryState(stateWithTimestampMap).build();
}

Example 2 with Operator

use of edu.snu.mist.core.operators.Operator in project mist by snuspl.

the class DefaultGroupCheckpointStore method checkpointGroupStates.

@Override
public CheckpointResult checkpointGroupStates(final Tuple<String, Group> tuple) {
    final String groupId = tuple.getKey();
    final Group group = tuple.getValue();
    final GroupCheckpoint checkpoint = group.checkpoint();
    try {
        // Write the file.
        final File storedFile = getGroupCheckpointFile(groupId);
        if (storedFile.exists()) {
            storedFile.delete();
            LOG.log(Level.INFO, "Checkpoint deleted for groupId: {0}", groupId);
        }
        final DataFileWriter<GroupCheckpoint> dataFileWriter = new DataFileWriter<>(groupCheckpointDatumWriter);
        dataFileWriter.create(checkpoint.getSchema(), storedFile);
        dataFileWriter.append(checkpoint);
        dataFileWriter.close();
        LOG.log(Level.INFO, "Checkpoint completed for groupId: {0}", groupId);
    } catch (final Exception e) {
        e.printStackTrace();
        return CheckpointResult.newBuilder().setIsSuccess(false).setMsg("Unsuccessful in checkpointing group " + tuple.getKey()).setPathToCheckpoint("").build();
    }
    // Delete all the unnecessary states within the stateMaps of stateful operators.
    for (final ExecutionDag ed : group.getExecutionDags().values()) {
        for (final ExecutionVertex ev : ed.getDag().getVertices()) {
            if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
                final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
                if (op instanceof StateHandler) {
                    final StateHandler stateHandler = (StateHandler) op;
                    stateHandler.removeOldStates(checkpoint.getCheckpointTimestamp());
                }
            }
        }
    }
    return CheckpointResult.newBuilder().setIsSuccess(true).setMsg("Successfully checkpointed group " + tuple.getKey()).setPathToCheckpoint(getGroupCheckpointFile(groupId).toString()).build();
}