Examples with Checkpoint org.apache.heron.spi.statefulstorage.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of org.apache.heron.spi.statefulstorage.Checkpoint in project heron by twitter.

the class CheckpointManagerServer method handleSaveInstanceStateRequest.

protected void handleSaveInstanceStateRequest(REQID rid, SocketChannel channel, CheckpointManager.SaveInstanceStateRequest request) {
    CheckpointInfo info = new CheckpointInfo(request.getCheckpoint().getCheckpointId(), request.getInstance());
    Checkpoint checkpoint;
    if (request.getCheckpoint().hasStateLocation()) {
        checkpoint = loadCheckpoint(request.getInstance(), request.getCheckpoint().getCheckpointId(), request.getCheckpoint().getStateLocation());
    } else {
        checkpoint = new Checkpoint(request.getCheckpoint());
    }
    LOG.info(String.format("Got a save checkpoint request for checkpointId %s " + " component %s instanceId %s on connection %s", info.getCheckpointId(), info.getComponent(), info.getInstanceId(), channel.socket().getRemoteSocketAddress()));
    Common.StatusCode statusCode = Common.StatusCode.OK;
    String errorMessage = "";
    try {
        statefulStorage.storeCheckpoint(info, checkpoint);
        LOG.info(String.format("Saved checkpoint for checkpointId %s compnent %s instanceId %s", info.getCheckpointId(), info.getComponent(), info.getInstanceId()));
    } catch (StatefulStorageException e) {
        errorMessage = String.format("Save checkpoint not successful for checkpointId " + "%s component %s instanceId %s", info.getCheckpointId(), info.getComponent(), info.getInstanceId());
        statusCode = Common.StatusCode.NOTOK;
        LOG.log(Level.WARNING, errorMessage, e);
    }
    CheckpointManager.SaveInstanceStateResponse.Builder responseBuilder = CheckpointManager.SaveInstanceStateResponse.newBuilder();
    responseBuilder.setStatus(Common.Status.newBuilder().setStatus(statusCode).setMessage(errorMessage));
    responseBuilder.setCheckpointId(request.getCheckpoint().getCheckpointId());
    responseBuilder.setInstance(request.getInstance());
    sendResponse(rid, channel, responseBuilder.build());
}

Example 2 with Checkpoint

use of org.apache.heron.spi.statefulstorage.Checkpoint in project heron by twitter.

the class CheckpointManagerServerTest method testGetInstanceState.

@Test
public void testGetInstanceState() throws Exception {
    final CheckpointInfo info = new CheckpointInfo(CHECKPOINT_ID, instance);
    final Checkpoint checkpoint = new Checkpoint(checkpointPartition);
    when(statefulStorage.restoreCheckpoint(any(CheckpointInfo.class))).thenReturn(checkpoint);
    runTest(TestRequestHandler.RequestType.GET_INSTANCE_STATE, new HeronServerTester.SuccessResponseHandler(CheckpointManager.GetInstanceStateResponse.class, new HeronServerTester.TestResponseHandler() {

        @Override
        public void handleResponse(HeronClient client, StatusCode status, Object ctx, Message response) throws Exception {
            verify(statefulStorage).restoreCheckpoint(info);
            assertEquals(checkpoint.getCheckpoint(), ((CheckpointManager.GetInstanceStateResponse) response).getCheckpoint());
        }
    }));
}

Example 3 with Checkpoint

use of org.apache.heron.spi.statefulstorage.Checkpoint in project heron by twitter.

the class CheckpointManagerServerTest method setup.

@BeforeClass
public static void setup() throws Exception {
    final String INSTANCE_ID = "instance_id";
    final String STMGR_ID = "stmgr_id";
    final int TASK_ID = 1;
    final int COMPONENT_INDEX = 1;
    final String COMPONENT_NAME = "component_name";
    final byte[] BYTES = "checkpoint manager server test bytes".getBytes();
    final String TOPO_ID = "topo_id";
    final String TOPO_NAME = "topo_name";
    PhysicalPlans.InstanceInfo info = PhysicalPlans.InstanceInfo.newBuilder().setTaskId(TASK_ID).setComponentIndex(COMPONENT_INDEX).setComponentName(COMPONENT_NAME).build();
    TopologyAPI.Topology topology = TopologyAPI.Topology.newBuilder().setId(TOPO_ID).setName(TOPO_NAME).setState(TopologyAPI.TopologyState.RUNNING).build();
    PhysicalPlans.PhysicalPlan pplan = PhysicalPlans.PhysicalPlan.newBuilder().setTopology(topology).build();
    instance = PhysicalPlans.Instance.newBuilder().setInstanceId(INSTANCE_ID).setStmgrId(STMGR_ID).setInfo(info).build();
    checkpointPartition = CheckpointManager.InstanceStateCheckpoint.newBuilder().setCheckpointId(CHECKPOINT_ID).setState(ByteString.copyFrom(BYTES)).build();
    checkpointComponentMetadata = CheckpointManager.CheckpointComponentMetadata.newBuilder().setComponentName(COMPONENT_NAME).setParallelism(2).build();
    saveInstanceStateRequest = CheckpointManager.SaveInstanceStateRequest.newBuilder().setInstance(instance).setCheckpoint(checkpointPartition).build();
    getInstanceStateRequest = CheckpointManager.GetInstanceStateRequest.newBuilder().setInstance(instance).setCheckpointId(CHECKPOINT_ID).build();
    cleanStatefulCheckpointRequest = CheckpointManager.CleanStatefulCheckpointRequest.newBuilder().setCleanAllCheckpoints(true).setOldestCheckpointPreserved(CHECKPOINT_ID).build();
    registerStmgrRequest = CheckpointManager.RegisterStMgrRequest.newBuilder().setTopologyId(TOPOLOGY_ID).setStmgrId(STMGR_ID).setTopologyName(TOPOLOGY_NAME).setPhysicalPlan(pplan).build();
    registerTManagerRequest = CheckpointManager.RegisterTManagerRequest.newBuilder().setTopologyId(TOPOLOGY_ID).setTopologyName(TOPOLOGY_NAME).build();
}

Example 4 with Checkpoint

use of org.apache.heron.spi.statefulstorage.Checkpoint in project heron by twitter.

the class LocalFileSystemStorage method restoreCheckpoint.

@Override
public Checkpoint restoreCheckpoint(CheckpointInfo info) throws StatefulStorageException {
    String path = getCheckpointPath(info.getCheckpointId(), info.getComponent(), info.getInstanceId());
    byte[] res = FileUtils.readFromFile(path);
    if (res.length != 0) {
        // Try to parse the protobuf
        CheckpointManager.InstanceStateCheckpoint state;
        try {
            state = CheckpointManager.InstanceStateCheckpoint.parseFrom(res);
        } catch (InvalidProtocolBufferException e) {
            throw new StatefulStorageException("Failed to parse the data", e);
        }
        return new Checkpoint(state);
    } else {
        throw new StatefulStorageException("Failed to parse the data");
    }
}

Example 5 with Checkpoint

use of org.apache.heron.spi.statefulstorage.Checkpoint in project heron by twitter.

the class HDFSStorageTest method testRestore.

@Test
public void testRestore() throws Exception {
    FSDataInputStream mockFSDataInputStream = mock(FSDataInputStream.class);
    when(mockFileSystem.open(any(Path.class))).thenReturn(mockFSDataInputStream);
    PowerMockito.spy(CheckpointManager.InstanceStateCheckpoint.class);
    PowerMockito.doReturn(instanceCheckpointState).when(CheckpointManager.InstanceStateCheckpoint.class, "parseFrom", mockFSDataInputStream);
    final CheckpointInfo info = new CheckpointInfo(StatefulStorageTestContext.CHECKPOINT_ID, instance);
    Checkpoint restoreCheckpoint = hdfsStorage.restoreCheckpoint(info);
    assertEquals(restoreCheckpoint.getCheckpoint(), instanceCheckpointState);
}