Examples with Checkpoint com.twitter.heron.spi.statefulstorage.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of com.twitter.heron.spi.statefulstorage.Checkpoint in project incubator-heron by apache.

the class LocalFileSystemStorage method restore.

@Override
public Checkpoint restore(String topologyName, String checkpointId, PhysicalPlans.Instance instanceInfo) throws StatefulStorageException {
    String path = getCheckpointPath(topologyName, checkpointId, instanceInfo.getInfo().getComponentName(), instanceInfo.getInfo().getTaskId());
    byte[] res = FileUtils.readFromFile(path);
    if (res.length != 0) {
        // Try to parse the protobuf
        CheckpointManager.InstanceStateCheckpoint state;
        try {
            state = CheckpointManager.InstanceStateCheckpoint.parseFrom(res);
        } catch (InvalidProtocolBufferException e) {
            throw new StatefulStorageException("Failed to parse the data", e);
        }
        return new Checkpoint(topologyName, instanceInfo, state);
    } else {
        throw new StatefulStorageException("Failed to parse the data");
    }
}

Example 2 with Checkpoint

use of com.twitter.heron.spi.statefulstorage.Checkpoint in project incubator-heron by apache.

the class HDFSStorageTest method testRestore.

@Test
public void testRestore() throws Exception {
    Checkpoint restoreCheckpoint = new Checkpoint(StatefulStorageTestContext.TOPOLOGY_NAME, instance, instanceCheckpointState);
    FSDataInputStream mockFSDataInputStream = mock(FSDataInputStream.class);
    when(mockFileSystem.open(any(Path.class))).thenReturn(mockFSDataInputStream);
    PowerMockito.spy(CheckpointManager.InstanceStateCheckpoint.class);
    PowerMockito.doReturn(instanceCheckpointState).when(CheckpointManager.InstanceStateCheckpoint.class, "parseFrom", mockFSDataInputStream);
    hdfsStorage.restore(StatefulStorageTestContext.TOPOLOGY_NAME, StatefulStorageTestContext.CHECKPOINT_ID, instance);
    assertEquals(restoreCheckpoint.getCheckpoint(), instanceCheckpointState);
}

Example 3 with Checkpoint

use of com.twitter.heron.spi.statefulstorage.Checkpoint in project incubator-heron by apache.

the class CheckpointManagerServerTest method testGetInstanceState.

@Test
public void testGetInstanceState() throws Exception {
    final Checkpoint checkpoint = new Checkpoint(TOPOLOGY_NAME, instance, instanceStateCheckpoint);
    when(statefulStorage.restore(TOPOLOGY_NAME, CHECKPOINT_ID, instance)).thenReturn(checkpoint);
    runTest(TestRequestHandler.RequestType.GET_INSTANCE_STATE, new HeronServerTester.SuccessResponseHandler(CheckpointManager.GetInstanceStateResponse.class, new HeronServerTester.TestResponseHandler() {

        @Override
        public void handleResponse(HeronClient client, StatusCode status, Object ctx, Message response) throws Exception {
            verify(statefulStorage).restore(TOPOLOGY_NAME, CHECKPOINT_ID, instance);
            assertEquals(checkpoint.getCheckpoint(), ((CheckpointManager.GetInstanceStateResponse) response).getCheckpoint());
        }
    }));
}

Example 4 with Checkpoint

use of com.twitter.heron.spi.statefulstorage.Checkpoint in project incubator-heron by apache.

the class CheckpointManagerServerTest method setup.

@BeforeClass
public static void setup() throws Exception {
    final String INSTANCE_ID = "instance_id";
    final String STMGR_ID = "stmgr_id";
    final int TASK_ID = 1;
    final int COMPONENT_INDEX = 1;
    final String COMPONENT_NAME = "component_name";
    final byte[] BYTES = "checkpoint manager server test bytes".getBytes();
    PhysicalPlans.InstanceInfo info = PhysicalPlans.InstanceInfo.newBuilder().setTaskId(TASK_ID).setComponentIndex(COMPONENT_INDEX).setComponentName(COMPONENT_NAME).build();
    instance = PhysicalPlans.Instance.newBuilder().setInstanceId(INSTANCE_ID).setStmgrId(STMGR_ID).setInfo(info).build();
    instanceStateCheckpoint = CheckpointManager.InstanceStateCheckpoint.newBuilder().setCheckpointId(CHECKPOINT_ID).setState(ByteString.copyFrom(BYTES)).build();
    saveInstanceStateRequest = CheckpointManager.SaveInstanceStateRequest.newBuilder().setInstance(instance).setCheckpoint(instanceStateCheckpoint).build();
    getInstanceStateRequest = CheckpointManager.GetInstanceStateRequest.newBuilder().setInstance(instance).setCheckpointId(CHECKPOINT_ID).build();
    cleanStatefulCheckpointRequest = CheckpointManager.CleanStatefulCheckpointRequest.newBuilder().setCleanAllCheckpoints(true).setOldestCheckpointPreserved(CHECKPOINT_ID).build();
    registerStmgrRequest = CheckpointManager.RegisterStMgrRequest.newBuilder().setTopologyId(TOPOLOGY_ID).setStmgrId(STMGR_ID).setTopologyName(TOPOLOGY_NAME).build();
    registerTMasterRequest = CheckpointManager.RegisterTMasterRequest.newBuilder().setTopologyId(TOPOLOGY_ID).setTopologyName(TOPOLOGY_NAME).build();
}

Example 5 with Checkpoint

use of com.twitter.heron.spi.statefulstorage.Checkpoint in project incubator-heron by apache.

the class CheckpointManagerServer method handleSaveInstanceStateRequest.

protected void handleSaveInstanceStateRequest(REQID rid, SocketChannel channel, CheckpointManager.SaveInstanceStateRequest request) {
    Checkpoint checkpoint = new Checkpoint(topologyName, request.getInstance(), request.getCheckpoint());
    LOG.info(String.format("Got a save checkpoint request for checkpointId %s " + " component %s instance %s on connection %s", checkpoint.getCheckpointId(), checkpoint.getComponent(), checkpoint.getInstance(), channel.socket().getRemoteSocketAddress()));
    Common.StatusCode statusCode = Common.StatusCode.OK;
    String errorMessage = "";
    try {
        statefulStorage.store(checkpoint);
        LOG.info(String.format("Saved checkpoint for checkpointId %s compnent %s instance %s", checkpoint.getCheckpointId(), checkpoint.getComponent(), checkpoint.getInstance()));
    } catch (StatefulStorageException e) {
        errorMessage = String.format("Save checkpoint not successful for checkpointId " + "%s component %s instance %s", checkpoint.getCheckpointId(), checkpoint.getComponent(), checkpoint.getInstance());
        statusCode = Common.StatusCode.NOTOK;
        LOG.log(Level.WARNING, errorMessage, e);
    }
    CheckpointManager.SaveInstanceStateResponse.Builder responseBuilder = CheckpointManager.SaveInstanceStateResponse.newBuilder();
    responseBuilder.setStatus(Common.Status.newBuilder().setStatus(statusCode).setMessage(errorMessage));
    responseBuilder.setCheckpointId(request.getCheckpoint().getCheckpointId());
    responseBuilder.setInstance(request.getInstance());
    sendResponse(rid, channel, responseBuilder.build());
}