Examples with CheckpointInfo org.apache.heron.spi.statefulstorage.CheckpointInfo used on opensource projects

Example 6 with CheckpointInfo

use of org.apache.heron.spi.statefulstorage.CheckpointInfo in project heron by twitter.

the class DlogStorageTest method testRestore.

@Test
public void testRestore() throws Exception {
    DLInputStream mockInputStream = mock(DLInputStream.class);
    doReturn(mockInputStream).when(dlogStorage).openInputStream(anyString());
    PowerMockito.spy(CheckpointManager.InstanceStateCheckpoint.class);
    PowerMockito.doReturn(checkpointPartition).when(CheckpointManager.InstanceStateCheckpoint.class, "parseFrom", mockInputStream);
    final CheckpointInfo info = new CheckpointInfo(StatefulStorageTestContext.CHECKPOINT_ID, instance);
    Checkpoint restoreCheckpoint = dlogStorage.restoreCheckpoint(info);
    assertEquals(restoreCheckpoint.getCheckpoint(), checkpointPartition);
}

Example 7 with CheckpointInfo

use of org.apache.heron.spi.statefulstorage.CheckpointInfo in project heron by twitter.

the class LocalFileSystemStorageTest method testRestore.

@Test
public void testRestore() throws Exception {
    PowerMockito.spy(FileUtils.class);
    PowerMockito.doReturn(checkpoint.toByteArray()).when(FileUtils.class, "readFromFile", anyString());
    final CheckpointInfo info = new CheckpointInfo(StatefulStorageTestContext.CHECKPOINT_ID, instance);
    Checkpoint ckpt = localFileSystemStorage.restoreCheckpoint(info);
    assertEquals(checkpoint, ckpt.getCheckpoint());
}

Example 8 with CheckpointInfo

use of org.apache.heron.spi.statefulstorage.CheckpointInfo in project heron by twitter.

the class LocalFileSystemStorageTest method testStore.

@Test
public void testStore() throws Exception {
    PowerMockito.spy(FileUtils.class);
    PowerMockito.doReturn(true).when(FileUtils.class, "createDirectory", anyString());
    PowerMockito.doReturn(true).when(FileUtils.class, "isFileExists", anyString());
    PowerMockito.doReturn(true).when(FileUtils.class, "isDirectoryExists", anyString());
    PowerMockito.doReturn(true).when(FileUtils.class, "writeToFile", anyString(), any(byte[].class), anyBoolean());
    PowerMockito.doReturn(false).when(FileUtils.class, "hasChildren", anyString());
    Checkpoint mockCheckpoint = mock(Checkpoint.class);
    when(mockCheckpoint.getCheckpoint()).thenReturn(checkpoint);
    final CheckpointInfo info = new CheckpointInfo(StatefulStorageTestContext.CHECKPOINT_ID, instance);
    localFileSystemStorage.storeCheckpoint(info, mockCheckpoint);
    PowerMockito.verifyStatic(times(1));
    FileUtils.writeToFile(anyString(), eq(checkpoint.toByteArray()), eq(true));
}

Example 9 with CheckpointInfo

use of org.apache.heron.spi.statefulstorage.CheckpointInfo in project heron by twitter.

the class CheckpointManagerServer method handleGetInstanceStateRequest.

protected void handleGetInstanceStateRequest(REQID rid, SocketChannel channel, CheckpointManager.GetInstanceStateRequest request) {
    CheckpointInfo info = new CheckpointInfo(request.getCheckpointId(), request.getInstance());
    LOG.info(String.format("Got a get checkpoint request for checkpointId %s " + " component %s instanceId %d on connection %s", info.getCheckpointId(), info.getComponent(), info.getInstanceId(), channel.socket().getRemoteSocketAddress()));
    CheckpointManager.GetInstanceStateResponse.Builder responseBuilder = CheckpointManager.GetInstanceStateResponse.newBuilder();
    responseBuilder.setInstance(request.getInstance());
    responseBuilder.setCheckpointId(request.getCheckpointId());
    String errorMessage = "";
    Common.StatusCode statusCode = Common.StatusCode.OK;
    if (!request.hasCheckpointId() || request.getCheckpointId().isEmpty()) {
        LOG.info("The checkpoint id was empty, this sending empty state");
        CheckpointManager.InstanceStateCheckpoint dummyState = CheckpointManager.InstanceStateCheckpoint.newBuilder().setCheckpointId(request.getCheckpointId()).setState(ByteString.EMPTY).build();
        responseBuilder.setCheckpoint(dummyState);
    } else {
        try {
            Checkpoint checkpoint = statefulStorage.restoreCheckpoint(info);
            LOG.info(String.format("Get checkpoint successful for checkpointId %s " + "component %s instanceId %d", info.getCheckpointId(), info.getComponent(), info.getInstanceId()));
            // Set the checkpoint-state in response
            if (spillState) {
                CheckpointManager.InstanceStateCheckpoint ckpt = checkpoint.getCheckpoint();
                String checkpointId = ckpt.getCheckpointId();
                // clean any possible existing states
                FileUtils.cleanDir(spillStateLocation);
                // spill state to local disk
                String stateLocation = spillStateLocation + checkpointId + "-" + UUID.randomUUID();
                if (!FileUtils.writeToFile(stateLocation, ckpt.getState().toByteArray(), true)) {
                    throw new RuntimeException("failed to spill state. Bailing out...");
                }
                LOG.info("spilled state to: " + stateLocation);
                ckpt = CheckpointManager.InstanceStateCheckpoint.newBuilder().setStateLocation(stateLocation).setCheckpointId(checkpointId).build();
                responseBuilder.setCheckpoint(ckpt);
            } else {
                responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
            }
        } catch (StatefulStorageException e) {
            errorMessage = String.format("Get checkpoint not successful for checkpointId %s " + "component %s instanceId %d", info.getCheckpointId(), info.getComponent(), info.getInstanceId());
            LOG.log(Level.WARNING, errorMessage, e);
            statusCode = Common.StatusCode.NOTOK;
        }
    }
    responseBuilder.setStatus(Common.Status.newBuilder().setStatus(statusCode).setMessage(errorMessage));
    sendResponse(rid, channel, responseBuilder.build());
}