Examples with StatefulStorageException org.apache.heron.spi.statefulstorage.StatefulStorageException used on opensource projects

Example 11 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class DlogStorage method restoreCheckpoint.

@Override
public Checkpoint restoreCheckpoint(CheckpointInfo info) throws StatefulStorageException {
    String checkpointPath = getCheckpointPath(topologyName, info.getCheckpointId(), info.getComponent(), info.getInstanceId());
    InputStream in = null;
    CheckpointManager.InstanceStateCheckpoint state;
    try {
        in = openInputStream(checkpointPath);
        state = CheckpointManager.InstanceStateCheckpoint.parseFrom(in);
    } catch (IOException ioe) {
        throw new StatefulStorageException("Failed to read checkpoint from " + checkpointPath, ioe);
    } finally {
        if (in != null) {
            final long num = ((DLInputStream) in).getNumOfBytesRead();
            LOG.info(() -> num + " bytes read");
        }
        SysUtils.closeIgnoringExceptions(in);
    }
    return new Checkpoint(state);
}

Example 12 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class DlogStorage method storeCheckpoint.

@Override
public void storeCheckpoint(CheckpointInfo info, Checkpoint checkpoint) throws StatefulStorageException {
    String checkpointPath = getCheckpointPath(topologyName, info.getCheckpointId(), info.getComponent(), info.getInstanceId());
    OutputStream out = null;
    try {
        out = openOutputStream(checkpointPath);
        LOG.info(() -> String.format("writing a check point of %d bytes", checkpoint.getCheckpoint().getSerializedSize()));
        checkpoint.getCheckpoint().writeTo(out);
        out.flush();
    } catch (IOException e) {
        throw new StatefulStorageException("Failed to persist checkpoint @ " + checkpointPath, e);
    } finally {
        if (out != null) {
            final long num = ((DLOutputStream) out).getNumOfBytesWritten();
            LOG.info(() -> num + "bytes written");
        }
        SysUtils.closeIgnoringExceptions(out);
    }
}

Example 13 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class CheckpointManager method setupStatefulStorage.

private static IStatefulStorage setupStatefulStorage(String topologyName, CheckpointManagerConfig checkpointManagerConfig) throws CheckpointManagerException {
    IStatefulStorage statefulStorage;
    String classname = checkpointManagerConfig.getStorageClassname();
    try {
        statefulStorage = (IStatefulStorage) Class.forName(classname).newInstance();
    } catch (InstantiationException e) {
        throw new CheckpointManagerException(classname + " class must have a no-arg constructor.", e);
    } catch (IllegalAccessException e) {
        throw new CheckpointManagerException(classname + " class must be concrete.", e);
    } catch (ClassNotFoundException e) {
        throw new CheckpointManagerException(classname + " class must be a class path.", e);
    }
    try {
        statefulStorage.init(topologyName, Collections.unmodifiableMap(checkpointManagerConfig.getStatefulStorageConfig()));
    } catch (StatefulStorageException e) {
        throw new CheckpointManagerException(classname + " init threw exception", e);
    }
    return statefulStorage;
}

Example 14 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class CheckpointManagerServer method handleCleanStatefulCheckpointRequest.

protected void handleCleanStatefulCheckpointRequest(REQID rid, SocketChannel channel, CheckpointManager.CleanStatefulCheckpointRequest request) {
    LOG.info(String.format("Got a clean request from %s running at host:port %s", request.toString(), channel.socket().getRemoteSocketAddress()));
    boolean deleteAll = request.hasCleanAllCheckpoints() && request.getCleanAllCheckpoints();
    Common.StatusCode statusCode = Common.StatusCode.OK;
    String errorMessage = "";
    try {
        statefulStorage.dispose(request.getOldestCheckpointPreserved(), deleteAll);
        LOG.info("Dispose checkpoint successful");
    } catch (StatefulStorageException e) {
        errorMessage = String.format("Request to dispose checkpoint failed for oldest Checkpoint " + "%s and deleteAll? %b", request.getOldestCheckpointPreserved(), deleteAll);
        statusCode = Common.StatusCode.NOTOK;
        LOG.log(Level.WARNING, errorMessage, e);
    }
    CheckpointManager.CleanStatefulCheckpointResponse.Builder responseBuilder = CheckpointManager.CleanStatefulCheckpointResponse.newBuilder();
    responseBuilder.setStatus(Common.Status.newBuilder().setStatus(statusCode).setMessage(errorMessage));
    sendResponse(rid, channel, responseBuilder.build());
}

Example 15 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class CheckpointManagerServer method handleGetInstanceStateRequest.

protected void handleGetInstanceStateRequest(REQID rid, SocketChannel channel, CheckpointManager.GetInstanceStateRequest request) {
    CheckpointInfo info = new CheckpointInfo(request.getCheckpointId(), request.getInstance());
    LOG.info(String.format("Got a get checkpoint request for checkpointId %s " + " component %s instanceId %d on connection %s", info.getCheckpointId(), info.getComponent(), info.getInstanceId(), channel.socket().getRemoteSocketAddress()));
    CheckpointManager.GetInstanceStateResponse.Builder responseBuilder = CheckpointManager.GetInstanceStateResponse.newBuilder();
    responseBuilder.setInstance(request.getInstance());
    responseBuilder.setCheckpointId(request.getCheckpointId());
    String errorMessage = "";
    Common.StatusCode statusCode = Common.StatusCode.OK;
    if (!request.hasCheckpointId() || request.getCheckpointId().isEmpty()) {
        LOG.info("The checkpoint id was empty, this sending empty state");
        CheckpointManager.InstanceStateCheckpoint dummyState = CheckpointManager.InstanceStateCheckpoint.newBuilder().setCheckpointId(request.getCheckpointId()).setState(ByteString.EMPTY).build();
        responseBuilder.setCheckpoint(dummyState);
    } else {
        try {
            Checkpoint checkpoint = statefulStorage.restoreCheckpoint(info);
            LOG.info(String.format("Get checkpoint successful for checkpointId %s " + "component %s instanceId %d", info.getCheckpointId(), info.getComponent(), info.getInstanceId()));
            // Set the checkpoint-state in response
            if (spillState) {
                CheckpointManager.InstanceStateCheckpoint ckpt = checkpoint.getCheckpoint();
                String checkpointId = ckpt.getCheckpointId();
                // clean any possible existing states
                FileUtils.cleanDir(spillStateLocation);
                // spill state to local disk
                String stateLocation = spillStateLocation + checkpointId + "-" + UUID.randomUUID();
                if (!FileUtils.writeToFile(stateLocation, ckpt.getState().toByteArray(), true)) {
                    throw new RuntimeException("failed to spill state. Bailing out...");
                }
                LOG.info("spilled state to: " + stateLocation);
                ckpt = CheckpointManager.InstanceStateCheckpoint.newBuilder().setStateLocation(stateLocation).setCheckpointId(checkpointId).build();
                responseBuilder.setCheckpoint(ckpt);
            } else {
                responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
            }
        } catch (StatefulStorageException e) {
            errorMessage = String.format("Get checkpoint not successful for checkpointId %s " + "component %s instanceId %d", info.getCheckpointId(), info.getComponent(), info.getInstanceId());
            LOG.log(Level.WARNING, errorMessage, e);
            statusCode = Common.StatusCode.NOTOK;
        }
    }
    responseBuilder.setStatus(Common.Status.newBuilder().setStatus(statusCode).setMessage(errorMessage));
    sendResponse(rid, channel, responseBuilder.build());
}