Examples with StatefulStorageException org.apache.heron.spi.statefulstorage.StatefulStorageException used on opensource projects

Example 6 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class LocalFileSystemStorage method cleanCheckpoints.

protected void cleanCheckpoints(File rootFile, int remaining) throws StatefulStorageException {
    if (FileUtils.isDirectoryExists(rootFile.getAbsolutePath()) && FileUtils.hasChildren(rootFile.getAbsolutePath())) {
        String[] children = rootFile.list();
        Arrays.sort(children);
        // only keep the latest N remaining files, delete others
        for (int i = 0; i < children.length - remaining; i++) {
            File ckptFile = new File(rootFile.getAbsolutePath(), children[i]);
            FileUtils.deleteDir(ckptFile, true);
            if (FileUtils.isDirectoryExists(ckptFile.getAbsolutePath())) {
                throw new StatefulStorageException("Failed to delete " + ckptFile.getAbsolutePath());
            }
        }
    }
}

Example 7 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class HDFSStorage method storeCheckpoint.

@Override
public void storeCheckpoint(CheckpointInfo info, Checkpoint checkpoint) throws StatefulStorageException {
    Path path = new Path(getCheckpointPath(info.getCheckpointId(), info.getComponent(), info.getInstanceId()));
    // We need to ensure the existence of directories structure,
    // since it is not guaranteed that FileSystem.create(..) always creates parents' dirs.
    String checkpointDir = getCheckpointDir(info.getCheckpointId(), info.getComponent());
    createDir(checkpointDir);
    FSDataOutputStream out = null;
    try {
        out = fileSystem.create(path);
        checkpoint.getCheckpoint().writeTo(out);
    } catch (IOException e) {
        throw new StatefulStorageException("Failed to persist", e);
    } finally {
        SysUtils.closeIgnoringExceptions(out);
    }
}

Example 8 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class HDFSStorage method dispose.

@Override
public void dispose(String oldestCheckpointPreserved, boolean deleteAll) throws StatefulStorageException {
    String topologyCheckpointRoot = getTopologyCheckpointRoot();
    Path topologyRootPath = new Path(topologyCheckpointRoot);
    if (deleteAll) {
        // Clean all checkpoint states
        try {
            fileSystem.delete(topologyRootPath, true);
            if (fileSystem.exists(topologyRootPath)) {
                throw new StatefulStorageException("Failed to delete " + topologyRootPath);
            }
        } catch (IOException e) {
            throw new StatefulStorageException("Error while deleting " + topologyRootPath, e);
        }
    } else {
        try {
            FileStatus[] statuses = fileSystem.listStatus(topologyRootPath);
            for (FileStatus status : statuses) {
                String name = status.getPath().getName();
                if (name.compareTo(oldestCheckpointPreserved) < 0) {
                    fileSystem.delete(status.getPath(), true);
                }
            }
            // Do a double check. Now all checkpoints with smaller checkpoint id should be cleaned
            statuses = fileSystem.listStatus(topologyRootPath);
            for (FileStatus status : statuses) {
                String name = status.getPath().getName();
                if (name.compareTo(oldestCheckpointPreserved) < 0) {
                    throw new StatefulStorageException("Error while deleting " + name);
                }
            }
        } catch (IOException e) {
            throw new StatefulStorageException("Failed to clean to: " + oldestCheckpointPreserved, e);
        }
    }
}

Example 9 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class HDFSStorage method init.

@Override
public void init(String topology, final Map<String, Object> conf) throws StatefulStorageException {
    LOG.info("Initializing... Config: " + conf.toString());
    LOG.info("Class path: " + System.getProperty("java.class.path"));
    this.topologyName = topology;
    checkpointRootPath = (String) conf.get(ROOT_PATH_KEY);
    // Notice, we pass the config folder via classpath
    // So hadoop will automatically search config files from classpath
    Configuration hadoopConfig = new Configuration();
    try {
        fileSystem = FileSystem.get(hadoopConfig);
        LOG.info("Hadoop FileSystem URI: " + fileSystem.getUri() + " ; Home Dir: " + fileSystem.getHomeDirectory());
    } catch (IOException e) {
        throw new StatefulStorageException("Failed to get hadoop file system", e);
    }
}

Example 10 with StatefulStorageException

use of org.apache.heron.spi.statefulstorage.StatefulStorageException in project heron by twitter.

the class HDFSStorage method restoreCheckpoint.

@Override
public Checkpoint restoreCheckpoint(CheckpointInfo info) throws StatefulStorageException {
    Path path = new Path(getCheckpointPath(info.getCheckpointId(), info.getComponent(), info.getInstanceId()));
    FSDataInputStream in = null;
    CheckpointManager.InstanceStateCheckpoint state = null;
    try {
        in = fileSystem.open(path);
        state = CheckpointManager.InstanceStateCheckpoint.parseFrom(in);
    } catch (IOException e) {
        throw new StatefulStorageException("Failed to read", e);
    } finally {
        SysUtils.closeIgnoringExceptions(in);
    }
    return new Checkpoint(state);
}