Examples with CompressionInfo io.hops.hopsworks.common.dataset.util.CompressionInfo used on opensource projects

Example 1 with CompressionInfo

use of io.hops.hopsworks.common.dataset.util.CompressionInfo in project hopsworks by logicalclocks.

the class DatasetController method unzip.

public void unzip(Project project, Users user, Path path, Path destPath) throws DatasetException {
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, user);
    checkFileExists(path, hdfsUser);
    CompressionInfo compressionInfo = new CompressionInfo(path, destPath);
    String stagingDir = settings.getStagingDir() + File.separator + compressionInfo.getStagingDirectory();
    File unzipDir = new File(stagingDir);
    unzipDir.mkdirs();
    settings.addUnzippingState(compressionInfo);
    ProcessDescriptor.Builder processDescriptorBuilder = new ProcessDescriptor.Builder().addCommand(settings.getHopsworksDomainDir() + "/bin/unzip-background.sh").addCommand(stagingDir).addCommand(path.toString()).addCommand(hdfsUser);
    if (destPath != null) {
        processDescriptorBuilder.addCommand(destPath.toString());
    }
    ProcessDescriptor processDescriptor = processDescriptorBuilder.ignoreOutErrStreams(true).build();
    try {
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        int result = processResult.getExitCode();
        if (result == 2) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_SIZE_ERROR, Level.WARNING);
        }
        if (result != 0) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_ERROR, Level.WARNING, "path: " + path.toString() + ", result: " + result);
        }
    } catch (IOException ex) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_ERROR, Level.SEVERE, "path: " + path.toString(), ex.getMessage(), ex);
    }
}

Example 2 with CompressionInfo

use of io.hops.hopsworks.common.dataset.util.CompressionInfo in project hopsworks by logicalclocks.

the class Settings method getZipState.

public synchronized String getZipState(String hdfsPath) {
    boolean zipOperation = false;
    boolean unzipOperation = false;
    CompressionInfo zipInfo = zippingFiles.stream().filter(zinfo -> zinfo.getHdfsPath().toString().equals(hdfsPath)).findAny().orElse(null);
    CompressionInfo unzipInfo = unzippingFiles.stream().filter(uzinfo -> uzinfo.getHdfsPath().toString().equals(hdfsPath)).findAny().orElse(null);
    String compressionDir = null;
    String fsmPath = null;
    if (zipInfo != null) {
        compressionDir = getStagingDir() + File.separator + zipInfo.getStagingDirectory();
        fsmPath = compressionDir + "/fsm.txt";
        zipOperation = true;
    } else if (unzipInfo != null) {
        compressionDir = getStagingDir() + File.separator + unzipInfo.getStagingDirectory();
        fsmPath = compressionDir + "/fsm.txt";
        unzipOperation = true;
    } else {
        return "NONE";
    }
    String state = "NOT_FOUND";
    try {
        state = new String(java.nio.file.Files.readAllBytes(Paths.get(fsmPath)));
        state = state.trim();
    } catch (IOException ex) {
        if (!java.nio.file.Files.exists(Paths.get(compressionDir))) {
            state = "NONE";
            // lazily remove the file, probably because it has finished zipping/unzipping
            if (zipOperation) {
                zippingFiles.remove(zipInfo);
            } else if (unzipOperation) {
                unzippingFiles.remove(unzipInfo);
            }
        }
    }
    // If a terminal state has been reached, removed the entry and the file.
    if (state.isEmpty() || state.compareTo("FAILED") == 0 || state.compareTo("SUCCESS") == 0) {
        try {
            if (zipOperation) {
                zippingFiles.remove(zipInfo);
            } else if (unzipOperation) {
                unzippingFiles.remove(unzipInfo);
            }
            java.nio.file.Files.deleteIfExists(Paths.get(fsmPath));
        } catch (IOException ex) {
            Logger.getLogger(Settings.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (state.isEmpty()) {
        state = "NOT_FOUND";
    }
    return state;
}

Example 3 with CompressionInfo

use of io.hops.hopsworks.common.dataset.util.CompressionInfo in project hopsworks by logicalclocks.

the class DatasetController method zip.

public void zip(Project project, Users user, Path path, Path destPath) throws DatasetException {
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, user);
    checkFileExists(path, hdfsUser);
    CompressionInfo compressionInfo = new CompressionInfo(path, destPath);
    String stagingDir = settings.getStagingDir() + File.separator + compressionInfo.getStagingDirectory();
    File zipDir = new File(stagingDir);
    zipDir.mkdirs();
    settings.addZippingState(compressionInfo);
    ProcessDescriptor.Builder processDescriptorBuilder = new ProcessDescriptor.Builder().addCommand(settings.getHopsworksDomainDir() + "/bin/zip-background.sh").addCommand(stagingDir).addCommand(path.toString()).addCommand(hdfsUser);
    if (destPath != null) {
        processDescriptorBuilder.addCommand(destPath.toString());
    }
    ProcessDescriptor processDescriptor = processDescriptorBuilder.ignoreOutErrStreams(true).build();
    try {
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        int result = processResult.getExitCode();
        if (result == 2) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_SIZE_ERROR, Level.WARNING);
        }
        if (result != 0) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_ERROR, Level.WARNING, "path: " + path.toString() + ", result: " + result);
        }
    } catch (IOException ex) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.COMPRESSION_ERROR, Level.SEVERE, "path: " + path.toString(), ex.getMessage(), ex);
    }
}