Examples with CompressionFileState org.smartdata.model.CompressionFileState used on opensource projects

Example 11 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class DecompressionAction method execute.

protected void execute() throws Exception {
    if (filePath == null) {
        throw new IllegalArgumentException("File path is missing.");
    }
    if (compressTmpPath == null) {
        throw new IllegalArgumentException("Compression tmp path is not specified!");
    }
    if (!dfsClient.exists(filePath)) {
        throw new ActionException("Failed to execute Compression Action: the given file doesn't exist!");
    }
    // Consider directory case.
    if (dfsClient.getFileInfo(filePath).isDir()) {
        appendLog("Decompression is not applicable to a directory.");
        return;
    }
    FileState fileState = HadoopUtil.getFileState(dfsClient, filePath);
    if (!(fileState instanceof CompressionFileState)) {
        appendLog("The file is already decompressed!");
        return;
    }
    OutputStream out = null;
    InputStream in = null;
    try {
        // No need to lock the file by append operation,
        // since compressed file cannot be modified.
        out = dfsClient.create(compressTmpPath, true);
        // Keep storage policy consistent.
        // The below statement is not supported on Hadoop-2.7.3 or CDH-5.10.1
        // String storagePolicyName = dfsClient.getStoragePolicy(filePath).getName();
        byte storagePolicyId = dfsClient.getFileInfo(filePath).getStoragePolicy();
        String storagePolicyName = SmartConstants.STORAGE_POLICY_MAP.get(storagePolicyId);
        if (!storagePolicyName.equals("UNDEF")) {
            dfsClient.setStoragePolicy(compressTmpPath, storagePolicyName);
        }
        in = dfsClient.open(filePath);
        long length = dfsClient.getFileInfo(filePath).getLen();
        outputDecompressedData(in, out, length);
        // Overwrite the original file with decompressed data
        dfsClient.setOwner(compressTmpPath, dfsClient.getFileInfo(filePath).getOwner(), dfsClient.getFileInfo(filePath).getGroup());
        dfsClient.setPermission(compressTmpPath, dfsClient.getFileInfo(filePath).getPermission());
        dfsClient.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE);
        appendLog("The given file is successfully decompressed by codec: " + ((CompressionFileState) fileState).getCompressionImpl());
    } catch (IOException e) {
        throw new IOException(e);
    } finally {
        if (out != null) {
            out.close();
        }
        if (in != null) {
            in.close();
        }
    }
}

Example 12 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class CompressionScheduler method onSubmit.

@Override
public boolean onSubmit(CmdletInfo cmdletInfo, ActionInfo actionInfo, int actionIndex) {
    String srcPath = actionInfo.getArgs().get(HdfsAction.FILE_PATH);
    if (!actions.contains(actionInfo.getActionName())) {
        return false;
    }
    if (fileLock.contains(srcPath)) {
        return false;
    }
    try {
        if (actionInfo.getActionName().equals(COMPRESSION_ACTION_ID) && !supportCompression(srcPath)) {
            return false;
        }
        if (actionInfo.getActionName().equals(DECOMPRESSION_ACTION_ID) && !supportDecompression(srcPath)) {
            return false;
        }
        // TODO remove this part
        CompressionFileState fileState = new CompressionFileState(srcPath, FileState.FileStage.PROCESSING);
        metaStore.insertUpdateFileState(fileState);
        return true;
    } catch (MetaStoreException e) {
        LOG.error("Failed to submit action due to metastore exception!", e);
        return false;
    } catch (IOException e) {
        LOG.error(e.getMessage());
        return false;
    }
}

Example 13 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class CompressionScheduler method supportDecompression.

public boolean supportDecompression(String path) throws MetaStoreException, IOException {
    if (path == null) {
        LOG.warn("File path is not specified!");
        return false;
    }
    // Exclude directory case
    if (dfsClient.getFileInfo(path).isDir()) {
        LOG.warn("Decompression is not applicable to a directory.");
        return false;
    }
    FileState fileState = metaStore.getFileState(path);
    if (fileState instanceof CompressionFileState) {
        return true;
    }
    LOG.debug("A compressed file path should be given!");
    return false;
}

Example 14 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class SmartDFSClient method getBlockLocations.

@Override
public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException {
    BlockLocation[] blockLocations = super.getBlockLocations(src, start, length);
    if (blockLocations.length == 0) {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompactFileState) {
            String containerFile = ((CompactFileState) fileState).getFileContainerInfo().getContainerFilePath();
            long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset();
            blockLocations = super.getBlockLocations(containerFile, offset + start, length);
            for (BlockLocation blockLocation : blockLocations) {
                blockLocation.setOffset(blockLocation.getOffset() - offset);
            }
            return blockLocations;
        }
    } else {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompressionFileState) {
            CompressionFileState compressionInfo = (CompressionFileState) fileState;
            Long[] originalPos = compressionInfo.getOriginalPos().clone();
            Long[] compressedPos = compressionInfo.getCompressedPos().clone();
            int startIndex = compressionInfo.getPosIndexByOriginalOffset(start);
            int endIndex = compressionInfo.getPosIndexByOriginalOffset(start + length - 1);
            long compressedStart = compressedPos[startIndex];
            long compressedLength = 0;
            if (endIndex < compressedPos.length - 1) {
                compressedLength = compressedPos[endIndex + 1] - compressedStart;
            } else {
                compressedLength = compressionInfo.getCompressedLength() - compressedStart;
            }
            LocatedBlocks originalLocatedBlocks = super.getLocatedBlocks(src, compressedStart, compressedLength);
            List<LocatedBlock> blocks = new ArrayList<>();
            for (LocatedBlock block : originalLocatedBlocks.getLocatedBlocks()) {
                // TODO handle CDH2.6 storage type
                // blocks.add(new LocatedBlock(
                // block.getBlock(),
                // block.getLocations(),
                // block.getStorageIDs(),
                // block.getStorageTypes(),
                // compressionInfo
                // .getPosIndexByCompressedOffset(block.getStartOffset()),
                // block.isCorrupt(),
                // block.getCachedLocations()
                // ));
                blocks.add(new LocatedBlock(block.getBlock(), block.getLocations(), compressionInfo.getPosIndexByCompressedOffset(block.getStartOffset()), block.isCorrupt()));
            }
            LocatedBlock lastLocatedBlock = originalLocatedBlocks.getLastLocatedBlock();
            long fileLength = compressionInfo.getOriginalLength();
            return new LocatedBlocks(fileLength, originalLocatedBlocks.isUnderConstruction(), blocks, lastLocatedBlock, originalLocatedBlocks.isLastBlockComplete(), originalLocatedBlocks.getFileEncryptionInfo()).getLocatedBlocks().toArray(new BlockLocation[0]);
        }
    }
    return blockLocations;
}

Example 15 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class CompressionAction method execute.

@Override
protected void execute() throws Exception {
    if (filePath == null) {
        throw new IllegalArgumentException("File path is missing.");
    }
    if (compressTmpPath == null) {
        throw new IllegalArgumentException("Compression tmp path is not specified!");
    }
    if (!compressionCodecList.contains(compressCodec)) {
        throw new ActionException("Compression Action failed due to unsupported codec: " + compressCodec);
    }
    appendLog(String.format("Compression Action started at %s for %s", Utils.getFormatedCurrentTime(), filePath));
    if (!dfsClient.exists(filePath)) {
        throw new ActionException("Failed to execute Compression Action: the given file doesn't exist!");
    }
    HdfsFileStatus srcFileStatus = dfsClient.getFileInfo(filePath);
    // Consider directory case.
    if (srcFileStatus.isDir()) {
        appendLog("Compression is not applicable to a directory.");
        return;
    }
    // Generate compressed file
    compressionFileState = new CompressionFileState(filePath, bufferSize, compressCodec);
    compressionFileState.setOriginalLength(srcFileStatus.getLen());
    OutputStream appendOut = null;
    DFSInputStream in = null;
    OutputStream out = null;
    try {
        if (srcFileStatus.getLen() == 0) {
            compressionFileInfo = new CompressionFileInfo(false, compressionFileState);
        } else {
            short replication = srcFileStatus.getReplication();
            long blockSize = srcFileStatus.getBlockSize();
            long fileSize = srcFileStatus.getLen();
            appendLog("File length: " + fileSize);
            bufferSize = getActualBuffSize(fileSize);
            // SmartDFSClient will fail to open compressing file with PROCESSING FileStage
            // set by Compression scheduler. But considering DfsClient may be used, we use
            // append operation to lock the file to avoid any modification.
            appendOut = CompatibilityHelperLoader.getHelper().getDFSClientAppend(dfsClient, filePath, bufferSize);
            in = dfsClient.open(filePath);
            out = dfsClient.create(compressTmpPath, true, replication, blockSize);
            // Keep storage policy consistent.
            // The below statement is not supported on Hadoop-2.7.3 or CDH-5.10.1
            // String storagePolicyName = dfsClient.getStoragePolicy(filePath).getName();
            byte storagePolicyId = srcFileStatus.getStoragePolicy();
            String storagePolicyName = SmartConstants.STORAGE_POLICY_MAP.get(storagePolicyId);
            if (!storagePolicyName.equals("UNDEF")) {
                dfsClient.setStoragePolicy(compressTmpPath, storagePolicyName);
            }
            compress(in, out);
            HdfsFileStatus destFileStatus = dfsClient.getFileInfo(compressTmpPath);
            dfsClient.setOwner(compressTmpPath, srcFileStatus.getOwner(), srcFileStatus.getGroup());
            dfsClient.setPermission(compressTmpPath, srcFileStatus.getPermission());
            compressionFileState.setCompressedLength(destFileStatus.getLen());
            appendLog("Compressed file length: " + destFileStatus.getLen());
            compressionFileInfo = new CompressionFileInfo(true, compressTmpPath, compressionFileState);
        }
        compressionFileState.setBufferSize(bufferSize);
        appendLog("Compression buffer size: " + bufferSize);
        appendLog("Compression codec: " + compressCodec);
        String compressionInfoJson = new Gson().toJson(compressionFileInfo);
        appendResult(compressionInfoJson);
        LOG.warn(compressionInfoJson);
        if (compressionFileInfo.needReplace()) {
            // Add to temp path
            // Please make sure content write to Xatte is less than 64K
            dfsClient.setXAttr(compressionFileInfo.getTempPath(), XATTR_NAME, SerializationUtils.serialize(compressionFileState), EnumSet.of(XAttrSetFlag.CREATE));
            // Rename operation is moved from CompressionScheduler.
            // Thus, modification for original file will be avoided.
            dfsClient.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE);
        } else {
            // Add to raw path
            dfsClient.setXAttr(filePath, XATTR_NAME, SerializationUtils.serialize(compressionFileState), EnumSet.of(XAttrSetFlag.CREATE));
        }
    } catch (IOException e) {
        throw new IOException(e);
    } finally {
        if (appendOut != null) {
            try {
                appendOut.close();
            } catch (IOException e) {
            // Hide the expected exception that the original file is missing.
            }
        }
        if (in != null) {
            in.close();
        }
        if (out != null) {
            out.close();
        }
    }
}