Examples with FileState org.smartdata.model.FileState used on opensource projects

Example 26 with FileState

use of org.smartdata.model.FileState in project SSM by Intel-bigdata.

the class SmartDFSClient method getBlockLocations.

@Override
public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException {
    BlockLocation[] blockLocations = super.getBlockLocations(src, start, length);
    if (blockLocations.length == 0) {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompactFileState) {
            String containerFile = ((CompactFileState) fileState).getFileContainerInfo().getContainerFilePath();
            long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset();
            blockLocations = super.getBlockLocations(containerFile, offset + start, length);
            for (BlockLocation blockLocation : blockLocations) {
                blockLocation.setOffset(blockLocation.getOffset() - offset);
            }
            return blockLocations;
        }
    } else {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompressionFileState) {
            CompressionFileState compressionInfo = (CompressionFileState) fileState;
            Long[] originalPos = compressionInfo.getOriginalPos().clone();
            Long[] compressedPos = compressionInfo.getCompressedPos().clone();
            int startIndex = compressionInfo.getPosIndexByOriginalOffset(start);
            int endIndex = compressionInfo.getPosIndexByOriginalOffset(start + length - 1);
            long compressedStart = compressedPos[startIndex];
            long compressedLength = 0;
            if (endIndex < compressedPos.length - 1) {
                compressedLength = compressedPos[endIndex + 1] - compressedStart;
            } else {
                compressedLength = compressionInfo.getCompressedLength() - compressedStart;
            }
            LocatedBlocks originalLocatedBlocks = super.getLocatedBlocks(src, compressedStart, compressedLength);
            List<LocatedBlock> blocks = new ArrayList<>();
            for (LocatedBlock block : originalLocatedBlocks.getLocatedBlocks()) {
                // TODO handle CDH2.6 storage type
                // blocks.add(new LocatedBlock(
                // block.getBlock(),
                // block.getLocations(),
                // block.getStorageIDs(),
                // block.getStorageTypes(),
                // compressionInfo
                // .getPosIndexByCompressedOffset(block.getStartOffset()),
                // block.isCorrupt(),
                // block.getCachedLocations()
                // ));
                blocks.add(new LocatedBlock(block.getBlock(), block.getLocations(), compressionInfo.getPosIndexByCompressedOffset(block.getStartOffset()), block.isCorrupt()));
            }
            LocatedBlock lastLocatedBlock = originalLocatedBlocks.getLastLocatedBlock();
            long fileLength = compressionInfo.getOriginalLength();
            return new LocatedBlocks(fileLength, originalLocatedBlocks.isUnderConstruction(), blocks, lastLocatedBlock, originalLocatedBlocks.isLastBlockComplete(), originalLocatedBlocks.getFileEncryptionInfo()).getLocatedBlocks().toArray(new BlockLocation[0]);
        }
    }
    return blockLocations;
}

Example 27 with FileState

use of org.smartdata.model.FileState in project SSM by Intel-bigdata.

the class SmartDFSClient method getFileLinkInfo.

@Override
public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
    HdfsFileStatus fileStatus = super.getFileLinkInfo(src);
    if (fileStatus.getLen() == 0) {
        String target = super.getLinkTarget(src);
        FileState fileState = getFileState(target);
        if (fileState instanceof CompactFileState) {
            fileStatus = getFileInfo(target);
        }
    }
    return fileStatus;
}

Example 28 with FileState

use of org.smartdata.model.FileState in project SSM by Intel-bigdata.

the class TestCompressDecompress method testSubmitCompressionAction.

@Test
public void testSubmitCompressionAction() throws Exception {
    // if (!loadedNative()) {
    // return;
    // }
    waitTillSSMExitSafeMode();
    // initDB();
    int arraySize = 1024 * 1024 * 80;
    String fileName = "/ssm/compression/file1";
    byte[] bytes = prepareFile(fileName, arraySize);
    MetaStore metaStore = ssm.getMetaStore();
    int bufSize = 1024 * 1024 * 10;
    CmdletManager cmdletManager = ssm.getCmdletManager();
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    FileState fileState = null;
    // metastore  test
    int n = 0;
    while (true) {
        fileState = metaStore.getFileState(fileName);
        if (FileState.FileType.COMPRESSION.equals(fileState.getFileType())) {
            break;
        }
        Thread.sleep(1000);
        if (n++ >= 20) {
            throw new Exception("Time out in waiting for getting expect file state.");
        }
    }
    Assert.assertEquals(FileState.FileStage.DONE, fileState.getFileStage());
    Assert.assertTrue(fileState instanceof CompressionFileState);
    CompressionFileState compressionFileState = (CompressionFileState) fileState;
    Assert.assertEquals(fileName, compressionFileState.getPath());
    Assert.assertEquals(bufSize, compressionFileState.getBufferSize());
    Assert.assertEquals(codec, compressionFileState.getCompressionImpl());
    Assert.assertEquals(arraySize, compressionFileState.getOriginalLength());
    Assert.assertTrue(compressionFileState.getCompressedLength() > 0);
    Assert.assertTrue(compressionFileState.getCompressedLength() < compressionFileState.getOriginalLength());
    // data accuracy test
    byte[] input = new byte[arraySize];
    DFSInputStream dfsInputStream = smartDFSClient.open(fileName);
    int offset = 0;
    while (true) {
        int len = dfsInputStream.read(input, offset, arraySize - offset);
        if (len <= 0) {
            break;
        }
        offset += len;
    }
    Assert.assertArrayEquals("original array not equals compress/decompressed array", input, bytes);
}

Example 29 with FileState

use of org.smartdata.model.FileState in project SSM by Intel-bigdata.

the class TestSmartClient method testGetFileState.

@Test
public void testGetFileState() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    String path = "/file1";
    FileState fileState = new NormalFileState(path);
    SmartClient client = new SmartClient(smartContext.getConf());
    FileState fileState1;
    // No entry in file_state table (Normal type as default)
    fileState1 = client.getFileState(path);
    Assert.assertEquals(fileState, fileState1);
    metaStore.insertUpdateFileState(fileState);
    fileState1 = client.getFileState(path);
    Assert.assertEquals(fileState, fileState1);
}

Example 30 with FileState

use of org.smartdata.model.FileState in project SSM by Intel-bigdata.

the class TestMetaStore method testDeleteFileState.

@Test
public void testDeleteFileState() throws MetaStoreException {
    // Normal file
    FileState fileState1 = new NormalFileState("/test1");
    metaStore.insertUpdateFileState(fileState1);
    Assert.assertEquals(fileState1, metaStore.getFileState("/test1"));
    // Compression & Processing (without compression info)
    // FileState fileState2 = new FileState("/test2", FileState.FileType.COMPRESSION,
    // FileState.FileStage.PROCESSING);
    // metaStore.insertUpdateFileState(fileState2);
    // Assert.assertEquals(fileState2, metaStore.getFileState("/test2"));
    // Compression & Done (with compression info)
    int bufferSize = 1024;
    long originalLen = 100;
    long compressedLen = 50;
    Long[] originPos = { 0L, 30L, 60L, 90L };
    Long[] compressedPos = { 0L, 13L, 30L, 41L };
    FileState fileState3 = new CompressionFileState("/test3", bufferSize, originalLen, compressedLen, originPos, compressedPos);
    metaStore.insertUpdateFileState(fileState3);
    compareCompressionInfo(fileState3, metaStore.getFileState("/test3"));
    // Delete /test3
    metaStore.deleteFileState("/test3");
    Assert.assertNull(metaStore.getCompressionInfo("/test3"));
    Assert.assertEquals(new NormalFileState("/test3"), metaStore.getFileState("/test3"));
    // Delete all
    metaStore.deleteAllFileState();
    Assert.assertEquals(new NormalFileState("/test1"), metaStore.getFileState("/test1"));
    Assert.assertEquals(new NormalFileState("/test2"), metaStore.getFileState("/test2"));
    Assert.assertEquals(new NormalFileState("/test3"), metaStore.getFileState("/test3"));
}