Examples with CompressionFileState org.smartdata.model.CompressionFileState used on opensource projects

Example 16 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class TestCompressDecompress method testSubmitCompressionAction.

@Test
public void testSubmitCompressionAction() throws Exception {
    // if (!loadedNative()) {
    // return;
    // }
    waitTillSSMExitSafeMode();
    // initDB();
    int arraySize = 1024 * 1024 * 80;
    String fileName = "/ssm/compression/file1";
    byte[] bytes = prepareFile(fileName, arraySize);
    MetaStore metaStore = ssm.getMetaStore();
    int bufSize = 1024 * 1024 * 10;
    CmdletManager cmdletManager = ssm.getCmdletManager();
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    FileState fileState = null;
    // metastore  test
    int n = 0;
    while (true) {
        fileState = metaStore.getFileState(fileName);
        if (FileState.FileType.COMPRESSION.equals(fileState.getFileType())) {
            break;
        }
        Thread.sleep(1000);
        if (n++ >= 20) {
            throw new Exception("Time out in waiting for getting expect file state.");
        }
    }
    Assert.assertEquals(FileState.FileStage.DONE, fileState.getFileStage());
    Assert.assertTrue(fileState instanceof CompressionFileState);
    CompressionFileState compressionFileState = (CompressionFileState) fileState;
    Assert.assertEquals(fileName, compressionFileState.getPath());
    Assert.assertEquals(bufSize, compressionFileState.getBufferSize());
    Assert.assertEquals(codec, compressionFileState.getCompressionImpl());
    Assert.assertEquals(arraySize, compressionFileState.getOriginalLength());
    Assert.assertTrue(compressionFileState.getCompressedLength() > 0);
    Assert.assertTrue(compressionFileState.getCompressedLength() < compressionFileState.getOriginalLength());
    // data accuracy test
    byte[] input = new byte[arraySize];
    DFSInputStream dfsInputStream = smartDFSClient.open(fileName);
    int offset = 0;
    while (true) {
        int len = dfsInputStream.read(input, offset, arraySize - offset);
        if (len <= 0) {
            break;
        }
        offset += len;
    }
    Assert.assertArrayEquals("original array not equals compress/decompressed array", input, bytes);
}

Example 17 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class TestMetaStore method testDeleteFileState.

@Test
public void testDeleteFileState() throws MetaStoreException {
    // Normal file
    FileState fileState1 = new NormalFileState("/test1");
    metaStore.insertUpdateFileState(fileState1);
    Assert.assertEquals(fileState1, metaStore.getFileState("/test1"));
    // Compression & Processing (without compression info)
    // FileState fileState2 = new FileState("/test2", FileState.FileType.COMPRESSION,
    // FileState.FileStage.PROCESSING);
    // metaStore.insertUpdateFileState(fileState2);
    // Assert.assertEquals(fileState2, metaStore.getFileState("/test2"));
    // Compression & Done (with compression info)
    int bufferSize = 1024;
    long originalLen = 100;
    long compressedLen = 50;
    Long[] originPos = { 0L, 30L, 60L, 90L };
    Long[] compressedPos = { 0L, 13L, 30L, 41L };
    FileState fileState3 = new CompressionFileState("/test3", bufferSize, originalLen, compressedLen, originPos, compressedPos);
    metaStore.insertUpdateFileState(fileState3);
    compareCompressionInfo(fileState3, metaStore.getFileState("/test3"));
    // Delete /test3
    metaStore.deleteFileState("/test3");
    Assert.assertNull(metaStore.getCompressionInfo("/test3"));
    Assert.assertEquals(new NormalFileState("/test3"), metaStore.getFileState("/test3"));
    // Delete all
    metaStore.deleteAllFileState();
    Assert.assertEquals(new NormalFileState("/test1"), metaStore.getFileState("/test1"));
    Assert.assertEquals(new NormalFileState("/test2"), metaStore.getFileState("/test2"));
    Assert.assertEquals(new NormalFileState("/test3"), metaStore.getFileState("/test3"));
}

Example 18 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class TestMetaStore method compareCompressionInfo.

private void compareCompressionInfo(FileState fileState1, FileState fileState2) {
    Assert.assertEquals(fileState1, fileState2);
    Assert.assertTrue(fileState1 instanceof CompressionFileState);
    Assert.assertTrue(fileState2 instanceof CompressionFileState);
    CompressionFileState compressionFileState1 = (CompressionFileState) fileState1;
    CompressionFileState compressionFileState2 = (CompressionFileState) fileState2;
    Assert.assertEquals(compressionFileState1.getBufferSize(), compressionFileState2.getBufferSize());
    Assert.assertEquals(compressionFileState1.getOriginalLength(), compressionFileState2.getOriginalLength());
    Assert.assertEquals(compressionFileState1.getCompressedLength(), compressionFileState2.getCompressedLength());
    Assert.assertArrayEquals(compressionFileState1.getOriginalPos(), compressionFileState2.getOriginalPos());
    Assert.assertArrayEquals(compressionFileState1.getCompressedPos(), compressionFileState2.getCompressedPos());
}

Example 19 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class TestMetaStore method testInsertUpdateFileState.

@Test
public void testInsertUpdateFileState() throws MetaStoreException {
    // Normal file
    FileState fileState = new NormalFileState("/test1");
    metaStore.insertUpdateFileState(fileState);
    Assert.assertEquals(fileState, metaStore.getFileState("/test1"));
    // Compression & Processing (without compression info)
    // fileState = new FileState("/test1", FileState.FileType.COMPRESSION,
    // FileState.FileStage.PROCESSING);
    // metaStore.insertUpdateFileState(fileState);
    // Assert.assertEquals(fileState, metaStore.getFileState("/test1"));
    // Compression & Done (with compression info)
    int bufferSize = 1024;
    long originalLen = 100;
    long compressedLen = 50;
    Long[] originPos = { 0L, 30L, 60L, 90L };
    Long[] compressedPos = { 0L, 13L, 30L, 41L };
    fileState = new CompressionFileState("/test1", bufferSize, originalLen, compressedLen, originPos, compressedPos);
    metaStore.insertUpdateFileState(fileState);
    compareCompressionInfo(fileState, metaStore.getFileState("/test1"));
}

Example 20 with CompressionFileState

use of org.smartdata.model.CompressionFileState in project SSM by Intel-bigdata.

the class TestCompressionFileDao method testInsertUpdate.

@Test
public void testInsertUpdate() throws Exception {
    CompressionFileState compressionInfo = new CompressionFileState("/test", 131072, compressionImpl, originalPos.toArray(new Long[0]), compressedPos.toArray(new Long[0]));
    // insert test
    compressionFileDao.insertUpdate(compressionInfo);
    Assert.assertTrue(compressionFileDao.getInfoByPath("/test").getOriginalPos()[0].equals(9000L));
    // update test
    compressionInfo.setOriginalLength(1000);
    compressionInfo.setCompressedLength(100);
    compressionFileDao.insertUpdate(compressionInfo);
    CompressionFileState newCompressionInfo = compressionFileDao.getInfoByPath("/test");
    Assert.assertEquals(compressionInfo.getOriginalLength(), newCompressionInfo.getOriginalLength());
    Assert.assertEquals(compressionInfo.getCompressedLength(), newCompressionInfo.getCompressedLength());
}