Examples with SmartFileSystem org.smartdata.hadoop.filesystem.SmartFileSystem used on opensource projects

Example 1 with SmartFileSystem

use of org.smartdata.hadoop.filesystem.SmartFileSystem in project SSM by Intel-bigdata.

the class TestCompressDecompress method testListLocatedStatus.

@Test
public void testListLocatedStatus() throws Exception {
    // if (!loadedNative()) {
    // return;
    // }
    waitTillSSMExitSafeMode();
    // initDB();
    SmartFileSystem smartDfs = new SmartFileSystem();
    smartDfs.initialize(dfs.getUri(), ssm.getContext().getConf());
    int arraySize = 1024 * 1024 * 8;
    String fileName = "/ssm/compression/file4";
    byte[] bytes = prepareFile(fileName, arraySize);
    // For uncompressed file, SmartFileSystem and DistributedFileSystem behave exactly the same
    RemoteIterator<LocatedFileStatus> iter1 = dfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat1 = iter1.next();
    RemoteIterator<LocatedFileStatus> iter2 = smartDfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat2 = iter2.next();
    Assert.assertEquals(stat1.getPath(), stat2.getPath());
    Assert.assertEquals(stat1.getBlockSize(), stat2.getBlockSize());
    Assert.assertEquals(stat1.getLen(), stat2.getLen());
    BlockLocation[] blockLocations1 = stat1.getBlockLocations();
    BlockLocation[] blockLocations2 = stat2.getBlockLocations();
    Assert.assertEquals(blockLocations1.length, blockLocations2.length);
    for (int i = 0; i < blockLocations1.length; i++) {
        Assert.assertEquals(blockLocations1[i].getLength(), blockLocations2[i].getLength());
        Assert.assertEquals(blockLocations1[i].getOffset(), blockLocations2[i].getOffset());
    }
    // Test compressed file
    int bufSize = 1024 * 1024;
    CmdletManager cmdletManager = ssm.getCmdletManager();
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    RemoteIterator<LocatedFileStatus> iter3 = dfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat3 = iter3.next();
    BlockLocation[] blockLocations3 = stat3.getBlockLocations();
    RemoteIterator<LocatedFileStatus> iter4 = smartDfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat4 = iter4.next();
    BlockLocation[] blockLocations4 = stat4.getBlockLocations();
    Assert.assertEquals(stat1.getPath(), stat4.getPath());
    Assert.assertEquals(stat1.getBlockSize(), stat4.getBlockSize());
    Assert.assertEquals(stat1.getLen(), stat4.getLen());
}