Examples with SmartDFSClient org.smartdata.hdfs.client.SmartDFSClient used on opensource projects

Example 1 with SmartDFSClient

use of org.smartdata.hdfs.client.SmartDFSClient in project SSM by Intel-bigdata.

the class TestCompressDecompress method init.

@Override
@Before
public void init() throws Exception {
    DEFAULT_BLOCK_SIZE = 1024 * 1024;
    super.init();
    // this.compressionImpl = "snappy";
    // this.compressionImpl = "Lz4";
    // this.compressionImpl = "Bzip2";
    this.codec = CompressionCodec.ZLIB;
    smartDFSClient = new SmartDFSClient(ssm.getContext().getConf());
}

Example 2 with SmartDFSClient

use of org.smartdata.hdfs.client.SmartDFSClient in project SSM by Intel-bigdata.

the class TestCompressDecompress method testUnsupportedMethod.

@Test
public void testUnsupportedMethod() throws Exception {
    // Concat, truncate and append are not supported
    // Create raw file
    Path path = new Path("/test/compress_files/");
    dfs.mkdirs(path);
    int rawLength = 1024 * 1024 * 8;
    String fileName = "/test/compress_files/file_0";
    DFSTestUtil.createFile(dfs, new Path(fileName), rawLength, (short) 1, 1);
    int bufSize = 1024 * 1024;
    waitTillSSMExitSafeMode();
    CmdletManager cmdletManager = ssm.getCmdletManager();
    // Compress files
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    SmartDFSClient smartDFSClient = new SmartDFSClient(smartContext.getConf());
    // Test unsupported methods on compressed file
    try {
        smartDFSClient.concat(fileName + "target", new String[] { fileName });
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("Compressed"));
    }
/*try {
      smartDFSClient.truncate(fileName, 100L);
    } catch (IOException e) {
      Assert.assertTrue(e.getMessage().contains("Compressed"));
    }*/
}

Example 3 with SmartDFSClient

use of org.smartdata.hdfs.client.SmartDFSClient in project SSM by Intel-bigdata.

the class TestCompressDecompress method testRename.

@Test
public void testRename() throws Exception {
    // Create raw file
    Path path = new Path("/test/compress_files/");
    dfs.mkdirs(path);
    int rawLength = 1024 * 1024 * 8;
    String fileName = "/test/compress_files/file_0";
    DFSTestUtil.createFile(dfs, new Path(fileName), rawLength, (short) 1, 1);
    int bufSize = 1024 * 1024;
    waitTillSSMExitSafeMode();
    CmdletManager cmdletManager = ssm.getCmdletManager();
    // Compress files
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    SmartDFSClient smartDFSClient = new SmartDFSClient(smartContext.getConf());
    smartDFSClient.rename("/test/compress_files/file_0", "/test/compress_files/file_4");
    Assert.assertTrue(smartDFSClient.exists("/test/compress_files/file_4"));
    HdfsFileStatus fileStatus = smartDFSClient.getFileInfo("/test/compress_files/file_4");
    Assert.assertEquals(rawLength, fileStatus.getLen());
}

Example 4 with SmartDFSClient

use of org.smartdata.hdfs.client.SmartDFSClient in project SSM by Intel-bigdata.

the class CmdletFactory method createAction.

public SmartAction createAction(long cmdletId, boolean isLastAction, LaunchAction launchAction) throws ActionException {
    SmartAction smartAction = ActionRegistry.createAction(launchAction.getActionType());
    smartAction.setContext(smartContext);
    smartAction.setCmdletId(cmdletId);
    smartAction.setLastAction(isLastAction);
    smartAction.init(launchAction.getArgs());
    smartAction.setActionId(launchAction.getActionId());
    if (smartAction instanceof HdfsAction) {
        try {
            ((HdfsAction) smartAction).setDfsClient(new SmartDFSClient(HadoopUtil.getNameNodeUri(smartContext.getConf()), smartContext.getConf(), getRpcServerAddress()));
        } catch (IOException e) {
            LOG.error("smartAction aid={} setDfsClient error", launchAction.getActionId(), e);
            throw new ActionException(e);
        }
    }
    /*
    else if (smartAction instanceof AlluxioAction) {
      FileSystem fs;
      try {
        fs =  AlluxioUtil.getAlluxioFs(smartContext);
      } catch (Exception e) {
        LOG.error("smartAction aid={} alluxio filesystem error", launchAction.getActionId(), e);
        throw new ActionException(e);
      }
      ((AlluxioAction) smartAction).setFileSystem(fs);
    }
    */
    return smartAction;
}

Example 5 with SmartDFSClient

use of org.smartdata.hdfs.client.SmartDFSClient in project SSM by Intel-bigdata.

the class SmartFileSystem method initialize.

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);
    this.smartDFSClient = new SmartDFSClient(conf);
}