Examples with DistCp org.apache.hadoop.tools.DistCp used on opensource projects

Example 1 with DistCp

use of org.apache.hadoop.tools.DistCp in project hadoop by apache.

the class DistCpTestUtils method assertRunDistCp.

private static void assertRunDistCp(int exitCode, String src, String dst, String[] options, Configuration conf) throws Exception {
    DistCp distCp = new DistCp(conf, null);
    String[] optsArr = new String[options.length + 2];
    System.arraycopy(options, 0, optsArr, 0, options.length);
    optsArr[optsArr.length - 2] = src;
    optsArr[optsArr.length - 1] = dst;
    assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr));
}

Example 2 with DistCp

use of org.apache.hadoop.tools.DistCp in project hive by apache.

the class Hadoop23Shims method runDistCp.

@Override
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
    DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
    options.setSyncFolder(true);
    options.setSkipCRC(true);
    options.preserve(FileAttribute.BLOCKSIZE);
    // Creates the command-line parameters for distcp
    String[] params = { "-update", "-skipcrccheck", src.toString(), dst.toString() };
    try {
        conf.setBoolean("mapred.mapper.new-api", true);
        DistCp distcp = new DistCp(conf, options);
        // added by HADOOP-10459
        if (distcp.run(params) == 0) {
            return true;
        } else {
            return false;
        }
    } catch (Exception e) {
        throw new IOException("Cannot execute DistCp process: " + e, e);
    } finally {
        conf.setBoolean("mapred.mapper.new-api", false);
    }
}

Example 3 with DistCp

use of org.apache.hadoop.tools.DistCp in project hadoop by apache.

the class AbstractContractDistCpTest method runDistCp.

/**
   * Executes DistCp and asserts that the job finished successfully.
   *
   * @param src source path
   * @param dst destination path
   * @throws Exception if there is a failure
   */
private void runDistCp(Path src, Path dst) throws Exception {
    DistCpOptions options = new DistCpOptions(Arrays.asList(src), dst);
    Job job = new DistCp(conf, options).execute();
    assertNotNull("Unexpected null job returned from DistCp execution.", job);
    assertTrue("DistCp job did not complete.", job.isComplete());
    assertTrue("DistCp job did not complete successfully.", job.isSuccessful());
}