Examples with DistCpOptions org.apache.hadoop.tools.DistCpOptions used on opensource projects

Example 6 with DistCpOptions

use of org.apache.hadoop.tools.DistCpOptions in project hadoop by apache.

the class TestCopyCommitter method testDeleteMissingFlatInterleavedFiles.

@Test
public void testDeleteMissingFlatInterleavedFiles() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(), taskAttemptContext.getTaskAttemptID().getJobID());
    Configuration conf = jobContext.getConfiguration();
    String sourceBase;
    String targetBase;
    FileSystem fs = null;
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        sourceBase = "/tmp1/" + String.valueOf(rand.nextLong());
        targetBase = "/tmp1/" + String.valueOf(rand.nextLong());
        TestDistCpUtils.createFile(fs, sourceBase + "/1");
        TestDistCpUtils.createFile(fs, sourceBase + "/3");
        TestDistCpUtils.createFile(fs, sourceBase + "/4");
        TestDistCpUtils.createFile(fs, sourceBase + "/5");
        TestDistCpUtils.createFile(fs, sourceBase + "/7");
        TestDistCpUtils.createFile(fs, sourceBase + "/8");
        TestDistCpUtils.createFile(fs, sourceBase + "/9");
        TestDistCpUtils.createFile(fs, targetBase + "/2");
        TestDistCpUtils.createFile(fs, targetBase + "/4");
        TestDistCpUtils.createFile(fs, targetBase + "/5");
        TestDistCpUtils.createFile(fs, targetBase + "/7");
        TestDistCpUtils.createFile(fs, targetBase + "/9");
        TestDistCpUtils.createFile(fs, targetBase + "/A");
        DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), new Path("/out"));
        options.setSyncFolder(true);
        options.setDeleteMissing(true);
        options.appendToConf(conf);
        CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
        Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
        listing.buildListing(listingFile, options);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);
        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);
        //Test for idempotent commit
        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);
    } catch (IOException e) {
        LOG.error("Exception encountered while testing for delete missing", e);
        Assert.fail("Delete missing failure");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp1");
        conf.set(DistCpConstants.CONF_LABEL_DELETE_MISSING, "false");
    }
}

Example 7 with DistCpOptions

use of org.apache.hadoop.tools.DistCpOptions in project hadoop by apache.

the class TestOptionsParser method testParseSkipCRC.

@Test
public void testParseSkipCRC() {
    DistCpOptions options = OptionsParser.parse(new String[] { "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertFalse(options.shouldSkipCRC());
    options = OptionsParser.parse(new String[] { "-update", "-skipcrccheck", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertTrue(options.shouldSyncFolder());
    Assert.assertTrue(options.shouldSkipCRC());
}

Example 8 with DistCpOptions

use of org.apache.hadoop.tools.DistCpOptions in project hadoop by apache.

the class TestOptionsParser method testSnapshotDiffOption.

// Test -diff or -rdiff
private void testSnapshotDiffOption(boolean isDiff) {
    final String optionStr = isDiff ? "-diff" : "-rdiff";
    final String optionLabel = isDiff ? DistCpOptionSwitch.DIFF.getConfigLabel() : DistCpOptionSwitch.RDIFF.getConfigLabel();
    Configuration conf = new Configuration();
    Assert.assertFalse(conf.getBoolean(optionLabel, false));
    DistCpOptions options = OptionsParser.parse(new String[] { "-update", optionStr, "s1", "s2", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    options.appendToConf(conf);
    Assert.assertTrue(conf.getBoolean(optionLabel, false));
    Assert.assertTrue(isDiff ? options.shouldUseDiff() : options.shouldUseRdiff());
    Assert.assertEquals("s1", options.getFromSnapshot());
    Assert.assertEquals("s2", options.getToSnapshot());
    options = OptionsParser.parse(new String[] { optionStr, "s1", ".", "-update", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    options.appendToConf(conf);
    Assert.assertTrue(conf.getBoolean(optionLabel, false));
    Assert.assertTrue(isDiff ? options.shouldUseDiff() : options.shouldUseRdiff());
    Assert.assertEquals("s1", options.getFromSnapshot());
    Assert.assertEquals(".", options.getToSnapshot());
    // -diff/-rdiff requires two option values
    try {
        OptionsParser.parse(new String[] { optionStr, "s1", "-update", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        fail(optionStr + " should fail with only one snapshot name");
    } catch (IllegalArgumentException e) {
        GenericTestUtils.assertExceptionContains("Must provide both the starting and ending snapshot names", e);
    }
    // make sure -diff/-rdiff is only valid when -update is specified
    try {
        OptionsParser.parse(new String[] { optionStr, "s1", "s2", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        fail(optionStr + " should fail if -update option is not specified");
    } catch (IllegalArgumentException e) {
        GenericTestUtils.assertExceptionContains("-diff/-rdiff is valid only with -update option", e);
    }
    try {
        options = OptionsParser.parse(new String[] { optionStr, "s1", "s2", "-update", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        assertFalse("-delete should be ignored when " + optionStr + " is specified", options.shouldDeleteMissing());
    } catch (IllegalArgumentException e) {
        fail("Got unexpected IllegalArgumentException: " + e.getMessage());
    }
    try {
        options = OptionsParser.parse(new String[] { optionStr, "s1", "s2", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        fail(optionStr + " should fail if -update option is not specified");
    } catch (IllegalArgumentException e) {
        assertFalse("-delete should be ignored when -diff is specified", options.shouldDeleteMissing());
        GenericTestUtils.assertExceptionContains("-diff/-rdiff is valid only with -update option", e);
    }
    try {
        OptionsParser.parse(new String[] { optionStr, "s1", "s2", "-delete", "-overwrite", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        fail(optionStr + " should fail if -update option is not specified");
    } catch (IllegalArgumentException e) {
        GenericTestUtils.assertExceptionContains("-diff/-rdiff is valid only with -update option", e);
    }
    final String optionStrOther = isDiff ? "-rdiff" : "-diff";
    try {
        OptionsParser.parse(new String[] { optionStr, "s1", "s2", optionStrOther, "s2", "s1", "-update", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        fail(optionStr + " should fail if " + optionStrOther + " is also specified");
    } catch (IllegalArgumentException e) {
        GenericTestUtils.assertExceptionContains("-diff and -rdiff are mutually exclusive", e);
    }
}

Example 9 with DistCpOptions

use of org.apache.hadoop.tools.DistCpOptions in project hadoop by apache.

the class TestOptionsParser method testParseDeleteMissing.

@Test
public void testParseDeleteMissing() {
    DistCpOptions options = OptionsParser.parse(new String[] { "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertFalse(options.shouldDeleteMissing());
    options = OptionsParser.parse(new String[] { "-update", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertTrue(options.shouldSyncFolder());
    Assert.assertTrue(options.shouldDeleteMissing());
    options = OptionsParser.parse(new String[] { "-overwrite", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertTrue(options.shouldOverwrite());
    Assert.assertTrue(options.shouldDeleteMissing());
    try {
        OptionsParser.parse(new String[] { "-atomic", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
        Assert.fail("Atomic and delete folders were allowed");
    } catch (IllegalArgumentException ignore) {
    }
}

Example 10 with DistCpOptions

use of org.apache.hadoop.tools.DistCpOptions in project hadoop by apache.

the class TestOptionsParser method testSourceListing.

@Test
public void testSourceListing() {
    DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" });
    Assert.assertEquals(options.getSourceFileListing(), new Path("hdfs://localhost:9820/source/first"));
}