Examples with CopyListingFileStatus org.apache.hadoop.tools.CopyListingFileStatus used on opensource projects

Example 26 with CopyListingFileStatus

use of org.apache.hadoop.tools.CopyListingFileStatus in project hadoop by apache.

the class TestCopyMapper method doTestIgnoreFailuresDoubleWrapped.

/**
   * This test covers the case where the CopyReadException is double-wrapped and
   * the mapper should be able to ignore this nested read exception.
   * @see #doTestIgnoreFailures
   */
private void doTestIgnoreFailuresDoubleWrapped(final boolean ignoreFailures) {
    try {
        deleteState();
        createSourceData();
        final UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");
        final CopyMapper copyMapper = new CopyMapper();
        final Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> context = tmpUser.doAs(new PrivilegedAction<Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text>>() {

            @Override
            public Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> run() {
                try {
                    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
                    return stubContext.getContext();
                } catch (Exception e) {
                    LOG.error("Exception encountered when get stub context", e);
                    throw new RuntimeException(e);
                }
            }
        });
        touchFile(SOURCE_PATH + "/src/file");
        mkdirs(TARGET_PATH);
        cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"), new FsPermission(FsAction.NONE, FsAction.NONE, FsAction.NONE));
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));
        context.getConfiguration().setBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), ignoreFailures);
        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {

            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered when get FileSystem.", e);
                    throw new RuntimeException(e);
                }
            }
        });
        tmpUser.doAs(new PrivilegedAction<Integer>() {

            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file"), new CopyListingFileStatus(tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file"))), context);
                    Assert.assertTrue("Should have thrown an IOException if not " + "ignoring failures", ignoreFailures);
                } catch (IOException e) {
                    LOG.error("Unexpected exception encountered. ", e);
                    Assert.assertFalse("Should not have thrown an IOException if " + "ignoring failures", ignoreFailures);
                // the IOException is not thrown again as it's expected
                } catch (Exception e) {
                    LOG.error("Exception encountered when the mapper copies file.", e);
                    throw new RuntimeException(e);
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Unexpected exception encountered. ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

Example 27 with CopyListingFileStatus

use of org.apache.hadoop.tools.CopyListingFileStatus in project hadoop by apache.

the class TestCopyMapper method testCopyReadableFiles.

@Test(timeout = 40000)
public void testCopyReadableFiles() {
    try {
        deleteState();
        createSourceData();
        UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");
        final CopyMapper copyMapper = new CopyMapper();
        final Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> context = tmpUser.doAs(new PrivilegedAction<Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text>>() {

            @Override
            public Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> run() {
                try {
                    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
                    return stubContext.getContext();
                } catch (Exception e) {
                    LOG.error("Exception encountered ", e);
                    throw new RuntimeException(e);
                }
            }
        });
        touchFile(SOURCE_PATH + "/src/file");
        mkdirs(TARGET_PATH);
        cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));
        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {

            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered ", e);
                    Assert.fail("Test failed: " + e.getMessage());
                    throw new RuntimeException("Test ought to fail here");
                }
            }
        });
        tmpUser.doAs(new PrivilegedAction<Integer>() {

            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file"), new CopyListingFileStatus(tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file"))), context);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

Example 28 with CopyListingFileStatus

use of org.apache.hadoop.tools.CopyListingFileStatus in project hadoop by apache.

the class TestDistCpUtils method testPreserveGroupOnDirectory.

@Test
public void testPreserveGroupOnDirectory() throws IOException {
    FileSystem fs = FileSystem.get(config);
    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);
    Path dst = new Path("/tmp/abc");
    Path src = new Path("/tmp/src");
    createDirectory(fs, src);
    createDirectory(fs, dst);
    fs.setPermission(src, fullPerm);
    fs.setOwner(src, "somebody", "somebody-group");
    fs.setPermission(dst, noPerm);
    fs.setOwner(dst, "nobody", "nobody-group");
    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
    // FileStatus.equals only compares path field, must explicitly compare all fields
    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
    Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
}

Example 29 with CopyListingFileStatus

use of org.apache.hadoop.tools.CopyListingFileStatus in project hadoop by apache.

the class TestDistCpUtils method testPreservePermissionOnDirectory.

@Test
public void testPreservePermissionOnDirectory() throws IOException {
    FileSystem fs = FileSystem.get(config);
    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);
    Path dst = new Path("/tmp/abc");
    Path src = new Path("/tmp/src");
    createDirectory(fs, src);
    createDirectory(fs, dst);
    fs.setPermission(src, fullPerm);
    fs.setOwner(src, "somebody", "somebody-group");
    fs.setPermission(dst, noPerm);
    fs.setOwner(dst, "nobody", "nobody-group");
    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
    // FileStatus.equals only compares path field, must explicitly compare all fields
    Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
}

Example 30 with CopyListingFileStatus

use of org.apache.hadoop.tools.CopyListingFileStatus in project hadoop by apache.

the class TestDistCpUtils method testPreserveUserOnFile.

@Test
public void testPreserveUserOnFile() throws IOException {
    FileSystem fs = FileSystem.get(config);
    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER);
    Path dst = new Path("/tmp/dest2");
    Path src = new Path("/tmp/src2");
    createFile(fs, src);
    createFile(fs, dst);
    fs.setPermission(src, fullPerm);
    fs.setOwner(src, "somebody", "somebody-group");
    fs.setTimes(src, 0, 0);
    fs.setReplication(src, (short) 1);
    fs.setPermission(dst, noPerm);
    fs.setOwner(dst, "nobody", "nobody-group");
    fs.setTimes(dst, 100, 100);
    fs.setReplication(dst, (short) 2);
    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
    // FileStatus.equals only compares path field, must explicitly compare all fields
    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
    Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}