Examples with FsShell org.apache.hadoop.fs.FsShell used on opensource projects

Example 16 with FsShell

use of org.apache.hadoop.fs.FsShell in project hadoop by apache.

the class TestDistCpSyncReverseBase method testSync.

/**
   * Test the basic functionality.
   */
@Test
public void testSync() throws Exception {
    if (isSrcNotSameAsTgt) {
        initData(source);
    }
    initData(target);
    enableAndCreateFirstSnapshot();
    final FsShell shell = new FsShell(conf);
    lsrSource("Before source: ", shell, source);
    lsr("Before target: ", shell, target);
    // make changes under target
    int numDeletedModified = changeData(target);
    createSecondSnapshotAtTarget();
    SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1");
    System.out.println(report);
    DistCpSync distCpSync = new DistCpSync(options, conf);
    lsr("Before sync target: ", shell, target);
    // do the sync
    Assert.assertTrue(distCpSync.sync());
    lsr("After sync target: ", shell, target);
    // make sure the source path has been updated to the snapshot path
    final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
    Assert.assertEquals(spath, options.getSourcePaths().get(0));
    // build copy listing
    final Path listingPath = new Path("/tmp/META/fileList.seq");
    CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
    listing.buildListing(listingPath, options);
    Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(conf, null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> context = stubContext.getContext();
    // Enable append
    context.getConfiguration().setBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), true);
    copyMapper.setup(context);
    for (Map.Entry<Text, CopyListingFileStatus> entry : copyListing.entrySet()) {
        copyMapper.map(entry.getKey(), entry.getValue(), context);
    }
    lsrSource("After mapper source: ", shell, source);
    lsr("After mapper target: ", shell, target);
    // verify that we only list modified and created files/directories
    Assert.assertEquals(numDeletedModified, copyListing.size());
    // verify that we only copied new appended data of f2 and the new file f1
    Assert.assertEquals(blockSize * 3, stubContext.getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED).getValue());
    // verify the source and target now has the same structure
    verifyCopy(dfs.getFileStatus(spath), dfs.getFileStatus(target), false);
}

Example 17 with FsShell

use of org.apache.hadoop.fs.FsShell in project hive by apache.

the class TestHdfsUtils method testSetFullFileStatusFailInheritAclsRecursive.

/**
   * Tests that {@link HdfsUtils#setFullFileStatus(Configuration, HdfsUtils.HadoopFileStatus, FileSystem, Path, boolean)}
   * does not thrown an exception when setting ACLs and with recursion.
   */
@Test
public void testSetFullFileStatusFailInheritAclsRecursive() throws Exception {
    Configuration conf = new Configuration();
    conf.set("dfs.namenode.acls.enabled", "true");
    Path fakeTarget = new Path("fakePath");
    HdfsUtils.HadoopFileStatus mockHadoopFileStatus = mock(HdfsUtils.HadoopFileStatus.class);
    FileStatus mockSourceStatus = mock(FileStatus.class);
    FsShell mockFsShell = mock(FsShell.class);
    AclStatus mockAclStatus = mock(AclStatus.class);
    when(mockSourceStatus.getPermission()).thenReturn(new FsPermission((short) 777));
    when(mockAclStatus.toString()).thenReturn("");
    when(mockHadoopFileStatus.getFileStatus()).thenReturn(mockSourceStatus);
    when(mockHadoopFileStatus.getAclEntries()).thenReturn(new ArrayList<AclEntry>());
    when(mockHadoopFileStatus.getAclStatus()).thenReturn(mockAclStatus);
    doThrow(RuntimeException.class).when(mockFsShell).run(any(String[].class));
    HdfsUtils.setFullFileStatus(conf, mockHadoopFileStatus, "", mock(FileSystem.class), fakeTarget, true, mockFsShell);
    verify(mockFsShell).run(new String[] { "-setfacl", "-R", "--set", any(String.class), fakeTarget.toString() });
}

Example 18 with FsShell

use of org.apache.hadoop.fs.FsShell in project hive by apache.

the class TestHdfsUtils method testSetFullFileStatusFailInheritGroupRecursive.

/**
   * Tests that {@link HdfsUtils#setFullFileStatus(Configuration, HdfsUtils.HadoopFileStatus, String, FileSystem, Path, boolean)}
   * does not throw an exception when setting the group and with recursion.
   */
@Test
public void testSetFullFileStatusFailInheritGroupRecursive() throws Exception {
    Configuration conf = new Configuration();
    conf.set("dfs.namenode.acls.enabled", "false");
    String fakeSourceGroup = "fakeGroup1";
    String fakeTargetGroup = "fakeGroup2";
    Path fakeTarget = new Path("fakePath");
    HdfsUtils.HadoopFileStatus mockHadoopFileStatus = mock(HdfsUtils.HadoopFileStatus.class);
    FileStatus mockSourceStatus = mock(FileStatus.class);
    FsShell mockFsShell = mock(FsShell.class);
    when(mockSourceStatus.getGroup()).thenReturn(fakeSourceGroup);
    when(mockHadoopFileStatus.getFileStatus()).thenReturn(mockSourceStatus);
    doThrow(RuntimeException.class).when(mockFsShell).run(any(String[].class));
    HdfsUtils.setFullFileStatus(conf, mockHadoopFileStatus, fakeTargetGroup, mock(FileSystem.class), fakeTarget, true, mockFsShell);
    verify(mockFsShell).run(new String[] { "-chgrp", "-R", fakeSourceGroup, fakeTarget.toString() });
}

Example 19 with FsShell

use of org.apache.hadoop.fs.FsShell in project hive by apache.

the class TestHdfsUtils method testSetFullFileStatusFailInheritPermsRecursive.

/**
   * Tests that {@link HdfsUtils#setFullFileStatus(Configuration, HdfsUtils.HadoopFileStatus, FileSystem, Path, boolean)}
   * does not thrown an exception when setting permissions and with recursion.
   */
@Test
public void testSetFullFileStatusFailInheritPermsRecursive() throws Exception {
    Configuration conf = new Configuration();
    conf.set("dfs.namenode.acls.enabled", "false");
    Path fakeTarget = new Path("fakePath");
    HdfsUtils.HadoopFileStatus mockHadoopFileStatus = mock(HdfsUtils.HadoopFileStatus.class);
    FileStatus mockSourceStatus = mock(FileStatus.class);
    FsShell mockFsShell = mock(FsShell.class);
    when(mockSourceStatus.getPermission()).thenReturn(new FsPermission((short) 777));
    when(mockHadoopFileStatus.getFileStatus()).thenReturn(mockSourceStatus);
    doThrow(RuntimeException.class).when(mockFsShell).run(any(String[].class));
    HdfsUtils.setFullFileStatus(conf, mockHadoopFileStatus, "", mock(FileSystem.class), fakeTarget, true, mockFsShell);
    verify(mockFsShell).run(new String[] { "-chmod", "-R", any(String.class), fakeTarget.toString() });
}

Example 20 with FsShell

use of org.apache.hadoop.fs.FsShell in project hive by apache.

the class DDLTask method unarchive.

private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc) throws HiveException, URISyntaxException {
    Table tbl = db.getTable(simpleDesc.getTableName());
    // Means user specified a table, not a partition
    if (simpleDesc.getPartSpec() == null) {
        throw new HiveException("UNARCHIVE is for partitions only");
    }
    if (tbl.getTableType() != TableType.MANAGED_TABLE) {
        throw new HiveException("UNARCHIVE can only be performed on managed tables");
    }
    Map<String, String> partSpec = simpleDesc.getPartSpec();
    PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
    List<Partition> partitions = db.getPartitions(tbl, partSpec);
    int partSpecLevel = partSpec.size();
    Path originalDir = null;
    // to keep backward compatibility
    if (partitions.isEmpty()) {
        throw new HiveException("No partition matches the specification");
    } else if (partSpecInfo.values.size() != tbl.getPartCols().size()) {
        // for partial specifications we need partitions to follow the scheme
        for (Partition p : partitions) {
            if (partitionInCustomLocation(tbl, p)) {
                String message = String.format("UNARCHIVE cannot run for partition " + "groups with custom locations like %s", p.getLocation());
                throw new HiveException(message);
            }
        }
        originalDir = partSpecInfo.createPath(tbl);
    } else {
        Partition p = partitions.get(0);
        if (ArchiveUtils.isArchived(p)) {
            originalDir = new Path(getOriginalLocation(p));
        } else {
            originalDir = new Path(p.getLocation());
        }
    }
    URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
    Path intermediateArchivedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
    Path intermediateExtractedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
    boolean recovery = false;
    if (pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) {
        recovery = true;
        console.printInfo("Starting recovery after failed UNARCHIVE");
    }
    for (Partition p : partitions) {
        checkArchiveProperty(partSpecLevel, recovery, p);
    }
    String archiveName = "data.har";
    FileSystem fs = null;
    try {
        fs = originalDir.getFileSystem(conf);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    // assume the archive is in the original dir, check if it exists
    Path archivePath = new Path(originalDir, archiveName);
    URI archiveUri = archivePath.toUri();
    ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf, archiveUri, originalUri);
    URI sourceUri = harHelper.getHarUri(originalUri);
    Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath());
    if (!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) {
        throw new HiveException("Haven't found any archive where it should be");
    }
    Path tmpPath = driverContext.getCtx().getExternalTmpPath(originalDir);
    try {
        fs = tmpPath.getFileSystem(conf);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    if (!pathExists(intermediateExtractedDir) && !pathExists(intermediateArchivedDir)) {
        try {
            // Copy the files out of the archive into the temporary directory
            String copySource = sourceDir.toString();
            String copyDest = tmpPath.toString();
            List<String> args = new ArrayList<String>();
            args.add("-cp");
            args.add(copySource);
            args.add(copyDest);
            console.printInfo("Copying " + copySource + " to " + copyDest);
            FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
            srcFs.initialize(sourceDir.toUri(), conf);
            FsShell fss = new FsShell(conf);
            int ret = 0;
            try {
                ret = ToolRunner.run(fss, args.toArray(new String[0]));
            } catch (Exception e) {
                e.printStackTrace();
                throw new HiveException(e);
            }
            if (ret != 0) {
                throw new HiveException("Error while copying files from archive, return code=" + ret);
            } else {
                console.printInfo("Successfully Copied " + copySource + " to " + copyDest);
            }
            console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir);
            if (fs.exists(intermediateExtractedDir)) {
                throw new HiveException("Invalid state: the intermediate extracted " + "directory already exists.");
            }
            fs.rename(tmpPath, intermediateExtractedDir);
        } catch (Exception e) {
            throw new HiveException(e);
        }
    }
    if (!pathExists(intermediateArchivedDir)) {
        try {
            console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir);
            fs.rename(originalDir, intermediateArchivedDir);
        } catch (IOException e) {
            throw new HiveException(e);
        }
    } else {
        console.printInfo(intermediateArchivedDir + " already exists. " + "Assuming it contains the archived version of the partition");
    }
    // (containing the archived version of the files) to intermediateArchiveDir
    if (!pathExists(originalDir)) {
        try {
            console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir);
            fs.rename(intermediateExtractedDir, originalDir);
        } catch (IOException e) {
            throw new HiveException(e);
        }
    } else {
        console.printInfo(originalDir + " already exists. " + "Assuming it contains the extracted files in the partition");
    }
    for (Partition p : partitions) {
        setUnArchived(p);
        try {
            db.alterPartition(simpleDesc.getTableName(), p, null);
        } catch (InvalidOperationException e) {
            throw new HiveException(e);
        }
    }
    // deleted. The user will need to call unarchive again to clear those up.
    if (pathExists(intermediateArchivedDir)) {
        deleteDir(intermediateArchivedDir);
    }
    if (recovery) {
        console.printInfo("Recovery after UNARCHIVE succeeded");
    }
    return 0;
}