use of org.apache.hadoop.tools.DistCpOptions.FileAttribute in project hadoop by apache.
the class TestDistCpUtils method testPreserveOnDirectoryUpwardRecursion.
@Test
public void testPreserveOnDirectoryUpwardRecursion() throws IOException {
FileSystem fs = FileSystem.get(config);
EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
// Remove ACL because tests run with dfs.namenode.acls.enabled false
attributes.remove(FileAttribute.ACL);
Path src = new Path("/tmp/src2");
Path f0 = new Path("/f0");
Path f1 = new Path("/d1/f1");
Path f2 = new Path("/d1/d2/f2");
Path d1 = new Path("/d1/");
Path d2 = new Path("/d1/d2/");
createFile(fs, src);
createFile(fs, f0);
createFile(fs, f1);
createFile(fs, f2);
fs.setPermission(src, almostFullPerm);
fs.setOwner(src, "somebody", "somebody-group");
fs.setTimes(src, 0, 0);
fs.setReplication(src, (short) 1);
fs.setPermission(d1, fullPerm);
fs.setOwner(d1, "anybody", "anybody-group");
fs.setTimes(d1, 400, 400);
fs.setReplication(d1, (short) 3);
fs.setPermission(d2, fullPerm);
fs.setOwner(d2, "anybody", "anybody-group");
fs.setTimes(d2, 300, 300);
fs.setReplication(d2, (short) 3);
fs.setPermission(f0, fullPerm);
fs.setOwner(f0, "anybody", "anybody-group");
fs.setTimes(f0, 200, 200);
fs.setReplication(f0, (short) 3);
fs.setPermission(f1, fullPerm);
fs.setOwner(f1, "anybody", "anybody-group");
fs.setTimes(f1, 200, 200);
fs.setReplication(f1, (short) 3);
fs.setPermission(f2, fullPerm);
fs.setOwner(f2, "anybody", "anybody-group");
fs.setTimes(f2, 200, 200);
fs.setReplication(f2, (short) 3);
CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
DistCpUtils.preserve(fs, d2, srcStatus, attributes, false);
cluster.triggerHeartbeats();
// FileStatus.equals only compares path field, must explicitly compare all fields
// attributes of src -> d2 ? should be yes
CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2));
Assert.assertTrue(srcStatus.getPermission().equals(d2Status.getPermission()));
Assert.assertTrue(srcStatus.getOwner().equals(d2Status.getOwner()));
Assert.assertTrue(srcStatus.getGroup().equals(d2Status.getGroup()));
Assert.assertTrue(srcStatus.getAccessTime() == d2Status.getAccessTime());
Assert.assertTrue(srcStatus.getModificationTime() == d2Status.getModificationTime());
Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication());
// attributes of src -> d1 ? should be no
CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1));
Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup()));
Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime());
Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime());
Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication());
// attributes of src -> f2 ? should be no
CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2));
Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup()));
Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime());
Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime());
Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication());
// attributes of src -> f1 ? should be no
CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1));
Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup()));
Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime());
Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime());
Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication());
// attributes of src -> f0 ? should be no
CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0));
Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup()));
Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime());
Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime());
Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication());
}
use of org.apache.hadoop.tools.DistCpOptions.FileAttribute in project hadoop by apache.
the class TestDistCpUtils method testPreserveTimestampOnFile.
@Test
public void testPreserveTimestampOnFile() throws IOException {
FileSystem fs = FileSystem.get(config);
EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);
Path dst = new Path("/tmp/dest2");
Path src = new Path("/tmp/src2");
createFile(fs, src);
createFile(fs, dst);
fs.setPermission(src, fullPerm);
fs.setOwner(src, "somebody", "somebody-group");
fs.setTimes(src, 0, 0);
fs.setReplication(src, (short) 1);
fs.setPermission(dst, noPerm);
fs.setOwner(dst, "nobody", "nobody-group");
fs.setTimes(dst, 100, 100);
fs.setReplication(dst, (short) 2);
CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
// FileStatus.equals only compares path field, must explicitly compare all fields
Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
use of org.apache.hadoop.tools.DistCpOptions.FileAttribute in project hadoop by apache.
the class TestDistCpUtils method testPreserveReplicationOnDirectory.
@Test
public void testPreserveReplicationOnDirectory() throws IOException {
FileSystem fs = FileSystem.get(config);
EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);
Path dst = new Path("/tmp/abc");
Path src = new Path("/tmp/src");
createDirectory(fs, src);
createDirectory(fs, dst);
fs.setPermission(src, fullPerm);
fs.setOwner(src, "somebody", "somebody-group");
fs.setReplication(src, (short) 1);
fs.setPermission(dst, noPerm);
fs.setOwner(dst, "nobody", "nobody-group");
fs.setReplication(dst, (short) 2);
CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
// FileStatus.equals only compares path field, must explicitly compare all fields
Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
// Replication shouldn't apply to dirs so this should still be 0 == 0
Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
use of org.apache.hadoop.tools.DistCpOptions.FileAttribute in project hadoop by apache.
the class CopyCommitter method preserveFileAttributesForDirectories.
// This method changes the target-directories' file-attributes (owner,
// user/group permissions, etc.) based on the corresponding source directories.
private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
final boolean syncOrOverwrite = syncFolder || overwrite;
LOG.info("About to preserve attributes: " + attrSymbols);
EnumSet<FileAttribute> attributes = DistCpUtils.unpackAttributes(attrSymbols);
final boolean preserveRawXattrs = conf.getBoolean(DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, false);
Path sourceListing = new Path(conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH));
FileSystem clusterFS = sourceListing.getFileSystem(conf);
SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(sourceListing));
long totalLen = clusterFS.getFileStatus(sourceListing).getLen();
Path targetRoot = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
long preservedEntries = 0;
try {
CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
Text srcRelPath = new Text();
// Iterate over every source path that was copied.
while (sourceReader.next(srcRelPath, srcFileStatus)) {
// in the map-task.
if (!srcFileStatus.isDirectory())
continue;
Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
//
if (targetRoot.equals(targetFile) && syncOrOverwrite)
continue;
FileSystem targetFS = targetFile.getFileSystem(conf);
DistCpUtils.preserve(targetFS, targetFile, srcFileStatus, attributes, preserveRawXattrs);
taskAttemptContext.progress();
taskAttemptContext.setStatus("Preserving status on directory entries. [" + sourceReader.getPosition() * 100 / totalLen + "%]");
}
} finally {
IOUtils.closeStream(sourceReader);
}
LOG.info("Preserved status on " + preservedEntries + " dir entries on target");
}
use of org.apache.hadoop.tools.DistCpOptions.FileAttribute in project hadoop by apache.
the class DistCpUtils method packAttributes.
/**
* Pack file preservation attributes into a string, containing
* just the first character of each preservation attribute
* @param attributes - Attribute set to preserve
* @return - String containing first letters of each attribute to preserve
*/
public static String packAttributes(EnumSet<FileAttribute> attributes) {
StringBuffer buffer = new StringBuffer(FileAttribute.values().length);
int len = 0;
for (FileAttribute attribute : attributes) {
buffer.append(attribute.name().charAt(0));
len++;
}
return buffer.substring(0, len);
}
Aggregations