use of org.apache.gobblin.data.management.copy.CopyableFile in project incubator-gobblin by apache.
the class InputStreamExtractorTest method testReadRecord.
@Test
public void testReadRecord() throws Exception {
CopyableFile file = getTestCopyableFile("inputStreamExtractorTest/first.txt");
FileAwareInputStreamExtractor extractor = new FileAwareInputStreamExtractor(FileSystem.getLocal(new Configuration()), file);
FileAwareInputStream fileAwareInputStream = extractor.readRecord(null);
Assert.assertEquals(fileAwareInputStream.getFile().getOrigin().getPath(), file.getOrigin().getPath());
Assert.assertEquals(IOUtils.toString(fileAwareInputStream.getInputStream()), "first");
Assert.assertNull(extractor.readRecord(null));
}
use of org.apache.gobblin.data.management.copy.CopyableFile in project incubator-gobblin by apache.
the class FileAwareInputStreamDataWriterTest method testCommit.
@Test
public void testCommit() throws IOException {
String destinationExistingToken = "destination";
String destinationAdditionalTokens = "path";
String fileName = "file";
// Asemble destination paths
Path destination = new Path(new Path(new Path("/", destinationExistingToken), destinationAdditionalTokens), fileName);
Path destinationWithoutLeadingSeparator = new Path(new Path(destinationExistingToken, destinationAdditionalTokens), fileName);
// Create temp directory
File tmpFile = Files.createTempDir();
tmpFile.deleteOnExit();
Path tmpPath = new Path(tmpFile.getAbsolutePath());
// create origin file
Path originFile = new Path(tmpPath, fileName);
this.fs.createNewFile(originFile);
// create stating dir
Path stagingDir = new Path(tmpPath, "staging");
this.fs.mkdirs(stagingDir);
// create output dir
Path outputDir = new Path(tmpPath, "output");
this.fs.mkdirs(outputDir);
// create copyable file
FileStatus status = this.fs.getFileStatus(originFile);
FsPermission readWrite = new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE);
FsPermission dirReadWrite = new FsPermission(FsAction.ALL, FsAction.READ_WRITE, FsAction.READ_WRITE);
OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), readWrite);
List<OwnerAndPermission> ancestorOwnerAndPermissions = Lists.newArrayList();
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
Properties properties = new Properties();
properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
CopyableFile cf = CopyableFile.fromOriginAndDestination(this.fs, status, destination, CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).publishDir(new Path("/target")).preserve(PreserveAttributes.fromMnemonicString("")).build()).destinationOwnerAndPermission(ownerAndPermission).ancestorsOwnerAndPermission(ancestorOwnerAndPermissions).build();
// create work unit state
WorkUnitState state = TestUtils.createTestWorkUnitState();
state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir.toUri().getPath());
state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir.toUri().getPath());
state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5));
CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source")));
CopySource.serializeCopyEntity(state, cf);
CopySource.serializeCopyableDataset(state, metadata);
// create writer
FileAwareInputStreamDataWriter writer = new FileAwareInputStreamDataWriter(state, 1, 0);
// create output of writer.write
Path writtenFile = writer.getStagingFilePath(cf);
this.fs.mkdirs(writtenFile.getParent());
this.fs.createNewFile(writtenFile);
// create existing directories in writer output
Path outputRoot = FileAwareInputStreamDataWriter.getPartitionOutputRoot(outputDir, cf.getDatasetAndPartition(metadata));
Path existingOutputPath = new Path(outputRoot, destinationExistingToken);
this.fs.mkdirs(existingOutputPath);
FileStatus fileStatus = this.fs.getFileStatus(existingOutputPath);
FsPermission existingPathPermission = fileStatus.getPermission();
// check initial state of the relevant directories
Assert.assertTrue(this.fs.exists(existingOutputPath));
Assert.assertEquals(this.fs.listStatus(existingOutputPath).length, 0);
writer.actualProcessedCopyableFile = Optional.of(cf);
// commit
writer.commit();
// check state of relevant paths after commit
Path expectedOutputPath = new Path(outputRoot, destinationWithoutLeadingSeparator);
Assert.assertTrue(this.fs.exists(expectedOutputPath));
fileStatus = this.fs.getFileStatus(expectedOutputPath);
Assert.assertEquals(fileStatus.getOwner(), ownerAndPermission.getOwner());
Assert.assertEquals(fileStatus.getGroup(), ownerAndPermission.getGroup());
Assert.assertEquals(fileStatus.getPermission(), readWrite);
// parent should have permissions set correctly
fileStatus = this.fs.getFileStatus(expectedOutputPath.getParent());
Assert.assertEquals(fileStatus.getPermission(), dirReadWrite);
// previously existing paths should not have permissions changed
fileStatus = this.fs.getFileStatus(existingOutputPath);
Assert.assertEquals(fileStatus.getPermission(), existingPathPermission);
Assert.assertFalse(this.fs.exists(writer.stagingDir));
}
use of org.apache.gobblin.data.management.copy.CopyableFile in project incubator-gobblin by apache.
the class FileAwareInputStreamDataWriterTest method testWriteWithEncryption.
@Test
public void testWriteWithEncryption() throws Exception {
byte[] streamString = "testEncryptedContents".getBytes("UTF-8");
byte[] expectedContents = new byte[streamString.length];
for (int i = 0; i < streamString.length; i++) {
expectedContents[i] = (byte) ((streamString[i] + 1) % 256);
}
FileStatus status = fs.getFileStatus(testTempPath);
OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission);
CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source")));
WorkUnitState state = TestUtils.createTestWorkUnitState();
state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString());
state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString());
state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5));
state.setProp("writer.encrypt." + EncryptionConfigParser.ENCRYPTION_ALGORITHM_KEY, "insecure_shift");
CopySource.serializeCopyEntity(state, cf);
CopySource.serializeCopyableDataset(state, metadata);
FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0);
FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(cf, StreamUtils.convertStream(new ByteArrayInputStream(streamString)));
dataWriter.write(fileAwareInputStream);
dataWriter.commit();
Path writtenFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), cf.getDatasetAndPartition(metadata).identifier()), cf.getDestination());
Assert.assertTrue(writtenFilePath.getName().endsWith("insecure_shift"), "Expected encryption name to be appended to destination");
Assert.assertEquals(IOUtils.toByteArray(new FileInputStream(writtenFilePath.toString())), expectedContents);
}
use of org.apache.gobblin.data.management.copy.CopyableFile in project incubator-gobblin by apache.
the class TimestampBasedCopyableDatasetTest method testCopyableFileGenerator.
/**
* Test {@link TimestampBasedCopyableDataset.CopyableFileGenerator} when src location is empty and also when it is null.
*/
@Test(expectedExceptions = RuntimeException.class)
public void testCopyableFileGenerator() {
Properties props = new Properties();
props.put(TimestampBasedCopyableDataset.COPY_POLICY, TimeBasedCopyPolicyForTest.class.getName());
props.put(TimestampBasedCopyableDataset.DATASET_VERSION_FINDER, TimestampedDatasetVersionFinderForTest.class.getName());
TimestampBasedCopyableDataset copyabledataset = new TimestampBasedCopyableDataset(localFs, props, new Path("dummy"));
CopyConfiguration configuration = mock(CopyConfiguration.class);
when(configuration.getPublishDir()).thenReturn(new Path("publishDir"));
ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>();
// The src path is empty.
TimestampedDatasetVersion emptyVersion = new TimestampedDatasetVersion(new DateTime(), new Path("dummy2"));
TimestampBasedCopyableDataset.CopyableFileGenerator emptyGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, emptyVersion, copyableFileList);
emptyGenerator.run();
Assert.assertEquals(copyableFileList.size(), 0);
// The src path is null.
TimestampedDatasetVersion versionHasNullPath = new TimestampedDatasetVersion(new DateTime(), null);
TimestampBasedCopyableDataset.CopyableFileGenerator exceptionGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, versionHasNullPath, copyableFileList);
exceptionGenerator.run();
}
use of org.apache.gobblin.data.management.copy.CopyableFile in project incubator-gobblin by apache.
the class TimestampBasedCopyableDatasetTest method testGetCopyableFiles.
/**
* Test the parallel execution to get copyable files in {@link TimestampBasedCopyableDataset#getCopyableFiles(FileSystem, CopyConfiguration)}.
*/
@Test
public void testGetCopyableFiles() throws IOException {
Properties props = new Properties();
props.put(TimestampBasedCopyableDataset.COPY_POLICY, TimeBasedCopyPolicyForTest.class.getName());
props.put(TimestampBasedCopyableDataset.DATASET_VERSION_FINDER, TimestampedDatasetVersionFinderForTest.class.getName());
TimestampBasedCopyableDataset copyabledataset = new TimestampBasedCopyableDatasetForTest(localFs, props, new Path("/data/tracking/PVE"));
Collection<CopyableFile> copyableFiles = copyabledataset.getCopyableFiles(localFs, null);
/**
* {@link TimestampedDatasetVersionFinderForTest} will return three versions, and each version will contain two files.
* So the total number of copyableFiles should be 6, and all should follow the pattern: dummy\/[\\d]\*\/file[12].
*/
Assert.assertEquals(copyableFiles.size(), 6);
Pattern pattern = Pattern.compile("dummy/[\\d]*/file[12]");
Set<String> resultFilesets = Sets.newHashSet();
for (CopyableFile copyableFile : copyableFiles) {
String copyableFileset = copyableFile.getFileSet();
Assert.assertTrue(pattern.matcher(copyableFileset).matches());
resultFilesets.add(copyableFileset);
}
Assert.assertEquals(resultFilesets.size(), 6);
}
Aggregations