Examples with FileSetDataset co.cask.cdap.data2.dataset2.lib.file.FileSetDataset used on opensource projects

Example 1 with FileSetDataset

use of co.cask.cdap.data2.dataset2.lib.file.FileSetDataset in project cdap by caskdata.

the class PartitionedFileSetDataset method onSuccess.

@Override
public void onSuccess() throws DataSetException {
    String outputPath = FileSetArguments.getOutputPath(runtimeArguments);
    // Either way, we can't do much here.
    if (outputPath == null) {
        return;
    }
    // its possible that there is no output key, if using the DynamicPartitioner, in which case
    // DynamicPartitioningOutputFormat is responsible for registering the partitions and the metadata
    PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning());
    if (outputKey != null) {
        Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(runtimeArguments);
        addPartition(outputKey, outputPath, metadata, true);
    }
    // currently, FileSetDataset#onSuccess is a no-op, but call it, in case it does something in the future
    ((FileSetDataset) files).onSuccess();
}

Example 2 with FileSetDataset

use of co.cask.cdap.data2.dataset2.lib.file.FileSetDataset in project cdap by caskdata.

the class FileSetTest method testRollback.

@Test
public void testRollback() throws IOException, TransactionFailureException, DatasetManagementException {
    // test deletion of an empty output directory
    FileSet fileSet1 = createFileset(testFileSetInstance1);
    Location outputLocation = fileSet1.getOutputLocation();
    Assert.assertFalse(outputLocation.exists());
    Assert.assertTrue(outputLocation.mkdirs());
    Assert.assertTrue(outputLocation.exists());
    ((FileSetDataset) fileSet1).onFailure();
    Assert.assertFalse(outputLocation.exists());
}

Example 3 with FileSetDataset

use of co.cask.cdap.data2.dataset2.lib.file.FileSetDataset in project cdap by caskdata.

the class FileSetTest method testRollbackWithNonEmptyDir.

@Test
public void testRollbackWithNonEmptyDir() throws IOException, TransactionFailureException, DatasetManagementException {
    FileSet fileSet1 = createFileset(testFileSetInstance1);
    Location outputDir = fileSet1.getOutputLocation();
    Assert.assertFalse(outputDir.exists());
    Assert.assertTrue(outputDir.mkdirs());
    Location outputFile = outputDir.append("outputFile");
    // this will create the outputFile
    outputFile.getOutputStream().close();
    Assert.assertTrue(outputFile.exists());
    Assert.assertTrue(outputDir.exists());
    ((FileSetDataset) fileSet1).onFailure();
    // both the output dir and file in it should still exist
    Assert.assertTrue(outputDir.exists());
    Assert.assertTrue(outputFile.exists());
}

Example 4 with FileSetDataset

use of co.cask.cdap.data2.dataset2.lib.file.FileSetDataset in project cdap by caskdata.

the class FileSetTest method testRollbackOfNonDirectoryOutput.

@Test
public void testRollbackOfNonDirectoryOutput() throws IOException, TransactionFailureException, DatasetManagementException {
    // test deletion of an output location, pointing to a non-directory file
    FileSet fileSet1 = createFileset(testFileSetInstance1);
    Location outputFile = fileSet1.getOutputLocation();
    Assert.assertFalse(outputFile.exists());
    outputFile.getOutputStream().close();
    Assert.assertTrue(outputFile.exists());
    ((FileSetDataset) fileSet1).onFailure();
    // the output file should still not be deleted
    Assert.assertTrue(outputFile.exists());
}