Examples with ResourceId org.apache.beam.sdk.io.fs.ResourceId used on opensource projects

Example 1 with ResourceId

use of org.apache.beam.sdk.io.fs.ResourceId in project beam by apache.

the class HadoopResourceIdTest method testResourceIdTester.

@Test
public void testResourceIdTester() throws Exception {
    ResourceId baseDirectory = FileSystems.matchNewResource("hdfs://" + hdfsClusterBaseUri.getPath(), true);
    ResourceIdTester.runResourceIdBattery(baseDirectory);
}

Example 2 with ResourceId

use of org.apache.beam.sdk.io.fs.ResourceId in project beam by apache.

the class FileBasedSinkTest method testGenerateOutputFilenames.

/**
   * Output filenames are generated correctly when an extension is supplied.
   */
@Test
public void testGenerateOutputFilenames() {
    List<ResourceId> expected;
    List<ResourceId> actual;
    ResourceId root = getBaseOutputDirectory();
    SimpleSink sink = new SimpleSink(root, "file", ".SSSSS.of.NNNNN", ".test");
    FilenamePolicy policy = sink.getFilenamePolicy();
    expected = Arrays.asList(root.resolve("file.00000.of.00003.test", StandardResolveOptions.RESOLVE_FILE), root.resolve("file.00001.of.00003.test", StandardResolveOptions.RESOLVE_FILE), root.resolve("file.00002.of.00003.test", StandardResolveOptions.RESOLVE_FILE));
    actual = generateDestinationFilenames(root, policy, 3);
    assertEquals(expected, actual);
    expected = Collections.singletonList(root.resolve("file.00000.of.00001.test", StandardResolveOptions.RESOLVE_FILE));
    actual = generateDestinationFilenames(root, policy, 1);
    assertEquals(expected, actual);
    expected = new ArrayList<>();
    actual = generateDestinationFilenames(root, policy, 0);
    assertEquals(expected, actual);
}

Example 3 with ResourceId

use of org.apache.beam.sdk.io.fs.ResourceId in project beam by apache.

the class FileBasedSinkTest method testRemoveTemporaryFiles.

/**
   * Create n temporary and output files and verify that removeTemporaryFiles only removes temporary
   * files.
   */
private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory) throws Exception {
    String prefix = "file";
    SimpleSink sink = new SimpleSink(getBaseOutputDirectory(), prefix, "", "");
    WriteOperation<String> writeOp = new SimpleSink.SimpleWriteOperation(sink, tempDirectory);
    List<File> temporaryFiles = new ArrayList<>();
    List<File> outputFiles = new ArrayList<>();
    for (int i = 0; i < numFiles; i++) {
        ResourceId tempResource = WriteOperation.buildTemporaryFilename(tempDirectory, prefix + i);
        File tmpFile = new File(tempResource.toString());
        tmpFile.getParentFile().mkdirs();
        assertTrue("not able to create new temp file", tmpFile.createNewFile());
        temporaryFiles.add(tmpFile);
        ResourceId outputFileId = getBaseOutputDirectory().resolve(prefix + i, StandardResolveOptions.RESOLVE_FILE);
        File outputFile = new File(outputFileId.toString());
        outputFile.getParentFile().mkdirs();
        assertTrue("not able to create new output file", outputFile.createNewFile());
        outputFiles.add(outputFile);
    }
    writeOp.removeTemporaryFiles(Collections.<ResourceId>emptySet(), true);
    for (int i = 0; i < numFiles; i++) {
        File temporaryFile = temporaryFiles.get(i);
        assertThat(String.format("temp file %s exists", temporaryFile), temporaryFile.exists(), is(false));
        File outputFile = outputFiles.get(i);
        assertThat(String.format("output file %s exists", outputFile), outputFile.exists(), is(true));
    }
}

Example 4 with ResourceId

use of org.apache.beam.sdk.io.fs.ResourceId in project beam by apache.

the class FileBasedSinkTest method testFileBasedWriterWithWritableByteChannelFactory.

/**
   * {@link Writer} writes to the {@link WritableByteChannel} provided by {@link
   * DrunkWritableByteChannelFactory}.
   */
@Test
public void testFileBasedWriterWithWritableByteChannelFactory() throws Exception {
    final String testUid = "testId";
    ResourceId root = getBaseOutputDirectory();
    WriteOperation<String> writeOp = new SimpleSink(root, "file", "-SS-of-NN", "txt", new DrunkWritableByteChannelFactory()).createWriteOperation();
    final Writer<String> writer = writeOp.createWriter();
    final ResourceId expectedFile = writeOp.tempDirectory.get().resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
    final List<String> expected = new ArrayList<>();
    expected.add("header");
    expected.add("header");
    expected.add("a");
    expected.add("a");
    expected.add("b");
    expected.add("b");
    expected.add("footer");
    expected.add("footer");
    writer.openUnwindowed(testUid, -1);
    writer.write("a");
    writer.write("b");
    final FileResult result = writer.close();
    assertEquals(expectedFile, result.getTempFilename());
    assertFileContains(expected, expectedFile);
}

Example 5 with ResourceId

use of org.apache.beam.sdk.io.fs.ResourceId in project beam by apache.

the class FileBasedSinkTest method generateTemporaryFilesForFinalize.

/** Generate n temporary files using the temporary file pattern of Writer. */
private List<File> generateTemporaryFilesForFinalize(int numFiles) throws Exception {
    List<File> temporaryFiles = new ArrayList<>();
    for (int i = 0; i < numFiles; i++) {
        ResourceId temporaryFile = WriteOperation.buildTemporaryFilename(getBaseTempDirectory(), "" + i);
        File tmpFile = new File(tmpFolder.getRoot(), temporaryFile.toString());
        tmpFile.getParentFile().mkdirs();
        assertTrue(tmpFile.createNewFile());
        temporaryFiles.add(tmpFile);
    }
    return temporaryFiles;
}