Examples with BoundedHCatalogSource org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource used on opensource projects

Example 1 with BoundedHCatalogSource

use of org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource in project beam by apache.

the class HCatalogIOTest method testSourceCanBeSerializedMultipleTimes.

/**
 * Regression test for BEAM-10694.
 */
@Test
public void testSourceCanBeSerializedMultipleTimes() throws Exception {
    ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
    HCatalogIO.Read spec = HCatalogIO.read().withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf())).withContext(context).withTable(TEST_TABLE);
    BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
    SerializableUtils.clone(SerializableUtils.clone(source));
}

Example 2 with BoundedHCatalogSource

use of org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource in project beam by apache.

the class HCatalogIOTest method testSourceEqualsSplits.

/**
 * Test of Read using SourceTestUtils.assertSourcesEqualReferenceSource(..).
 */
@Test
@NeedsTestData
public void testSourceEqualsSplits() throws Exception {
    final int numRows = 1500;
    final int numSamples = 10;
    final long bytesPerRow = 15;
    ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
    HCatalogIO.Read spec = HCatalogIO.read().withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf())).withContext(context).withTable(TEST_TABLE);
    BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
    List<BoundedSource<HCatRecord>> unSplitSource = source.split(-1, OPTIONS);
    assertEquals(1, unSplitSource.size());
    List<BoundedSource<HCatRecord>> splits = source.split(numRows * bytesPerRow / numSamples, OPTIONS);
    assertTrue(splits.size() >= 1);
    SourceTestUtils.assertSourcesEqualReferenceSource(unSplitSource.get(0), splits, OPTIONS);
}

Example 3 with BoundedHCatalogSource

use of org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource in project beam by apache.

the class HCatalogIOTest method testReadFromSource.

/**
 * Test of Read using SourceTestUtils.readFromSource(..).
 */
@Test
@NeedsTestData
public void testReadFromSource() throws Exception {
    ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
    HCatalogIO.Read spec = HCatalogIO.read().withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf())).withContext(context).withTable(TEST_TABLE);
    List<String> records = new ArrayList<>();
    for (int i = 0; i < context.numSplits(); i++) {
        BoundedHCatalogSource source = new BoundedHCatalogSource(spec.withSplitId(i));
        for (HCatRecord record : SourceTestUtils.readFromSource(source, OPTIONS)) {
            records.add(record.get(0).toString());
        }
    }
    assertThat(records, containsInAnyOrder(getExpectedRecords(TEST_RECORDS_COUNT).toArray()));
}