Examples with ApexPipelineOptions org.apache.beam.runners.apex.ApexPipelineOptions used on opensource projects

Example 11 with ApexPipelineOptions

use of org.apache.beam.runners.apex.ApexPipelineOptions in project beam by apache.

the class FlattenPCollectionTranslatorTest method testFlattenSingleCollection.

@Test
public void testFlattenSingleCollection() {
    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
    Pipeline p = Pipeline.create();
    PCollection<String> single = p.apply(Create.of(Collections.singletonList("1")));
    PCollectionList.of(single).apply(Flatten.<String>pCollections()).apply(ParDo.of(new EmbeddedCollector()));
    DAG dag = TestApexRunner.translate(p, options);
    Assert.assertNotNull(dag.getOperatorMeta("ParDo(EmbeddedCollector)/ParMultiDo(EmbeddedCollector)"));
}

Example 12 with ApexPipelineOptions

use of org.apache.beam.runners.apex.ApexPipelineOptions in project beam by apache.

the class GroupByKeyTranslatorTest method test.

@SuppressWarnings({ "unchecked" })
@Test
public void test() throws Exception {
    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
    options.setApplicationName("GroupByKey");
    options.setRunner(ApexRunner.class);
    Pipeline p = Pipeline.create(options);
    List<KV<String, Instant>> data = Lists.newArrayList(KV.of("foo", new Instant(1000)), KV.of("foo", new Instant(1000)), KV.of("foo", new Instant(2000)), KV.of("bar", new Instant(1000)), KV.of("bar", new Instant(2000)), KV.of("bar", new Instant(2000)));
    // expected results assume outputAtLatestInputTimestamp
    List<KV<Instant, KV<String, Long>>> expected = Lists.newArrayList(KV.of(new Instant(1000), KV.of("foo", 2L)), KV.of(new Instant(1000), KV.of("bar", 1L)), KV.of(new Instant(2000), KV.of("foo", 1L)), KV.of(new Instant(2000), KV.of("bar", 2L)));
    p.apply(Read.from(new TestSource(data, new Instant(5000)))).apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))).withTimestampCombiner(TimestampCombiner.LATEST)).apply(Count.<String>perElement()).apply(ParDo.of(new KeyedByTimestamp<KV<String, Long>>())).apply(ParDo.of(new EmbeddedCollector()));
    ApexRunnerResult result = (ApexRunnerResult) p.run();
    result.getApexDAG();
    long timeout = System.currentTimeMillis() + 30000;
    while (System.currentTimeMillis() < timeout) {
        if (EmbeddedCollector.RESULTS.containsAll(expected)) {
            break;
        }
        Thread.sleep(1000);
    }
    Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS);
}