Examples with ApexRunnerResult org.apache.beam.runners.apex.ApexRunnerResult used on opensource projects

Example 6 with ApexRunnerResult

use of org.apache.beam.runners.apex.ApexRunnerResult in project beam by apache.

the class ReadUnboundTranslatorTest method test.

@Test
public void test() throws Exception {
    ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class);
    EmbeddedCollector.RESULTS.clear();
    options.setApplicationName("ReadUnbound");
    options.setRunner(ApexRunner.class);
    Pipeline p = Pipeline.create(options);
    List<String> collection = Lists.newArrayList("1", "2", "3", "4", "5");
    CollectionSource<String> source = new CollectionSource<>(collection, StringUtf8Coder.of());
    p.apply(Read.from(source)).apply(ParDo.of(new EmbeddedCollector()));
    ApexRunnerResult result = (ApexRunnerResult) p.run();
    DAG dag = result.getApexDAG();
    DAG.OperatorMeta om = dag.getOperatorMeta("Read(CollectionSource)");
    Assert.assertNotNull(om);
    Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class);
    long timeout = System.currentTimeMillis() + 30000;
    while (System.currentTimeMillis() < timeout) {
        if (EmbeddedCollector.RESULTS.containsAll(collection)) {
            break;
        }
        LOG.info("Waiting for expected results.");
        Thread.sleep(1000);
    }
    Assert.assertEquals(Sets.newHashSet(collection), EmbeddedCollector.RESULTS);
}

Example 7 with ApexRunnerResult

use of org.apache.beam.runners.apex.ApexRunnerResult in project beam by apache.

the class GroupByKeyTranslatorTest method test.

@SuppressWarnings({ "unchecked" })
@Test
public void test() throws Exception {
    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
    options.setApplicationName("GroupByKey");
    options.setRunner(ApexRunner.class);
    Pipeline p = Pipeline.create(options);
    List<KV<String, Instant>> data = Lists.newArrayList(KV.of("foo", new Instant(1000)), KV.of("foo", new Instant(1000)), KV.of("foo", new Instant(2000)), KV.of("bar", new Instant(1000)), KV.of("bar", new Instant(2000)), KV.of("bar", new Instant(2000)));
    // expected results assume outputAtLatestInputTimestamp
    List<KV<Instant, KV<String, Long>>> expected = Lists.newArrayList(KV.of(new Instant(1000), KV.of("foo", 2L)), KV.of(new Instant(1000), KV.of("bar", 1L)), KV.of(new Instant(2000), KV.of("foo", 1L)), KV.of(new Instant(2000), KV.of("bar", 2L)));
    p.apply(Read.from(new TestSource(data, new Instant(5000)))).apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))).withTimestampCombiner(TimestampCombiner.LATEST)).apply(Count.<String>perElement()).apply(ParDo.of(new KeyedByTimestamp<KV<String, Long>>())).apply(ParDo.of(new EmbeddedCollector()));
    ApexRunnerResult result = (ApexRunnerResult) p.run();
    result.getApexDAG();
    long timeout = System.currentTimeMillis() + 30000;
    while (System.currentTimeMillis() < timeout) {
        if (EmbeddedCollector.RESULTS.containsAll(expected)) {
            break;
        }
        Thread.sleep(1000);
    }
    Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS);
}