Examples with DoFn org.apache.beam.sdk.transforms.DoFn used on opensource projects

Example 26 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DoFnSignaturesTest method testWindowParamOnTimer.

@Test
public void testWindowParamOnTimer() throws Exception {
    final String timerId = "some-timer-id";
    final String timerDeclarationId = TimerDeclaration.PREFIX + timerId;
    DoFnSignature sig = DoFnSignatures.getSignature(new DoFn<String, String>() {

        @TimerId(timerId)
        private final TimerSpec myfield1 = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void process(ProcessContext c) {
        }

        @OnTimer(timerId)
        public void onTimer(BoundedWindow w) {
        }
    }.getClass());
    assertThat(sig.onTimerMethods().get(timerDeclarationId).extraParameters().size(), equalTo(1));
    assertThat(sig.onTimerMethods().get(timerDeclarationId).extraParameters().get(0), instanceOf(WindowParameter.class));
}

Example 27 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DoFnSignaturesTest method testSimpleTimerIdAnonymousDoFn.

@Test
public void testSimpleTimerIdAnonymousDoFn() throws Exception {
    DoFnSignature sig = DoFnSignatures.getSignature(new DoFn<KV<String, Integer>, Long>() {

        @TimerId("foo")
        private final TimerSpec bizzle = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void foo(ProcessContext context) {
        }

        @OnTimer("foo")
        public void onFoo() {
        }
    }.getClass());
    final String timerDeclarationId = TimerDeclaration.PREFIX + "foo";
    assertThat(sig.timerDeclarations().size(), equalTo(1));
    DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get(timerDeclarationId);
    assertThat(decl.id(), equalTo(timerDeclarationId));
    assertThat(decl.field().getName(), equalTo("bizzle"));
}

Example 28 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class WindowTest method testMergingCustomWindowsWithoutCustomWindowTypes.

@Test
@Category({ ValidatesRunner.class, UsesCustomWindowMerging.class })
public void testMergingCustomWindowsWithoutCustomWindowTypes() {
    Instant startInstant = new Instant(0L);
    PCollection<KV<String, Integer>> inputCollection = pipeline.apply(Create.timestamped(TimestampedValue.of(KV.of("a", 1), startInstant.plus(Duration.standardSeconds(1))), TimestampedValue.of(KV.of("a", 2), startInstant.plus(Duration.standardSeconds(2))), TimestampedValue.of(KV.of("a", 3), startInstant.plus(Duration.standardSeconds(3))), TimestampedValue.of(KV.of("a", 4), startInstant.plus(Duration.standardSeconds(4))), TimestampedValue.of(KV.of("a", 5), startInstant.plus(Duration.standardSeconds(5)))));
    PCollection<KV<String, Integer>> windowedCollection = inputCollection.apply(Window.into(new WindowOddEvenMergingBuckets<>()));
    PCollection<String> result = windowedCollection.apply(GroupByKey.create()).apply(ParDo.of(new DoFn<KV<String, Iterable<Integer>>, String>() {

        @ProcessElement
        public void processElement(ProcessContext c, BoundedWindow window) {
            List<Integer> elements = Lists.newArrayList();
            c.element().getValue().forEach(elements::add);
            Collections.sort(elements);
            c.output(elements.toString());
        }
    }));
    PAssert.that("Wrong output collection", result).containsInAnyOrder("[2, 4]", "[1, 3, 5]");
    pipeline.run();
}

Example 29 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class Snippets method accessingValueProviderInfoAfterRunSnip1.

public static void accessingValueProviderInfoAfterRunSnip1(String[] args) {
    MyOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(MyOptions.class);
    // Create pipeline.
    Pipeline p = Pipeline.create(options);
    // Add a branch for logging the ValueProvider value.
    p.apply(Create.of(1)).apply(ParDo.of(new DoFn<Integer, Integer>() {

        // Define the DoFn that logs the ValueProvider value.
        @ProcessElement
        public void process(ProcessContext c) {
            MyOptions ops = c.getPipelineOptions().as(MyOptions.class);
            // This example logs the ValueProvider value, but you could store it by
            // pushing it to an external database.
            LOG.info("Option StringValue was {}", ops.getStringValue());
        }
    }));
    // The main pipeline.
    p.apply(Create.of(1, 2, 3, 4)).apply(Sum.integersGlobally());
    p.run();
}

Example 30 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class ExecutableStageTranslationTest method testOperatorNameGeneration.

@Test
public /* Test for generating readable operator names during translation. */
void testOperatorNameGeneration() throws Exception {
    Pipeline p = Pipeline.create();
    p.apply(Impulse.create()).apply(ParDo.of(new DoFn<byte[], String>() {

        @ProcessElement
        public void processElement(ProcessContext processContext, OutputReceiver<String> outputReceiver) {
        }
    })).apply("MyName", ParDo.of(new DoFn<String, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext processContext, OutputReceiver<Integer> outputReceiver) {
        }
    })).apply(// Avoid nested Anonymous ParDo
    "Composite/Nested/ParDo", ParDo.of(new DoFn<Integer, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext processContext, OutputReceiver<Integer> outputReceiver) {
        }
    }));
    ExecutableStage firstEnvStage = GreedyPipelineFuser.fuse(PipelineTranslation.toProto(p)).getFusedStages().stream().findFirst().get();
    RunnerApi.ExecutableStagePayload basePayload = RunnerApi.ExecutableStagePayload.parseFrom(firstEnvStage.toPTransform("foo").getSpec().getPayload());
    String executableStageName = ExecutableStageTranslation.generateNameFromStagePayload(basePayload);
    assertThat(executableStageName, is("[3]{ParDo(Anonymous), MyName, Composite}"));
}