Examples with DoFn org.apache.beam.sdk.transforms.DoFn used on opensource projects

Example 31 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class PTransformMatchersTest method parDoWithFnTypeWithNoMatch.

@Test
public void parDoWithFnTypeWithNoMatch() {
    DoFn<Object, Object> fn = new DoFn<Object, Object>() {

        @ProcessElement
        public void process(ProcessContext ctxt) {
        }
    };
    AppliedPTransform<?, ?, ?> parDoSingle = getAppliedTransform(ParDo.of(fn));
    AppliedPTransform<?, ?, ?> parDoMulti = getAppliedTransform(ParDo.of(fn).withOutputTags(new TupleTag<>(), TupleTagList.empty()));
    PTransformMatcher matcher = PTransformMatchers.parDoWithFnType(doFnWithState.getClass());
    assertThat(matcher.matches(parDoSingle), is(false));
    assertThat(matcher.matches(parDoMulti), is(false));
}

Example 32 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class PTransformMatchersTest method classEqualToMatchesSameClass.

@Test
public void classEqualToMatchesSameClass() {
    PTransformMatcher matcher = PTransformMatchers.classEqualTo(ParDo.SingleOutput.class);
    AppliedPTransform<?, ?, ?> application = getAppliedTransform(ParDo.of(new DoFn<KV<String, Integer>, Integer>() {

        @ProcessElement
        public void doStuff(ProcessContext ctxt) {
        }
    }));
    assertThat(matcher.matches(application), is(true));
}

Example 33 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testMutatingOutputThenOutputDoFnError.

/**
 * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the {@link
 * DirectRunner}.
 */
@Test
public void testMutatingOutputThenOutputDoFnError() throws Exception {
    Pipeline pipeline = getPipeline();
    pipeline.apply(Create.of(42)).apply(ParDo.of(new DoFn<Integer, List<Integer>>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
            c.output(outputList);
            outputList.set(0, 37);
            c.output(outputList);
        }
    }));
    thrown.expect(IllegalMutationException.class);
    thrown.expectMessage("output");
    thrown.expectMessage("must not be mutated");
    pipeline.run();
}

Example 34 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testFromOptionsIfIgnoredFieldsGettingDropped.

/**
 * Tests that {@link DirectRunner#fromOptions(PipelineOptions)} drops {@link PipelineOptions}
 * marked with {@link JsonIgnore} fields.
 */
@Test
public void testFromOptionsIfIgnoredFieldsGettingDropped() {
    TestSerializationOfOptions options = PipelineOptionsFactory.fromArgs("--foo=testValue", "--ignoredField=overridden", "--runner=DirectRunner").as(TestSerializationOfOptions.class);
    assertEquals("testValue", options.getFoo());
    assertEquals("overridden", options.getIgnoredField());
    Pipeline p = Pipeline.create(options);
    PCollection<Integer> pc = p.apply(Create.of("1")).apply(ParDo.of(new DoFn<String, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            TestSerializationOfOptions options = c.getPipelineOptions().as(TestSerializationOfOptions.class);
            assertEquals("testValue", options.getFoo());
            assertEquals("not overridden", options.getIgnoredField());
            c.output(Integer.parseInt(c.element()));
        }
    }));
    PAssert.that(pc).containsInAnyOrder(1);
    p.run();
}

Example 35 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testMutatingInputDoFnError.

/**
 * Tests that a {@link DoFn} that mutates its input with a good equals() fails in the {@link
 * DirectRunner}.
 */
@Test
public void testMutatingInputDoFnError() throws Exception {
    Pipeline pipeline = getPipeline();
    pipeline.apply(Create.of(Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6)).withCoder(ListCoder.of(VarIntCoder.of()))).apply(ParDo.of(new DoFn<List<Integer>, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            List<Integer> inputList = c.element();
            inputList.set(0, 37);
            c.output(12);
        }
    }));
    thrown.expect(IllegalMutationException.class);
    thrown.expectMessage("Input");
    thrown.expectMessage("must not be mutated");
    pipeline.run();
}