Examples with DoFn org.apache.beam.sdk.transforms.DoFn used on opensource projects

Example 36 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testMutatingOutputWithEnforcementDisabledSucceeds.

/**
 * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the {@link
 * DirectRunner}.
 */
@Test
public void testMutatingOutputWithEnforcementDisabledSucceeds() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    options.setRunner(DirectRunner.class);
    options.as(DirectOptions.class).setEnforceImmutability(false);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(42)).apply(ParDo.of(new DoFn<Integer, List<Integer>>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
            c.output(outputList);
            outputList.set(0, 37);
            c.output(outputList);
        }
    }));
    pipeline.run();
}

Example 37 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testMutatingInputCoderDoFnError.

/**
 * Tests that a {@link DoFn} that mutates an input with a bad equals() still fails in the {@link
 * DirectRunner}.
 */
@Test
public void testMutatingInputCoderDoFnError() throws Exception {
    Pipeline pipeline = getPipeline();
    pipeline.apply(Create.of(new byte[] { 0x1, 0x2, 0x3 }, new byte[] { 0x4, 0x5, 0x6 })).apply(ParDo.of(new DoFn<byte[], Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            byte[] inputArray = c.element();
            inputArray[0] = 0xa;
            c.output(13);
        }
    }));
    thrown.expect(IllegalMutationException.class);
    thrown.expectMessage("Input");
    thrown.expectMessage("must not be mutated");
    pipeline.run();
}

Example 38 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method testUnencodableOutputElement.

@Test
public void testUnencodableOutputElement() throws Exception {
    Pipeline p = getPipeline();
    PCollection<Long> pcollection = p.apply(Create.of((Void) null)).apply(ParDo.of(new DoFn<Void, Long>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output(null);
        }
    })).setCoder(VarLongCoder.of());
    pcollection.apply(ParDo.of(new DoFn<Long, Long>() {

        @ProcessElement
        public void unreachable(ProcessContext c) {
            fail("Pipeline should fail to encode a null Long in VarLongCoder");
        }
    }));
    thrown.expectCause(isA(CoderException.class));
    thrown.expectMessage("cannot encode a null Long");
    p.run();
}

Example 39 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DirectRunnerTest method transformDisplayDataExceptionShouldFail.

@Test
public void transformDisplayDataExceptionShouldFail() {
    DoFn<Integer, Integer> brokenDoFn = new DoFn<Integer, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) throws Exception {
        }

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            throw new RuntimeException("oh noes!");
        }
    };
    Pipeline p = getPipeline();
    p.apply(Create.of(1, 2, 3)).apply(ParDo.of(brokenDoFn));
    thrown.expectMessage(brokenDoFn.getClass().getName());
    thrown.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!")));
    p.run();
}

Example 40 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class ParDoMultiOverrideFactory method getReplacementForApplication.

@SuppressWarnings("unchecked")
private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication(AppliedPTransform<PCollection<? extends InputT>, PCollectionTuple, PTransform<PCollection<? extends InputT>, PCollectionTuple>> application) throws IOException {
    DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application);
    DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
    if (signature.processElement().isSplittable()) {
        return SplittableParDo.forAppliedParDo((AppliedPTransform) application);
    } else if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0 || signature.timerFamilyDeclarations().size() > 0) {
        return new GbkThenStatefulParDo(fn, ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application), ParDoTranslation.getSideInputs(application), ParDoTranslation.getSchemaInformation(application), ParDoTranslation.getSideInputMapping(application));
    } else {
        return application.getTransform();
    }
}