Examples with Pipeline org.apache.beam.sdk.Pipeline used on opensource projects

Example 46 with Pipeline

use of org.apache.beam.sdk.Pipeline in project beam by apache.

the class PCollectionListTest method testExpandWithDuplicates.

@Test
public void testExpandWithDuplicates() {
    Pipeline p = TestPipeline.create();
    PCollection<Long> createOne = p.apply("CreateOne", Create.of(1L, 2L, 3L));
    PCollectionList<Long> list = PCollectionList.of(createOne).and(createOne).and(createOne);
    assertThat(list.expand().values(), Matchers.<PValue>containsInAnyOrder(createOne, createOne, createOne));
}

Example 47 with Pipeline

use of org.apache.beam.sdk.Pipeline in project beam by apache.

the class DirectRunnerTest method splitsInputs.

@Test
public void splitsInputs() {
    Pipeline p = getPipeline();
    PCollection<Long> longs = p.apply(Read.from(MustSplitSource.of(CountingSource.upTo(3))));
    PAssert.that(longs).containsInAnyOrder(0L, 1L, 2L);
    p.run();
}

Example 48 with Pipeline

use of org.apache.beam.sdk.Pipeline in project beam by apache.

the class DirectRunnerTest method testMutatingInputDoFnError.

/**
   * Tests that a {@link DoFn} that mutates its input with a good equals() fails in the
   * {@link DirectRunner}.
   */
@Test
public void testMutatingInputDoFnError() throws Exception {
    Pipeline pipeline = getPipeline();
    pipeline.apply(Create.of(Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6)).withCoder(ListCoder.of(VarIntCoder.of()))).apply(ParDo.of(new DoFn<List<Integer>, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            List<Integer> inputList = c.element();
            inputList.set(0, 37);
            c.output(12);
        }
    }));
    thrown.expect(IllegalMutationException.class);
    thrown.expectMessage("Input");
    thrown.expectMessage("must not be mutated");
    pipeline.run();
}

Example 49 with Pipeline

use of org.apache.beam.sdk.Pipeline in project beam by apache.

the class DirectRunnerTest method testMutatingInputCoderDoFnError.

/**
   * Tests that a {@link DoFn} that mutates an input with a bad equals() still fails
   * in the {@link DirectRunner}.
   */
@Test
public void testMutatingInputCoderDoFnError() throws Exception {
    Pipeline pipeline = getPipeline();
    pipeline.apply(Create.of(new byte[] { 0x1, 0x2, 0x3 }, new byte[] { 0x4, 0x5, 0x6 })).apply(ParDo.of(new DoFn<byte[], Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            byte[] inputArray = c.element();
            inputArray[0] = 0xa;
            c.output(13);
        }
    }));
    thrown.expect(IllegalMutationException.class);
    thrown.expectMessage("Input");
    thrown.expectMessage("must not be mutated");
    pipeline.run();
}

Example 50 with Pipeline

use of org.apache.beam.sdk.Pipeline in project beam by apache.

the class DirectRunnerTest method wordCountShouldSucceed.

@Test
public void wordCountShouldSucceed() throws Throwable {
    Pipeline p = getPipeline();
    PCollection<KV<String, Long>> counts = p.apply(Create.of("foo", "bar", "foo", "baz", "bar", "foo")).apply(MapElements.via(new SimpleFunction<String, String>() {

        @Override
        public String apply(String input) {
            return input;
        }
    })).apply(Count.<String>perElement());
    PCollection<String> countStrs = counts.apply(MapElements.via(new SimpleFunction<KV<String, Long>, String>() {

        @Override
        public String apply(KV<String, Long> input) {
            String str = String.format("%s: %s", input.getKey(), input.getValue());
            return str;
        }
    }));
    PAssert.that(countStrs).containsInAnyOrder("baz: 1", "bar: 2", "foo: 3");
    DirectPipelineResult result = ((DirectPipelineResult) p.run());
    result.waitUntilFinish();
}