Examples with PBegin org.apache.beam.sdk.values.PBegin used on opensource projects

Example 1 with PBegin

use of org.apache.beam.sdk.values.PBegin in project beam by apache.

the class DataflowGroupByKeyTest method testGroupByKeyServiceUnbounded.

@Test
public void testGroupByKeyServiceUnbounded() {
    Pipeline p = createTestServiceRunner();
    PCollection<KV<String, Integer>> input = p.apply(new PTransform<PBegin, PCollection<KV<String, Integer>>>() {

        @Override
        public PCollection<KV<String, Integer>> expand(PBegin input) {
            return PCollection.<KV<String, Integer>>createPrimitiveOutputInternal(input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED).setTypeDescriptor(new TypeDescriptor<KV<String, Integer>>() {
            });
        }
    });
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("GroupByKey cannot be applied to non-bounded PCollection in the GlobalWindow without " + "a trigger. Use a Window.into or Window.triggering transform prior to GroupByKey.");
    input.apply("GroupByKey", GroupByKey.<String, Integer>create());
}

Example 2 with PBegin

use of org.apache.beam.sdk.values.PBegin in project beam by apache.

the class SdkComponentsTest method registerTransformAfterChildren.

@Test
public void registerTransformAfterChildren() throws IOException {
    Create.Values<Long> create = Create.of(1L, 2L, 3L);
    GenerateSequence createChild = GenerateSequence.from(0);
    PCollection<Long> pt = pipeline.apply(create);
    String userName = "my_transform";
    String childUserName = "my_transform/my_nesting";
    AppliedPTransform<?, ?, ?> transform = AppliedPTransform.<PBegin, PCollection<Long>, Create.Values<Long>>of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
    AppliedPTransform<?, ?, ?> childTransform = AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of(childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline);
    String childId = components.registerPTransform(childTransform, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
    String parentId = components.registerPTransform(transform, Collections.<AppliedPTransform<?, ?, ?>>singletonList(childTransform));
    Components components = this.components.toComponents();
    assertThat(components.getTransformsOrThrow(parentId).getSubtransforms(0), equalTo(childId));
    assertThat(components.getTransformsOrThrow(childId).getSubtransformsCount(), equalTo(0));
}

Example 3 with PBegin

use of org.apache.beam.sdk.values.PBegin in project beam by apache.

the class SdkComponentsTest method registerTransformWithUnregisteredChildren.

/**
   * Tests that trying to register a transform which has unregistered children throws.
   */
@Test
public void registerTransformWithUnregisteredChildren() throws IOException {
    Create.Values<Long> create = Create.of(1L, 2L, 3L);
    GenerateSequence createChild = GenerateSequence.from(0);
    PCollection<Long> pt = pipeline.apply(create);
    String userName = "my_transform";
    String childUserName = "my_transform/my_nesting";
    AppliedPTransform<?, ?, ?> transform = AppliedPTransform.<PBegin, PCollection<Long>, Create.Values<Long>>of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
    AppliedPTransform<?, ?, ?> childTransform = AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of(childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline);
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(childTransform.toString());
    components.registerPTransform(transform, Collections.<AppliedPTransform<?, ?, ?>>singletonList(childTransform));
}

Example 4 with PBegin

use of org.apache.beam.sdk.values.PBegin in project beam by apache.

the class PTransformTranslationTest method generateSequence.

private static AppliedPTransform<?, ?, ?> generateSequence(Pipeline pipeline) {
    GenerateSequence sequence = GenerateSequence.from(0);
    PCollection<Long> pcollection = pipeline.apply(sequence);
    return AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of("Count", pipeline.begin().expand(), pcollection.expand(), sequence, pipeline);
}

Example 5 with PBegin

use of org.apache.beam.sdk.values.PBegin in project beam by apache.

the class CreateTest method testCreateTimestampedDefaultOutputCoderUsingCoder.

@Test
public void testCreateTimestampedDefaultOutputCoderUsingCoder() throws Exception {
    Coder<Record> coder = new RecordCoder();
    PBegin pBegin = PBegin.in(p);
    Create.TimestampedValues<Record> values = Create.timestamped(TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.<Record>of(new Record2(), new Instant(0))).withCoder(coder);
    Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
    assertThat(defaultCoder, equalTo(coder));
}