Search in sources :

Example 1 with BigEndianLongCoder

use of org.apache.beam.sdk.coders.BigEndianLongCoder in project beam by apache.

the class SdkComponentsTest method translatePipeline.

@Test
public void translatePipeline() {
    BigEndianLongCoder customCoder = BigEndianLongCoder.of();
    PCollection<Long> elems = pipeline.apply(GenerateSequence.from(0L).to(207L));
    PCollection<Long> counted = elems.apply(Count.<Long>globally()).setCoder(customCoder);
    PCollection<Long> windowed = counted.apply(Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7))).triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(19))).accumulatingFiredPanes().withAllowedLateness(Duration.standardMinutes(3L)));
    final WindowingStrategy<?, ?> windowedStrategy = windowed.getWindowingStrategy();
    PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.<String, Long>of("foo"));
    PCollection<KV<String, Iterable<Long>>> grouped = keyed.apply(GroupByKey.<String, Long>create());
    final RunnerApi.Pipeline pipelineProto = SdkComponents.translatePipeline(pipeline);
    pipeline.traverseTopologically(new PipelineVisitor.Defaults() {

        Set<Node> transforms = new HashSet<>();

        Set<PCollection<?>> pcollections = new HashSet<>();

        Set<Equivalence.Wrapper<? extends Coder<?>>> coders = new HashSet<>();

        Set<WindowingStrategy<?, ?>> windowingStrategies = new HashSet<>();

        @Override
        public void leaveCompositeTransform(Node node) {
            if (node.isRootNode()) {
                assertThat("Unexpected number of PTransforms", pipelineProto.getComponents().getTransformsCount(), equalTo(transforms.size()));
                assertThat("Unexpected number of PCollections", pipelineProto.getComponents().getPcollectionsCount(), equalTo(pcollections.size()));
                assertThat("Unexpected number of Coders", pipelineProto.getComponents().getCodersCount(), equalTo(coders.size()));
                assertThat("Unexpected number of Windowing Strategies", pipelineProto.getComponents().getWindowingStrategiesCount(), equalTo(windowingStrategies.size()));
            } else {
                transforms.add(node);
            }
        }

        @Override
        public void visitPrimitiveTransform(Node node) {
            transforms.add(node);
        }

        @Override
        public void visitValue(PValue value, Node producer) {
            if (value instanceof PCollection) {
                PCollection pc = (PCollection) value;
                pcollections.add(pc);
                addCoders(pc.getCoder());
                windowingStrategies.add(pc.getWindowingStrategy());
                addCoders(pc.getWindowingStrategy().getWindowFn().windowCoder());
            }
        }

        private void addCoders(Coder<?> coder) {
            coders.add(Equivalence.<Coder<?>>identity().wrap(coder));
            if (coder instanceof StructuredCoder) {
                for (Coder<?> component : ((StructuredCoder<?>) coder).getComponents()) {
                    addCoders(component);
                }
            }
        }
    });
}
Also used : Node(org.apache.beam.sdk.runners.TransformHierarchy.Node) WindowingStrategy(org.apache.beam.sdk.values.WindowingStrategy) RunnerApi(org.apache.beam.sdk.common.runner.v1.RunnerApi) PipelineVisitor(org.apache.beam.sdk.Pipeline.PipelineVisitor) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) HashSet(java.util.HashSet) Coder(org.apache.beam.sdk.coders.Coder) SetCoder(org.apache.beam.sdk.coders.SetCoder) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) KvCoder(org.apache.beam.sdk.coders.KvCoder) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) IterableCoder(org.apache.beam.sdk.coders.IterableCoder) VarLongCoder(org.apache.beam.sdk.coders.VarLongCoder) StructuredCoder(org.apache.beam.sdk.coders.StructuredCoder) ByteArrayCoder(org.apache.beam.sdk.coders.ByteArrayCoder) KV(org.apache.beam.sdk.values.KV) PValue(org.apache.beam.sdk.values.PValue) PCollection(org.apache.beam.sdk.values.PCollection) StructuredCoder(org.apache.beam.sdk.coders.StructuredCoder) Test(org.junit.Test)

Example 2 with BigEndianLongCoder

use of org.apache.beam.sdk.coders.BigEndianLongCoder in project beam by apache.

the class PipelineTranslationTest method testPipelines.

@Parameters(name = "{index}")
public static Iterable<Pipeline> testPipelines() {
    Pipeline trivialPipeline = Pipeline.create();
    trivialPipeline.apply(Create.of(1, 2, 3));
    Pipeline sideInputPipeline = Pipeline.create();
    final PCollectionView<String> singletonView = sideInputPipeline.apply(Create.of("foo")).apply(View.asSingleton());
    sideInputPipeline.apply(Create.of("main input")).apply(ParDo.of(new DoFn<String, String>() {

        @ProcessElement
        public void process(ProcessContext c) {
            // actually never executed and no effect on translation
            c.sideInput(singletonView);
        }
    }).withSideInputs(singletonView));
    Pipeline complexPipeline = Pipeline.create();
    BigEndianLongCoder customCoder = BigEndianLongCoder.of();
    PCollection<Long> elems = complexPipeline.apply(GenerateSequence.from(0L).to(207L));
    PCollection<Long> counted = elems.apply(Count.globally()).setCoder(customCoder);
    PCollection<Long> windowed = counted.apply(Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7))).triggering(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(19))).accumulatingFiredPanes().withAllowedLateness(Duration.standardMinutes(3L)));
    windowed.getWindowingStrategy();
    PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.of("foo"));
    keyed.apply(GroupByKey.create());
    return ImmutableList.of(trivialPipeline, sideInputPipeline, complexPipeline);
}
Also used : ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) KV(org.apache.beam.sdk.values.KV) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) Pipeline(org.apache.beam.sdk.Pipeline) Parameters(org.junit.runners.Parameterized.Parameters)

Example 3 with BigEndianLongCoder

use of org.apache.beam.sdk.coders.BigEndianLongCoder in project beam by apache.

the class LatestTest method testGloballyOutputCoder.

@Test
public void testGloballyOutputCoder() {
    p.enableAbandonedNodeEnforcement(false);
    BigEndianLongCoder inputCoder = BigEndianLongCoder.of();
    PCollection<Long> output = p.apply(Create.of(1L, 2L).withCoder(inputCoder)).apply(Latest.globally());
    Coder<Long> outputCoder = output.getCoder();
    assertThat(outputCoder, instanceOf(NullableCoder.class));
    assertEquals(inputCoder, ((NullableCoder<?>) outputCoder).getValueCoder());
}
Also used : NullableCoder(org.apache.beam.sdk.coders.NullableCoder) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) Test(org.junit.Test)

Aggregations

BigEndianLongCoder (org.apache.beam.sdk.coders.BigEndianLongCoder)3 KV (org.apache.beam.sdk.values.KV)2 Test (org.junit.Test)2 HashSet (java.util.HashSet)1 Pipeline (org.apache.beam.sdk.Pipeline)1 PipelineVisitor (org.apache.beam.sdk.Pipeline.PipelineVisitor)1 ByteArrayCoder (org.apache.beam.sdk.coders.ByteArrayCoder)1 Coder (org.apache.beam.sdk.coders.Coder)1 IterableCoder (org.apache.beam.sdk.coders.IterableCoder)1 KvCoder (org.apache.beam.sdk.coders.KvCoder)1 NullableCoder (org.apache.beam.sdk.coders.NullableCoder)1 SetCoder (org.apache.beam.sdk.coders.SetCoder)1 StringUtf8Coder (org.apache.beam.sdk.coders.StringUtf8Coder)1 StructuredCoder (org.apache.beam.sdk.coders.StructuredCoder)1 VarLongCoder (org.apache.beam.sdk.coders.VarLongCoder)1 RunnerApi (org.apache.beam.sdk.common.runner.v1.RunnerApi)1 Node (org.apache.beam.sdk.runners.TransformHierarchy.Node)1 PCollection (org.apache.beam.sdk.values.PCollection)1 PValue (org.apache.beam.sdk.values.PValue)1 WindowingStrategy (org.apache.beam.sdk.values.WindowingStrategy)1