Examples with Coder org.apache.beam.model.pipeline.v1.RunnerApi.Coder used on opensource projects

Example 21 with Coder

use of org.apache.beam.model.pipeline.v1.RunnerApi.Coder in project beam by apache.

the class PipelineValidator method validateCombine.

private static void validateCombine(String id, PTransform transform, Components components, Set<String> requirements) throws Exception {
    CombinePayload payload = CombinePayload.parseFrom(transform.getSpec().getPayload());
    checkArgument(components.containsCoders(payload.getAccumulatorCoderId()), "Transform %s uses unknown accumulator coder id %s", id, payload.getAccumulatorCoderId());
}

Example 22 with Coder

use of org.apache.beam.model.pipeline.v1.RunnerApi.Coder in project beam by apache.

the class ModelCodersTest method windowedValueCoderComponentsToConstructor.

@Test
public void windowedValueCoderComponentsToConstructor() throws IOException {
    FullWindowedValueCoder<Iterable<KV<String, Integer>>> javaCoder = FullWindowedValueCoder.of(IterableCoder.of(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())), IntervalWindowCoder.of());
    MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder);
    WindowedValueCoderComponents windowedValueCoderComponents = ModelCoders.getWindowedValueCoderComponents(coderAndComponents.getCoder());
    Coder windowedCoder = ModelCoders.windowedValueCoder(windowedValueCoderComponents.elementCoderId(), windowedValueCoderComponents.windowCoderId());
    assertThat(windowedCoder, equalTo(coderAndComponents.getCoder()));
}

Example 23 with Coder

use of org.apache.beam.model.pipeline.v1.RunnerApi.Coder in project beam by apache.

the class ModelCodersTest method kvCoderComponentsToConstructor.

@Test
public void kvCoderComponentsToConstructor() throws IOException {
    KvCoder<byte[], Iterable<Long>> javaCoder = KvCoder.of(ByteArrayCoder.of(), IterableCoder.of(LengthPrefixCoder.of(VarLongCoder.of())));
    MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder);
    KvCoderComponents kvCoderComponents = ModelCoders.getKvCoderComponents(coderAndComponents.getCoder());
    Coder kvCoder = ModelCoders.kvCoder(kvCoderComponents.keyCoderId(), kvCoderComponents.valueCoderId());
    assertThat(kvCoder, equalTo(coderAndComponents.getCoder()));
}

Example 24 with Coder

use of org.apache.beam.model.pipeline.v1.RunnerApi.Coder in project beam by apache.

the class PCollectionTranslationTest method testEncodeDecodeFields.

@Test
public void testEncodeDecodeFields() throws Exception {
    SdkComponents sdkComponents = SdkComponents.create();
    sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java"));
    RunnerApi.PCollection protoCollection = PCollectionTranslation.toProto(testCollection, sdkComponents);
    RehydratedComponents protoComponents = RehydratedComponents.forComponents(sdkComponents.toComponents());
    Coder<?> decodedCoder = protoComponents.getCoder(protoCollection.getCoderId());
    WindowingStrategy<?, ?> decodedStrategy = protoComponents.getWindowingStrategy(protoCollection.getWindowingStrategyId());
    IsBounded decodedIsBounded = PCollectionTranslation.isBounded(protoCollection);
    assertThat(decodedCoder, equalTo(testCollection.getCoder()));
    assertThat(decodedStrategy, equalTo(testCollection.getWindowingStrategy().withEnvironmentId(sdkComponents.getOnlyEnvironmentId()).fixDefaults()));
    assertThat(decodedIsBounded, equalTo(testCollection.isBounded()));
}

Example 25 with Coder

use of org.apache.beam.model.pipeline.v1.RunnerApi.Coder in project beam by apache.

the class PipelineTranslationTest method getAccumulatorCoder.

private static Coder<?> getAccumulatorCoder(AppliedPTransform<?, ?, ?> transform) throws IOException {
    SdkComponents sdkComponents = SdkComponents.create(transform.getPipeline().getOptions());
    String id = getCombinePayload(transform, sdkComponents).map(CombinePayload::getAccumulatorCoderId).orElseThrow(() -> new IOException("Transform does not contain an AccumulatorCoder"));
    Components components = sdkComponents.toComponents();
    return CoderTranslation.fromProto(components.getCodersOrThrow(id), RehydratedComponents.forComponents(components), TranslationContext.DEFAULT);
}