Examples with Coder org.apache.beam.sdk.coders.Coder used on opensource projects

Example 36 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class Create method getDefaultCreateCoder.

private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry coderRegistry, SchemaRegistry schemaRegistry, Iterable<T> elems) throws CannotProvideCoderException {
    checkArgument(!Iterables.isEmpty(elems), "Can not determine a default Coder for a 'Create' PTransform that " + "has no elements.  Either add elements, call Create.empty(Coder)," + " Create.empty(TypeDescriptor), or call 'withCoder(Coder)' or " + "'withType(TypeDescriptor)' on the PTransform.");
    // First try to deduce a coder using the types of the elements.
    Class<?> elementClazz = Void.class;
    for (T elem : elems) {
        if (elem == null) {
            continue;
        }
        Class<?> clazz = elem.getClass();
        if (elementClazz.equals(Void.class)) {
            elementClazz = clazz;
        } else if (!elementClazz.equals(clazz)) {
            // Elements are not the same type, require a user-specified coder.
            throw new CannotProvideCoderException(String.format("Cannot provide coder for %s: The elements are not all of the same class.", Create.class.getSimpleName()));
        }
    }
    TypeDescriptor<T> typeDescriptor = (TypeDescriptor<T>) TypeDescriptor.of(elementClazz);
    if (elementClazz.getTypeParameters().length == 0) {
        try {
            Coder<T> coder = SchemaCoder.of(schemaRegistry.getSchema(typeDescriptor), typeDescriptor, schemaRegistry.getToRowFunction(typeDescriptor), schemaRegistry.getFromRowFunction(typeDescriptor));
            return coder;
        } catch (NoSuchSchemaException e) {
        // No schema.
        }
        try {
            // elementClazz is a wildcard type
            @SuppressWarnings("unchecked") Coder<T> coder = (Coder<T>) coderRegistry.getCoder(typeDescriptor);
            return coder;
        } catch (CannotProvideCoderException exc) {
        // Can't get a coder from the class of the elements, try from elements next.
        }
    }
    // If that fails, try to deduce a coder using the elements themselves
    return (Coder<T>) inferCoderFromObjects(coderRegistry, schemaRegistry, elems);
}

Example 37 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class ParDo method finishSpecifyingStateSpecs.

private static void finishSpecifyingStateSpecs(DoFn<?, ?> fn, CoderRegistry coderRegistry, SchemaRegistry schemaRegistry, Coder<?> inputCoder) {
    DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
    Map<String, DoFnSignature.StateDeclaration> stateDeclarations = signature.stateDeclarations();
    for (DoFnSignature.StateDeclaration stateDeclaration : stateDeclarations.values()) {
        try {
            StateSpec<?> stateSpec = (StateSpec<?>) stateDeclaration.field().get(fn);
            Coder[] coders;
            try {
                coders = schemasForStateSpecTypes(stateDeclaration, schemaRegistry);
            } catch (NoSuchSchemaException e) {
                coders = codersForStateSpecTypes(stateDeclaration, coderRegistry, inputCoder);
            }
            stateSpec.offerCoders(coders);
            stateSpec.finishSpecifying();
        } catch (IllegalAccessException e) {
            throw new RuntimeException(e);
        }
    }
}

Example 38 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class DynamicProtoCoderTest method testDynamicNestedRepeatedMessage.

@Test
public void testDynamicNestedRepeatedMessage() throws Exception {
    DynamicMessage message = DynamicMessage.newBuilder(MessageA.getDescriptor()).setField(MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD1_FIELD_NUMBER), "foo").addRepeatedField(MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD2_FIELD_NUMBER), DynamicMessage.newBuilder(MessageB.getDescriptor()).setField(MessageB.getDescriptor().findFieldByNumber(MessageB.FIELD1_FIELD_NUMBER), true).build()).addRepeatedField(MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD2_FIELD_NUMBER), DynamicMessage.newBuilder(MessageB.getDescriptor()).setField(MessageB.getDescriptor().findFieldByNumber(MessageB.FIELD1_FIELD_NUMBER), false).build()).build();
    Coder<DynamicMessage> coder = DynamicProtoCoder.of(message.getDescriptorForType());
    // Special code to check the DynamicMessage equality (@see IsDynamicMessageEqual)
    for (Coder.Context context : ALL_CONTEXTS) {
        CoderProperties.coderDecodeEncodeInContext(coder, context, message, IsDynamicMessageEqual.equalTo(message));
    }
}

Example 39 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class FnApiDoFnRunner method trySplitForElementAndRestriction.

private HandlesSplits.SplitResult trySplitForElementAndRestriction(double fractionOfRemainder, Duration resumeDelay) {
    KV<Instant, WatermarkEstimatorStateT> watermarkAndState;
    WindowedSplitResult windowedSplitResult = null;
    synchronized (splitLock) {
        // There is nothing to split if we are between element and restriction processing calls.
        if (currentTracker == null) {
            return null;
        }
        // Make sure to get the output watermark before we split to ensure that the lower bound
        // applies to the residual.
        watermarkAndState = currentWatermarkEstimator.getWatermarkAndState();
        SplitResultsWithStopIndex splitResult = computeSplitForProcessOrTruncate(currentElement, currentRestriction, currentWindow, currentWindows, currentWatermarkEstimatorState, fractionOfRemainder, currentTracker, null, watermarkAndState, windowCurrentIndex, windowStopIndex);
        if (splitResult == null) {
            return null;
        }
        windowStopIndex = splitResult.getNewWindowStopIndex();
        // Populate the size of primary/residual.
        windowedSplitResult = calculateRestrictionSize(splitResult.getWindowSplit(), PTransformTranslation.SPLITTABLE_PROCESS_SIZED_ELEMENTS_AND_RESTRICTIONS_URN + "/GetSize");
    }
    Coder fullInputCoder = WindowedValue.getFullCoder(inputCoder, windowCoder);
    return constructSplitResult(windowedSplitResult, null, fullInputCoder, initialWatermark, watermarkAndState, pTransformId, mainInputId, pTransform.getOutputsMap().keySet(), resumeDelay);
}

Example 40 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class FnApiDoFnRunner method trySplitForWindowObservingTruncateRestriction.

private HandlesSplits.SplitResult trySplitForWindowObservingTruncateRestriction(double fractionOfRemainder, HandlesSplits splitDelegate) {
    WindowedSplitResult windowedSplitResult = null;
    HandlesSplits.SplitResult downstreamSplitResult = null;
    synchronized (splitLock) {
        // There is nothing to split if we are between truncate processing calls.
        if (currentWindow == null) {
            return null;
        }
        SplitResultsWithStopIndex splitResult = computeSplitForProcessOrTruncate(currentElement, currentRestriction, currentWindow, currentWindows, currentWatermarkEstimatorState, fractionOfRemainder, null, splitDelegate, null, windowCurrentIndex, windowStopIndex);
        if (splitResult == null) {
            return null;
        }
        windowStopIndex = splitResult.getNewWindowStopIndex();
        windowedSplitResult = calculateRestrictionSize(splitResult.getWindowSplit(), PTransformTranslation.SPLITTABLE_TRUNCATE_SIZED_RESTRICTION_URN + "/GetSize");
        downstreamSplitResult = splitResult.getDownstreamSplit();
    }
    // Note that the assumption here is the fullInputCoder of the Truncate transform should be the
    // the same as the SDF/Process transform.
    Coder fullInputCoder = WindowedValue.getFullCoder(inputCoder, windowCoder);
    return constructSplitResult(windowedSplitResult, downstreamSplitResult, fullInputCoder, initialWatermark, null, pTransformId, mainInputId, pTransform.getOutputsMap().keySet(), null);
}