Examples with NoSuchSchemaException org.apache.beam.sdk.schemas.NoSuchSchemaException used on opensource projects

Example 1 with NoSuchSchemaException

use of org.apache.beam.sdk.schemas.NoSuchSchemaException in project beam by apache.

the class ParDo method finishSpecifyingStateSpecs.

private static void finishSpecifyingStateSpecs(DoFn<?, ?> fn, CoderRegistry coderRegistry, SchemaRegistry schemaRegistry, Coder<?> inputCoder) {
    DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
    Map<String, DoFnSignature.StateDeclaration> stateDeclarations = signature.stateDeclarations();
    for (DoFnSignature.StateDeclaration stateDeclaration : stateDeclarations.values()) {
        try {
            StateSpec<?> stateSpec = (StateSpec<?>) stateDeclaration.field().get(fn);
            Coder[] coders;
            try {
                coders = schemasForStateSpecTypes(stateDeclaration, schemaRegistry);
            } catch (NoSuchSchemaException e) {
                coders = codersForStateSpecTypes(stateDeclaration, coderRegistry, inputCoder);
            }
            stateSpec.offerCoders(coders);
            stateSpec.finishSpecifying();
        } catch (IllegalAccessException e) {
            throw new RuntimeException(e);
        }
    }
}

Example 2 with NoSuchSchemaException

use of org.apache.beam.sdk.schemas.NoSuchSchemaException in project beam by apache.

the class Create method getDefaultCreateCoder.

private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry coderRegistry, SchemaRegistry schemaRegistry, Iterable<T> elems) throws CannotProvideCoderException {
    checkArgument(!Iterables.isEmpty(elems), "Can not determine a default Coder for a 'Create' PTransform that " + "has no elements.  Either add elements, call Create.empty(Coder)," + " Create.empty(TypeDescriptor), or call 'withCoder(Coder)' or " + "'withType(TypeDescriptor)' on the PTransform.");
    // First try to deduce a coder using the types of the elements.
    Class<?> elementClazz = Void.class;
    for (T elem : elems) {
        if (elem == null) {
            continue;
        }
        Class<?> clazz = elem.getClass();
        if (elementClazz.equals(Void.class)) {
            elementClazz = clazz;
        } else if (!elementClazz.equals(clazz)) {
            // Elements are not the same type, require a user-specified coder.
            throw new CannotProvideCoderException(String.format("Cannot provide coder for %s: The elements are not all of the same class.", Create.class.getSimpleName()));
        }
    }
    TypeDescriptor<T> typeDescriptor = (TypeDescriptor<T>) TypeDescriptor.of(elementClazz);
    if (elementClazz.getTypeParameters().length == 0) {
        try {
            Coder<T> coder = SchemaCoder.of(schemaRegistry.getSchema(typeDescriptor), typeDescriptor, schemaRegistry.getToRowFunction(typeDescriptor), schemaRegistry.getFromRowFunction(typeDescriptor));
            return coder;
        } catch (NoSuchSchemaException e) {
        // No schema.
        }
        try {
            // elementClazz is a wildcard type
            @SuppressWarnings("unchecked") Coder<T> coder = (Coder<T>) coderRegistry.getCoder(typeDescriptor);
            return coder;
        } catch (CannotProvideCoderException exc) {
        // Can't get a coder from the class of the elements, try from elements next.
        }
    }
    // If that fails, try to deduce a coder using the elements themselves
    return (Coder<T>) inferCoderFromObjects(coderRegistry, schemaRegistry, elems);
}

Example 3 with NoSuchSchemaException

use of org.apache.beam.sdk.schemas.NoSuchSchemaException in project beam by apache.

the class StorageApiWritesShardedRecords method expand.

@Override
public PCollection<Void> expand(PCollection<KV<ShardedKey<DestinationT>, Iterable<byte[]>>> input) {
    String operationName = input.getName() + "/" + getName();
    // Append records to the Storage API streams.
    PCollection<KV<String, Operation>> written = input.apply("Write Records", ParDo.of(new WriteRecordsDoFn(operationName)).withSideInputs(dynamicDestinations.getSideInputs()));
    SchemaCoder<Operation> operationCoder;
    try {
        SchemaRegistry schemaRegistry = input.getPipeline().getSchemaRegistry();
        operationCoder = SchemaCoder.of(schemaRegistry.getSchema(Operation.class), TypeDescriptor.of(Operation.class), schemaRegistry.getToRowFunction(Operation.class), schemaRegistry.getFromRowFunction(Operation.class));
    } catch (NoSuchSchemaException e) {
        throw new RuntimeException(e);
    }
    // Send all successful writes to be flushed.
    return written.setCoder(KvCoder.of(StringUtf8Coder.of(), operationCoder)).apply(Window.<KV<String, Operation>>configure().triggering(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(1)))).discardingFiredPanes()).apply("maxFlushPosition", Combine.perKey(Max.naturalOrder(new Operation(-1, false)))).apply("Flush and finalize writes", ParDo.of(new StorageApiFlushAndFinalizeDoFn(bqServices)));
}

Example 4 with NoSuchSchemaException

use of org.apache.beam.sdk.schemas.NoSuchSchemaException in project beam by apache.

the class PCollection method inferCoderOrFail.

/**
 * If the coder is not explicitly set, this sets the coder for this {@link PCollection} to the
 * best coder that can be inferred based upon the known {@link TypeDescriptor}. By default, this
 * is null, but can and should be improved by subclasses.
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
private CoderOrFailure<T> inferCoderOrFail(PInput input, PTransform<?, ?> transform, CoderRegistry coderRegistry, SchemaRegistry schemaRegistry) {
    // First option for a coder: use the Coder set on this PValue.
    if (coderOrFailure.coder != null) {
        return coderOrFailure;
    }
    // Second option for a coder: use the default Coder from the producing PTransform.
    CannotProvideCoderException inputCoderException;
    try {
        return new CoderOrFailure<>(((PTransform) transform).getDefaultOutputCoder(input, this), null);
    } catch (CannotProvideCoderException exc) {
        inputCoderException = exc;
    }
    TypeDescriptor<T> token = getTypeDescriptor();
    // If there is a schema registered for the type, attempt to create a SchemaCoder.
    if (token != null) {
        try {
            SchemaCoder<T> schemaCoder = SchemaCoder.of(schemaRegistry.getSchema(token), token, schemaRegistry.getToRowFunction(token), schemaRegistry.getFromRowFunction(token));
            return new CoderOrFailure<>(schemaCoder, null);
        } catch (NoSuchSchemaException esc) {
        // No schema.
        }
    }
    // Fourth option for a coder: Look in the coder registry.
    CannotProvideCoderException inferFromTokenException = null;
    if (token != null) {
        try {
            return new CoderOrFailure<>(coderRegistry.getCoder(token), null);
        } catch (CannotProvideCoderException exc) {
            inferFromTokenException = exc;
            // and the error message itself.
            if (transform instanceof ParDo.MultiOutput && exc.getReason() == ReasonCode.TYPE_ERASURE) {
                inferFromTokenException = new CannotProvideCoderException(exc.getMessage() + " If this error occurs for an output of the producing ParDo, verify that the " + "TupleTag for this output is constructed with proper type information (see " + "TupleTag Javadoc) or explicitly set the Coder to use if this is not possible.");
            }
        }
    }
    // Build up the error message and list of causes.
    StringBuilder messageBuilder = new StringBuilder().append("Unable to return a default Coder for ").append(this).append(". Correct one of the following root causes:");
    // No exception, but give the user a message about .setCoder() has not been called.
    messageBuilder.append("\n  No Coder has been manually specified; ").append(" you may do so using .setCoder().");
    if (inferFromTokenException != null) {
        messageBuilder.append("\n  Inferring a Coder from the CoderRegistry failed: ").append(inferFromTokenException.getMessage());
    }
    if (inputCoderException != null) {
        messageBuilder.append("\n  Using the default output Coder from the producing PTransform failed: ").append(inputCoderException.getMessage());
    }
    // Build and throw the exception.
    return new CoderOrFailure<>(null, messageBuilder.toString());
}

Example 5 with NoSuchSchemaException

use of org.apache.beam.sdk.schemas.NoSuchSchemaException in project beam by apache.

the class Neo4jIO method getOutputPCollection.

private static <ParameterT, OutputT> PCollection<OutputT> getOutputPCollection(PCollection<ParameterT> input, DoFn<ParameterT, OutputT> writeFn, @Nullable Coder<OutputT> coder) {
    PCollection<OutputT> output = input.apply(ParDo.of(writeFn));
    if (coder != null) {
        output.setCoder(coder);
        try {
            TypeDescriptor<OutputT> typeDesc = coder.getEncodedTypeDescriptor();
            SchemaRegistry registry = input.getPipeline().getSchemaRegistry();
            Schema schema = registry.getSchema(typeDesc);
            output.setSchema(schema, typeDesc, registry.getToRowFunction(typeDesc), registry.getFromRowFunction(typeDesc));
        } catch (NoSuchSchemaException e) {
        // ignore
        }
    }
    return output;
}