Search in sources :

Example 1 with PipelineRunner

use of org.apache.beam.sdk.PipelineRunner in project beam by apache.

the class PipelineOptionsFactory method parseObjects.

/**
 * Using the parsed string arguments, we convert the strings to the expected return type of the
 * methods that are found on the passed-in class.
 *
 * <p>For any return type that is expected to be an array or a collection, we further split up
 * each string on ','.
 *
 * <p>We special case the "runner" option. It is mapped to the class of the {@link PipelineRunner}
 * based off of the {@link PipelineRunner PipelineRunners} simple class name. If the provided
 * runner name is not registered via a {@link PipelineRunnerRegistrar}, we attempt to obtain the
 * class that the name represents using {@link Class#forName(String)} and use the result class if
 * it subclasses {@link PipelineRunner}.
 *
 * <p>If strict parsing is enabled, unknown options or options that cannot be converted to the
 * expected java type using an {@link ObjectMapper} will be ignored.
 */
private static <T extends PipelineOptions> Map<String, Object> parseObjects(Class<T> klass, ListMultimap<String, String> options, boolean strictParsing) {
    Map<String, Method> propertyNamesToGetters = Maps.newHashMap();
    Cache cache = CACHE.get();
    cache.validateWellFormed(klass);
    @SuppressWarnings("unchecked") Iterable<PropertyDescriptor> propertyDescriptors = cache.getPropertyDescriptors(Stream.concat(getRegisteredOptions().stream(), Stream.of(klass)).collect(Collectors.toSet()));
    for (PropertyDescriptor descriptor : propertyDescriptors) {
        propertyNamesToGetters.put(descriptor.getName(), descriptor.getReadMethod());
    }
    Map<String, Object> convertedOptions = Maps.newHashMap();
    for (final Map.Entry<String, Collection<String>> entry : options.asMap().entrySet()) {
        try {
            // Either off by one or off by two character errors.
            if (!propertyNamesToGetters.containsKey(entry.getKey())) {
                SortedSet<String> closestMatches = new TreeSet<>(Sets.filter(propertyNamesToGetters.keySet(), input -> StringUtils.getLevenshteinDistance(entry.getKey(), input) <= 2));
                switch(closestMatches.size()) {
                    case 0:
                        throw new IllegalArgumentException(String.format("Class %s missing a property named '%s'.", klass, entry.getKey()));
                    case 1:
                        throw new IllegalArgumentException(String.format("Class %s missing a property named '%s'. Did you mean '%s'?", klass, entry.getKey(), Iterables.getOnlyElement(closestMatches)));
                    default:
                        throw new IllegalArgumentException(String.format("Class %s missing a property named '%s'. Did you mean one of %s?", klass, entry.getKey(), closestMatches));
                }
            }
            Method method = propertyNamesToGetters.get(entry.getKey());
            // Only allow empty argument values for String, String Array, and Collection<String>.
            Class<?> returnType = method.getReturnType();
            JavaType type = MAPPER.getTypeFactory().constructType(method.getGenericReturnType());
            if ("runner".equals(entry.getKey())) {
                String runner = Iterables.getOnlyElement(entry.getValue());
                final Map<String, Class<? extends PipelineRunner<?>>> pipelineRunners = cache.supportedPipelineRunners;
                if (pipelineRunners.containsKey(runner.toLowerCase())) {
                    convertedOptions.put("runner", pipelineRunners.get(runner.toLowerCase(ROOT)));
                } else {
                    try {
                        Class<?> runnerClass = Class.forName(runner, true, ReflectHelpers.findClassLoader());
                        if (!PipelineRunner.class.isAssignableFrom(runnerClass)) {
                            throw new IllegalArgumentException(String.format("Class '%s' does not implement PipelineRunner. " + "Supported pipeline runners %s", runner, cache.getSupportedRunners()));
                        }
                        convertedOptions.put("runner", runnerClass);
                    } catch (ClassNotFoundException e) {
                        String msg = String.format("Unknown 'runner' specified '%s', supported pipeline runners %s", runner, cache.getSupportedRunners());
                        throw new IllegalArgumentException(msg, e);
                    }
                }
            } else if (isCollectionOrArrayOfAllowedTypes(returnType, type)) {
                // Split any strings with ","
                List<String> values = entry.getValue().stream().flatMap(input -> Arrays.stream(input.split(","))).collect(Collectors.toList());
                if (values.contains("")) {
                    checkEmptyStringAllowed(returnType, type, method.getGenericReturnType().toString());
                }
                convertedOptions.put(entry.getKey(), MAPPER.convertValue(values, type));
            } else if (isSimpleType(returnType, type)) {
                String value = Iterables.getOnlyElement(entry.getValue());
                if (value.isEmpty()) {
                    checkEmptyStringAllowed(returnType, type, method.getGenericReturnType().toString());
                }
                convertedOptions.put(entry.getKey(), MAPPER.convertValue(value, type));
            } else {
                String value = Iterables.getOnlyElement(entry.getValue());
                if (value.isEmpty()) {
                    checkEmptyStringAllowed(returnType, type, method.getGenericReturnType().toString());
                }
                try {
                    convertedOptions.put(entry.getKey(), tryParseObject(value, method));
                } catch (IOException e) {
                    throw new IllegalArgumentException("Unable to parse JSON value " + value, e);
                }
            }
        } catch (IllegalArgumentException e) {
            if (strictParsing) {
                throw e;
            } else {
                LOG.warn("Strict parsing is disabled, ignoring option '{}' with value '{}' because {}", entry.getKey(), entry.getValue(), e.getMessage());
            }
        }
    }
    return convertedOptions;
}
Also used : Arrays(java.util.Arrays) PipelineOptionDescriptor(org.apache.beam.model.jobmanagement.v1.JobApi.PipelineOptionDescriptor) Joiner(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) TreeTraversingParser(com.fasterxml.jackson.databind.node.TreeTraversingParser) DefaultSerializerProvider(com.fasterxml.jackson.databind.ser.DefaultSerializerProvider) ImmutableSet(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet) Strings(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings) KeyFor(org.checkerframework.checker.nullness.qual.KeyFor) Map(java.util.Map) JsonSerializer(com.fasterxml.jackson.databind.JsonSerializer) JsonNode(com.fasterxml.jackson.databind.JsonNode) BigInteger(java.math.BigInteger) Predicates(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates) Iterators(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators) BeanProperty(com.fasterxml.jackson.databind.BeanProperty) Set(java.util.Set) TreeMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap) IntrospectionException(java.beans.IntrospectionException) Stream(java.util.stream.Stream) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting) PropertyDescriptor(java.beans.PropertyDescriptor) RowSortedTable(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.RowSortedTable) TypeResolutionContext(com.fasterxml.jackson.databind.introspect.TypeResolutionContext) JsonDeserialize(com.fasterxml.jackson.databind.annotation.JsonDeserialize) Preconditions.checkNotNull(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull) InjectableValues(com.fasterxml.jackson.databind.InjectableValues) Proxy(java.lang.reflect.Proxy) Experimental(org.apache.beam.sdk.annotations.Experimental) DefaultDeserializationContext(com.fasterxml.jackson.databind.deser.DefaultDeserializationContext) Optional(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional) TreeSet(java.util.TreeSet) ImmutableListMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap) ArrayList(java.util.ArrayList) Introspector(java.beans.Introspector) SortedSetMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SortedSetMultimap) PipelineRunner(org.apache.beam.sdk.PipelineRunner) BeanInfo(java.beans.BeanInfo) PipelineOptionType(org.apache.beam.model.jobmanagement.v1.JobApi.PipelineOptionType) Preconditions.checkArgument(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) StreamSupport(java.util.stream.StreamSupport) StringUtils(org.apache.beam.sdk.util.StringUtils) JavaType(com.fasterxml.jackson.databind.JavaType) LinkedHashSet(java.util.LinkedHashSet) PipelineRunnerRegistrar(org.apache.beam.sdk.runners.PipelineRunnerRegistrar) IOException(java.io.IOException) TreeMap(java.util.TreeMap) BeanPropertyDefinition(com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition) TokenBuffer(com.fasterxml.jackson.databind.util.TokenBuffer) Ordering(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering) SortedSet(java.util.SortedSet) TypeDeserializer(com.fasterxml.jackson.databind.jsontype.TypeDeserializer) LoggerFactory(org.slf4j.LoggerFactory) TreeBasedTable(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeBasedTable) Sets(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) AnnotatedMember(com.fasterxml.jackson.databind.introspect.AnnotatedMember) TypeWrappedDeserializer(com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer) JsonDeserializer(com.fasterxml.jackson.databind.JsonDeserializer) JsonParseException(com.fasterxml.jackson.core.JsonParseException) Method(java.lang.reflect.Method) MethodProperty(com.fasterxml.jackson.databind.deser.impl.MethodProperty) SimpleBeanPropertyDefinition(com.fasterxml.jackson.databind.util.SimpleBeanPropertyDefinition) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) List(java.util.List) Type(java.lang.reflect.Type) Modifier(java.lang.reflect.Modifier) ReflectHelpers(org.apache.beam.sdk.util.common.ReflectHelpers) Annotation(java.lang.annotation.Annotation) ListMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) SortedMap(java.util.SortedMap) Required(org.apache.beam.sdk.options.Validation.Required) CaseFormat(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ImmutableSortedSet(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet) JsonSerialize(com.fasterxml.jackson.databind.annotation.JsonSerialize) TypeBindings(com.fasterxml.jackson.databind.type.TypeBindings) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) AnnotationCollector(com.fasterxml.jackson.databind.introspect.AnnotationCollector) ROOT(java.util.Locale.ROOT) AnnotatedMethod(com.fasterxml.jackson.databind.introspect.AnnotatedMethod) NoSuchElementException(java.util.NoSuchElementException) Nonnull(javax.annotation.Nonnull) Predicate(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate) Nullable(org.checkerframework.checker.nullness.qual.Nullable) PrintStream(java.io.PrintStream) DisplayData(org.apache.beam.sdk.transforms.display.DisplayData) Logger(org.slf4j.Logger) JsonParser(com.fasterxml.jackson.core.JsonParser) Iterator(java.util.Iterator) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Lists(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists) Comparator(java.util.Comparator) Collections(java.util.Collections) PropertyDescriptor(java.beans.PropertyDescriptor) PipelineRunner(org.apache.beam.sdk.PipelineRunner) Method(java.lang.reflect.Method) AnnotatedMethod(com.fasterxml.jackson.databind.introspect.AnnotatedMethod) IOException(java.io.IOException) JavaType(com.fasterxml.jackson.databind.JavaType) TreeSet(java.util.TreeSet) Collection(java.util.Collection) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) Map(java.util.Map) TreeMap(java.util.TreeMap) SortedMap(java.util.SortedMap)

Aggregations

JsonIgnore (com.fasterxml.jackson.annotation.JsonIgnore)1 JsonParseException (com.fasterxml.jackson.core.JsonParseException)1 JsonParser (com.fasterxml.jackson.core.JsonParser)1 BeanProperty (com.fasterxml.jackson.databind.BeanProperty)1 InjectableValues (com.fasterxml.jackson.databind.InjectableValues)1 JavaType (com.fasterxml.jackson.databind.JavaType)1 JsonDeserializer (com.fasterxml.jackson.databind.JsonDeserializer)1 JsonMappingException (com.fasterxml.jackson.databind.JsonMappingException)1 JsonNode (com.fasterxml.jackson.databind.JsonNode)1 JsonSerializer (com.fasterxml.jackson.databind.JsonSerializer)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 JsonDeserialize (com.fasterxml.jackson.databind.annotation.JsonDeserialize)1 JsonSerialize (com.fasterxml.jackson.databind.annotation.JsonSerialize)1 DefaultDeserializationContext (com.fasterxml.jackson.databind.deser.DefaultDeserializationContext)1 MethodProperty (com.fasterxml.jackson.databind.deser.impl.MethodProperty)1 TypeWrappedDeserializer (com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer)1 AnnotatedMember (com.fasterxml.jackson.databind.introspect.AnnotatedMember)1 AnnotatedMethod (com.fasterxml.jackson.databind.introspect.AnnotatedMethod)1 AnnotationCollector (com.fasterxml.jackson.databind.introspect.AnnotationCollector)1 BeanPropertyDefinition (com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition)1