Examples with FlowletMethod co.cask.cdap.internal.specification.FlowletMethod used on opensource projects

Example 1 with FlowletMethod

use of co.cask.cdap.internal.specification.FlowletMethod in project cdap by caskdata.

the class FlowletProgramRunner method createProcessSpecification.

/**
   * Creates all {@link ProcessSpecification} for the process methods of the flowlet class.
   *
   * @param flowletType Type of the flowlet class represented by {@link TypeToken}.
   * @param processMethodFactory A {@link ProcessMethodFactory} for creating {@link ProcessMethod}.
   * @param processSpecFactory A {@link ProcessSpecificationFactory} for creating {@link ProcessSpecification}.
   * @param result A {@link Collection} for storing newly created {@link ProcessSpecification}.
   * @return The same {@link Collection} as the {@code result} parameter.
   */
@SuppressWarnings("unchecked")
private <T extends Collection<ProcessSpecification<?>>> T createProcessSpecification(BasicFlowletContext flowletContext, TypeToken<? extends Flowlet> flowletType, ProcessMethodFactory processMethodFactory, ProcessSpecificationFactory processSpecFactory, T result) throws Exception {
    Set<FlowletMethod> seenMethods = Sets.newHashSet();
    // Walk up the hierarchy of flowlet class to get all ProcessInput and Tick methods
    for (TypeToken<?> type : flowletType.getTypes().classes()) {
        if (type.getRawType().equals(Object.class)) {
            break;
        }
        // Extracts all process and tick methods
        for (Method method : type.getRawType().getDeclaredMethods()) {
            if (method.isSynthetic() || method.isBridge()) {
                continue;
            }
            if (!seenMethods.add(FlowletMethod.create(method, flowletType.getType()))) {
                // up the class hierarchy.
                continue;
            }
            ProcessInput processInputAnnotation = method.getAnnotation(ProcessInput.class);
            Tick tickAnnotation = method.getAnnotation(Tick.class);
            if (processInputAnnotation == null && tickAnnotation == null) {
                // Neither a process nor a tick method.
                continue;
            }
            int maxRetries = (tickAnnotation == null) ? processInputAnnotation.maxRetries() : tickAnnotation.maxRetries();
            ProcessMethod processMethod = processMethodFactory.create(method, maxRetries);
            Set<String> inputNames;
            Schema schema;
            TypeToken<?> dataType;
            ConsumerConfig consumerConfig;
            int batchSize = 1;
            if (tickAnnotation != null) {
                inputNames = ImmutableSet.of();
                consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
                schema = Schema.of(Schema.Type.NULL);
                dataType = TypeToken.of(void.class);
            } else {
                inputNames = Sets.newHashSet(processInputAnnotation.value());
                if (inputNames.isEmpty()) {
                    // If there is no input name, it would be ANY_INPUT
                    inputNames.add(FlowletDefinition.ANY_INPUT);
                }
                // If batch mode then generate schema for Iterator's parameter type
                dataType = flowletType.resolveType(method.getGenericParameterTypes()[0]);
                consumerConfig = getConsumerConfig(flowletContext, method);
                Integer processBatchSize = getBatchSize(method, flowletContext);
                if (processBatchSize != null) {
                    if (dataType.getRawType().equals(Iterator.class)) {
                        Preconditions.checkArgument(dataType.getType() instanceof ParameterizedType, "Only ParameterizedType is supported for batch Iterator.");
                        dataType = flowletType.resolveType(((ParameterizedType) dataType.getType()).getActualTypeArguments()[0]);
                    }
                    batchSize = processBatchSize;
                }
                try {
                    schema = schemaGenerator.generate(dataType.getType());
                } catch (UnsupportedTypeException e) {
                    throw Throwables.propagate(e);
                }
            }
            ProcessSpecification processSpec = processSpecFactory.create(inputNames, schema, dataType, processMethod, consumerConfig, batchSize, tickAnnotation);
            // Add processSpec
            if (processSpec != null) {
                result.add(processSpec);
            }
        }
    }
    Preconditions.checkArgument(!result.isEmpty(), "No inputs found for flowlet '%s' of flow '%s' of application '%s' (%s)", flowletContext.getFlowletId(), flowletContext.getFlowId(), flowletContext.getApplicationId(), flowletType);
    return result;
}

Example 2 with FlowletMethod

use of co.cask.cdap.internal.specification.FlowletMethod in project cdap by caskdata.

the class FlowUtils method addConsumerGroup.

/**
   * Finds all consumer group for the given queue from the given flowlet.
   */
private static void addConsumerGroup(final QueueSpecification queueSpec, final Type flowletType, final long groupId, final int groupSize, final SchemaGenerator schemaGenerator, final Collection<ConsumerGroupConfig> groupConfigs) {
    final Set<FlowletMethod> seenMethods = Sets.newHashSet();
    Reflections.visit(null, flowletType, new MethodVisitor() {

        @Override
        public void visit(Object instance, Type inspectType, Type declareType, Method method) throws Exception {
            if (!seenMethods.add(FlowletMethod.create(method, inspectType))) {
                // up the class hierarchy.
                return;
            }
            ProcessInput processInputAnnotation = method.getAnnotation(ProcessInput.class);
            if (processInputAnnotation == null) {
                // Consumer has to be process method
                return;
            }
            Set<String> inputNames = Sets.newHashSet(processInputAnnotation.value());
            if (inputNames.isEmpty()) {
                // If there is no input name, it would be ANY_INPUT
                inputNames.add(FlowletDefinition.ANY_INPUT);
            }
            TypeToken<?> inspectTypeToken = TypeToken.of(inspectType);
            TypeToken<?> dataType = inspectTypeToken.resolveType(method.getGenericParameterTypes()[0]);
            // For batch mode and if the parameter is Iterator, need to get the actual data type from the Iterator.
            if (method.isAnnotationPresent(Batch.class) && Iterator.class.equals(dataType.getRawType())) {
                Preconditions.checkArgument(dataType.getType() instanceof ParameterizedType, "Only ParameterizedType is supported for batch Iterator.");
                dataType = inspectTypeToken.resolveType(((ParameterizedType) dataType.getType()).getActualTypeArguments()[0]);
            }
            Schema schema = schemaGenerator.generate(dataType.getType());
            if (queueSpec.getInputSchema().equals(schema) && (inputNames.contains(queueSpec.getQueueName().getSimpleName()) || inputNames.contains(FlowletDefinition.ANY_INPUT))) {
                groupConfigs.add(createConsumerGroupConfig(groupId, groupSize, method));
            }
        }
    });
}