Examples with Tick co.cask.cdap.api.annotation.Tick used on opensource projects

Example 1 with Tick

use of co.cask.cdap.api.annotation.Tick in project cdap by caskdata.

the class ProcessMethodExtractor method visit.

@Override
public void visit(Object instance, Type inspectType, Type declareType, Method method) throws Exception {
    if (!seenMethods.add(FlowletMethod.create(method, inspectType))) {
        // up the class hierarchy.
        return;
    }
    ProcessInput processInputAnnotation = method.getAnnotation(ProcessInput.class);
    Tick tickAnnotation = method.getAnnotation(Tick.class);
    TypeToken<?> inspectTypeToken = TypeToken.of(inspectType);
    if (processInputAnnotation == null && tickAnnotation == null) {
        return;
    }
    // Check for tick method
    if (tickAnnotation != null) {
        checkArgument(processInputAnnotation == null, "Tick method %s.%s should not have ProcessInput.", inspectTypeToken.getRawType().getName(), method);
        checkArgument(method.getParameterTypes().length == 0, "Tick method %s.%s cannot have parameters.", inspectTypeToken.getRawType().getName(), method);
        return;
    }
    Type[] methodParams = method.getGenericParameterTypes();
    checkArgument(methodParams.length > 0 && methodParams.length <= 2, "Parameter missing from process method %s.%s.", inspectTypeToken.getRawType().getName(), method);
    // If there is more than one parameter there can only be exactly two; the second one must be InputContext type
    if (methodParams.length == 2) {
        checkArgument(InputContext.class.equals(TypeToken.of(methodParams[1]).getRawType()), "Second parameter must be InputContext type for process method %s.%s.", inspectTypeToken.getRawType().getName(), method);
    }
    // Extract the Input type from the first parameter of the process method
    Type inputType = getInputType(inspectTypeToken, method, inspectTypeToken.resolveType(methodParams[0]).getType());
    checkArgument(Reflections.isResolved(inputType), "Invalid type in %s.%s. Only Class or ParameterizedType are supported.", inspectTypeToken.getRawType().getName(), method);
    List<String> inputNames = new LinkedList<>();
    if (processInputAnnotation.value().length == 0) {
        inputNames.add(FlowletDefinition.ANY_INPUT);
    } else {
        Collections.addAll(inputNames, processInputAnnotation.value());
    }
    for (String inputName : inputNames) {
        Set<Type> types = inputTypes.get(inputName);
        if (types == null) {
            types = new HashSet<>();
            inputTypes.put(inputName, types);
        }
        checkArgument(types.add(inputType), "Same type already defined for the same input name %s in process method %s.%s.", inputName, inspectTypeToken.getRawType().getName(), method);
    }
}

Example 2 with Tick

use of co.cask.cdap.api.annotation.Tick in project cdap by caskdata.

the class FlowletProgramRunner method processSpecificationFactory.

private ProcessSpecificationFactory processSpecificationFactory(final BasicFlowletContext flowletContext, final DataFabricFacade dataFabricFacade, final QueueReaderFactory queueReaderFactory, final String flowletName, final Table<Node, String, Set<QueueSpecification>> queueSpecs, final ImmutableList.Builder<ConsumerSupplier<?>> queueConsumerSupplierBuilder, final SchemaCache schemaCache) {
    return new ProcessSpecificationFactory() {

        @Override
        public <T> ProcessSpecification create(Set<String> inputNames, Schema schema, TypeToken<T> dataType, ProcessMethod<T> method, ConsumerConfig consumerConfig, int batchSize, Tick tickAnnotation) throws Exception {
            List<QueueReader<T>> queueReaders = Lists.newLinkedList();
            for (Map.Entry<Node, Set<QueueSpecification>> entry : queueSpecs.column(flowletName).entrySet()) {
                for (QueueSpecification queueSpec : entry.getValue()) {
                    final QueueName queueName = queueSpec.getQueueName();
                    if (queueSpec.getInputSchema().equals(schema) && (inputNames.contains(queueName.getSimpleName()) || inputNames.contains(FlowletDefinition.ANY_INPUT))) {
                        Node sourceNode = entry.getKey();
                        if (sourceNode.getType() == FlowletConnection.Type.STREAM) {
                            ConsumerSupplier<StreamConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig);
                            queueConsumerSupplierBuilder.add(consumerSupplier);
                            // No decoding is needed, as a process method can only have StreamEvent as type for consuming stream
                            Function<StreamEvent, T> decoder = wrapInputDecoder(flowletContext, null, queueName, new Function<StreamEvent, T>() {

                                @Override
                                @SuppressWarnings("unchecked")
                                public T apply(StreamEvent input) {
                                    return (T) input;
                                }
                            });
                            queueReaders.add(queueReaderFactory.createStreamReader(queueName.toStreamId(), consumerSupplier, batchSize, decoder));
                        } else {
                            int numGroups = getNumGroups(Iterables.concat(queueSpecs.row(entry.getKey()).values()), queueName);
                            Function<ByteBuffer, T> decoder = wrapInputDecoder(// the producer flowlet,
                            flowletContext, // the producer flowlet,
                            entry.getKey().getName(), queueName, createInputDatumDecoder(dataType, schema, schemaCache));
                            ConsumerSupplier<QueueConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig, numGroups);
                            queueConsumerSupplierBuilder.add(consumerSupplier);
                            queueReaders.add(queueReaderFactory.createQueueReader(consumerSupplier, batchSize, decoder));
                        }
                    }
                }
            }
            // If inputs is needed but there is no available input queue, return null
            if (!inputNames.isEmpty() && queueReaders.isEmpty()) {
                return null;
            }
            return new ProcessSpecification<>(new RoundRobinQueueReader<>(queueReaders), method, tickAnnotation);
        }
    };
}

Example 3 with Tick

use of co.cask.cdap.api.annotation.Tick in project cdap by caskdata.

the class FlowletProgramRunner method createProcessSpecification.

/**
   * Creates all {@link ProcessSpecification} for the process methods of the flowlet class.
   *
   * @param flowletType Type of the flowlet class represented by {@link TypeToken}.
   * @param processMethodFactory A {@link ProcessMethodFactory} for creating {@link ProcessMethod}.
   * @param processSpecFactory A {@link ProcessSpecificationFactory} for creating {@link ProcessSpecification}.
   * @param result A {@link Collection} for storing newly created {@link ProcessSpecification}.
   * @return The same {@link Collection} as the {@code result} parameter.
   */
@SuppressWarnings("unchecked")
private <T extends Collection<ProcessSpecification<?>>> T createProcessSpecification(BasicFlowletContext flowletContext, TypeToken<? extends Flowlet> flowletType, ProcessMethodFactory processMethodFactory, ProcessSpecificationFactory processSpecFactory, T result) throws Exception {
    Set<FlowletMethod> seenMethods = Sets.newHashSet();
    // Walk up the hierarchy of flowlet class to get all ProcessInput and Tick methods
    for (TypeToken<?> type : flowletType.getTypes().classes()) {
        if (type.getRawType().equals(Object.class)) {
            break;
        }
        // Extracts all process and tick methods
        for (Method method : type.getRawType().getDeclaredMethods()) {
            if (method.isSynthetic() || method.isBridge()) {
                continue;
            }
            if (!seenMethods.add(FlowletMethod.create(method, flowletType.getType()))) {
                // up the class hierarchy.
                continue;
            }
            ProcessInput processInputAnnotation = method.getAnnotation(ProcessInput.class);
            Tick tickAnnotation = method.getAnnotation(Tick.class);
            if (processInputAnnotation == null && tickAnnotation == null) {
                // Neither a process nor a tick method.
                continue;
            }
            int maxRetries = (tickAnnotation == null) ? processInputAnnotation.maxRetries() : tickAnnotation.maxRetries();
            ProcessMethod processMethod = processMethodFactory.create(method, maxRetries);
            Set<String> inputNames;
            Schema schema;
            TypeToken<?> dataType;
            ConsumerConfig consumerConfig;
            int batchSize = 1;
            if (tickAnnotation != null) {
                inputNames = ImmutableSet.of();
                consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
                schema = Schema.of(Schema.Type.NULL);
                dataType = TypeToken.of(void.class);
            } else {
                inputNames = Sets.newHashSet(processInputAnnotation.value());
                if (inputNames.isEmpty()) {
                    // If there is no input name, it would be ANY_INPUT
                    inputNames.add(FlowletDefinition.ANY_INPUT);
                }
                // If batch mode then generate schema for Iterator's parameter type
                dataType = flowletType.resolveType(method.getGenericParameterTypes()[0]);
                consumerConfig = getConsumerConfig(flowletContext, method);
                Integer processBatchSize = getBatchSize(method, flowletContext);
                if (processBatchSize != null) {
                    if (dataType.getRawType().equals(Iterator.class)) {
                        Preconditions.checkArgument(dataType.getType() instanceof ParameterizedType, "Only ParameterizedType is supported for batch Iterator.");
                        dataType = flowletType.resolveType(((ParameterizedType) dataType.getType()).getActualTypeArguments()[0]);
                    }
                    batchSize = processBatchSize;
                }
                try {
                    schema = schemaGenerator.generate(dataType.getType());
                } catch (UnsupportedTypeException e) {
                    throw Throwables.propagate(e);
                }
            }
            ProcessSpecification processSpec = processSpecFactory.create(inputNames, schema, dataType, processMethod, consumerConfig, batchSize, tickAnnotation);
            // Add processSpec
            if (processSpec != null) {
                result.add(processSpec);
            }
        }
    }
    Preconditions.checkArgument(!result.isEmpty(), "No inputs found for flowlet '%s' of flow '%s' of application '%s' (%s)", flowletContext.getFlowletId(), flowletContext.getFlowId(), flowletContext.getApplicationId(), flowletType);
    return result;
}