Examples with InputContext co.cask.cdap.api.flow.flowlet.InputContext used on opensource projects

Example 1 with InputContext

use of co.cask.cdap.api.flow.flowlet.InputContext in project cdap by caskdata.

the class ProcessMethodExtractor method visit.

@Override
public void visit(Object instance, Type inspectType, Type declareType, Method method) throws Exception {
    if (!seenMethods.add(FlowletMethod.create(method, inspectType))) {
        // up the class hierarchy.
        return;
    }
    ProcessInput processInputAnnotation = method.getAnnotation(ProcessInput.class);
    Tick tickAnnotation = method.getAnnotation(Tick.class);
    TypeToken<?> inspectTypeToken = TypeToken.of(inspectType);
    if (processInputAnnotation == null && tickAnnotation == null) {
        return;
    }
    // Check for tick method
    if (tickAnnotation != null) {
        checkArgument(processInputAnnotation == null, "Tick method %s.%s should not have ProcessInput.", inspectTypeToken.getRawType().getName(), method);
        checkArgument(method.getParameterTypes().length == 0, "Tick method %s.%s cannot have parameters.", inspectTypeToken.getRawType().getName(), method);
        return;
    }
    Type[] methodParams = method.getGenericParameterTypes();
    checkArgument(methodParams.length > 0 && methodParams.length <= 2, "Parameter missing from process method %s.%s.", inspectTypeToken.getRawType().getName(), method);
    // If there is more than one parameter there can only be exactly two; the second one must be InputContext type
    if (methodParams.length == 2) {
        checkArgument(InputContext.class.equals(TypeToken.of(methodParams[1]).getRawType()), "Second parameter must be InputContext type for process method %s.%s.", inspectTypeToken.getRawType().getName(), method);
    }
    // Extract the Input type from the first parameter of the process method
    Type inputType = getInputType(inspectTypeToken, method, inspectTypeToken.resolveType(methodParams[0]).getType());
    checkArgument(Reflections.isResolved(inputType), "Invalid type in %s.%s. Only Class or ParameterizedType are supported.", inspectTypeToken.getRawType().getName(), method);
    List<String> inputNames = new LinkedList<>();
    if (processInputAnnotation.value().length == 0) {
        inputNames.add(FlowletDefinition.ANY_INPUT);
    } else {
        Collections.addAll(inputNames, processInputAnnotation.value());
    }
    for (String inputName : inputNames) {
        Set<Type> types = inputTypes.get(inputName);
        if (types == null) {
            types = new HashSet<>();
            inputTypes.put(inputName, types);
        }
        checkArgument(types.add(inputType), "Same type already defined for the same input name %s in process method %s.%s.", inputName, inspectTypeToken.getRawType().getName(), method);
    }
}

Example 2 with InputContext

use of co.cask.cdap.api.flow.flowlet.InputContext in project cdap by caskdata.

the class FlowletProcessDriver method postProcess.

/**
   * Process the process result. This method never throws.
   */
private void postProcess(ProcessMethodCallback callback, TransactionContext txContext, InputDatum input, ProcessMethod.ProcessResult result) {
    InputContext inputContext = input.getInputContext();
    Throwable failureCause = null;
    FailureReason.Type failureType = FailureReason.Type.IO_ERROR;
    try {
        if (result.isSuccess()) {
            // If it is a retry input, force the dequeued entries into current transaction.
            if (input.getRetry() > 0) {
                input.reclaim();
            }
            txContext.finish();
        } else {
            failureCause = result.getCause();
            failureType = FailureReason.Type.USER;
            txContext.abort();
        }
    } catch (Throwable e) {
        LOG.error("Transaction operation failed: {}", e.getMessage(), e);
        failureType = FailureReason.Type.IO_ERROR;
        if (failureCause == null) {
            failureCause = e;
        }
        try {
            if (result.isSuccess()) {
                txContext.abort();
            }
        } catch (Throwable ex) {
            LOG.error("Fail to abort transaction: {}", inputContext, ex);
        }
    }
    try {
        if (failureCause == null) {
            callback.onSuccess(result.getEvent(), inputContext);
        } else {
            callback.onFailure(result.getEvent(), inputContext, new FailureReason(failureType, failureCause.getMessage(), failureCause), createInputAcknowledger(input));
        }
    } catch (Throwable t) {
        LOG.error("Failed to invoke callback.", t);
    }
}

Example 3 with InputContext

use of co.cask.cdap.api.flow.flowlet.InputContext in project cdap by caskdata.

the class FlowletProcessDriver method processMethodCallback.

private <T> ProcessMethodCallback processMethodCallback(final PriorityQueue<FlowletProcessEntry<?>> processQueue, final FlowletProcessEntry<T> processEntry, final InputDatum<T> input) {
    // If it is generator flowlet, processCount is 1.
    final int processedCount = processEntry.getProcessSpec().getProcessMethod().needsInput() ? input.size() : 1;
    return new ProcessMethodCallback() {

        private final LoadingCache<String, MetricsContext> queueMetricsCollectors = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS).build(new CacheLoader<String, MetricsContext>() {

            @Override
            public MetricsContext load(String key) throws Exception {
                return flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, key);
            }
        });

        @Override
        public void onSuccess(Object object, InputContext inputContext) {
            try {
                gaugeEventProcessed(input.getQueueName());
                txCallback.onSuccess(object, inputContext);
            } catch (Throwable t) {
                LOG.error("Exception on onSuccess call: {}", flowletContext, t);
            } finally {
                enqueueEntry();
            }
        }

        @Override
        public void onFailure(Object inputObject, InputContext inputContext, FailureReason reason, InputAcknowledger inputAcknowledger) {
            LOG.warn("Process failure: {}, {}, input: {}", flowletContext, reason.getMessage(), input, reason.getCause());
            FailurePolicy failurePolicy;
            try {
                flowletContext.getProgramMetrics().increment("process.errors", 1);
                failurePolicy = txCallback.onFailure(inputObject, inputContext, reason);
                if (failurePolicy == null) {
                    failurePolicy = FailurePolicy.RETRY;
                    LOG.info("Callback returns null for failure policy. Default to {}.", failurePolicy);
                }
            } catch (Throwable t) {
                LOG.error("Exception on onFailure call: {}", flowletContext, t);
                failurePolicy = FailurePolicy.RETRY;
            }
            if (input.getRetry() >= processEntry.getProcessSpec().getProcessMethod().getMaxRetries()) {
                LOG.info("Too many retries, ignores the input: {}", input);
                failurePolicy = FailurePolicy.IGNORE;
            }
            if (failurePolicy == FailurePolicy.RETRY) {
                FlowletProcessEntry retryEntry = processEntry.isRetry() ? processEntry : FlowletProcessEntry.create(processEntry.getProcessSpec(), new ProcessSpecification<>(new SingleItemQueueReader<>(input), processEntry.getProcessSpec().getProcessMethod(), null));
                processQueue.offer(retryEntry);
            } else if (failurePolicy == FailurePolicy.IGNORE) {
                try {
                    gaugeEventProcessed(input.getQueueName());
                    inputAcknowledger.ack();
                } catch (Throwable t) {
                    LOG.error("Fatal problem, fail to ack an input: {}", flowletContext, t);
                } finally {
                    enqueueEntry();
                }
            }
        }

        private void enqueueEntry() {
            processQueue.offer(processEntry.resetRetry());
        }

        private void gaugeEventProcessed(QueueName inputQueueName) {
            if (processEntry.isTick()) {
                flowletContext.getProgramMetrics().increment("process.ticks.processed", processedCount);
            } else if (inputQueueName == null) {
                flowletContext.getProgramMetrics().increment("process.events.processed", processedCount);
            } else {
                queueMetricsCollectors.getUnchecked(inputQueueName.getSimpleName()).increment("process.events.processed", processedCount);
            }
        }
    };
}

Example 4 with InputContext

use of co.cask.cdap.api.flow.flowlet.InputContext in project cdap by caskdata.

the class ReflectionProcessMethod method invoke.

@SuppressWarnings("unchecked")
@Override
public ProcessResult<T> invoke(InputDatum<T> input) {
    try {
        Preconditions.checkState(!hasParam || input.needProcess(), "Empty input provided to method that needs input.");
        InputContext inputContext = input.getInputContext();
        if (hasParam) {
            if (needsIterator) {
                invoke(method, input.iterator(), inputContext);
            } else {
                for (T event : input) {
                    invoke(method, event, inputContext);
                }
            }
        } else {
            method.invoke(flowlet);
        }
        return createResult(input, null);
    } catch (Throwable t) {
        return createResult(input, t.getCause());
    }
}