Examples with FailureReason co.cask.cdap.api.flow.flowlet.FailureReason used on opensource projects

Example 1 with FailureReason

use of co.cask.cdap.api.flow.flowlet.FailureReason in project cdap by caskdata.

the class FlowletProcessDriver method postProcess.

/**
   * Process the process result. This method never throws.
   */
private void postProcess(ProcessMethodCallback callback, TransactionContext txContext, InputDatum input, ProcessMethod.ProcessResult result) {
    InputContext inputContext = input.getInputContext();
    Throwable failureCause = null;
    FailureReason.Type failureType = FailureReason.Type.IO_ERROR;
    try {
        if (result.isSuccess()) {
            // If it is a retry input, force the dequeued entries into current transaction.
            if (input.getRetry() > 0) {
                input.reclaim();
            }
            txContext.finish();
        } else {
            failureCause = result.getCause();
            failureType = FailureReason.Type.USER;
            txContext.abort();
        }
    } catch (Throwable e) {
        LOG.error("Transaction operation failed: {}", e.getMessage(), e);
        failureType = FailureReason.Type.IO_ERROR;
        if (failureCause == null) {
            failureCause = e;
        }
        try {
            if (result.isSuccess()) {
                txContext.abort();
            }
        } catch (Throwable ex) {
            LOG.error("Fail to abort transaction: {}", inputContext, ex);
        }
    }
    try {
        if (failureCause == null) {
            callback.onSuccess(result.getEvent(), inputContext);
        } else {
            callback.onFailure(result.getEvent(), inputContext, new FailureReason(failureType, failureCause.getMessage(), failureCause), createInputAcknowledger(input));
        }
    } catch (Throwable t) {
        LOG.error("Failed to invoke callback.", t);
    }
}

Example 2 with FailureReason

use of co.cask.cdap.api.flow.flowlet.FailureReason in project cdap by caskdata.

the class FlowletProcessDriver method processMethodCallback.

private <T> ProcessMethodCallback processMethodCallback(final PriorityQueue<FlowletProcessEntry<?>> processQueue, final FlowletProcessEntry<T> processEntry, final InputDatum<T> input) {
    // If it is generator flowlet, processCount is 1.
    final int processedCount = processEntry.getProcessSpec().getProcessMethod().needsInput() ? input.size() : 1;
    return new ProcessMethodCallback() {

        private final LoadingCache<String, MetricsContext> queueMetricsCollectors = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS).build(new CacheLoader<String, MetricsContext>() {

            @Override
            public MetricsContext load(String key) throws Exception {
                return flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, key);
            }
        });

        @Override
        public void onSuccess(Object object, InputContext inputContext) {
            try {
                gaugeEventProcessed(input.getQueueName());
                txCallback.onSuccess(object, inputContext);
            } catch (Throwable t) {
                LOG.error("Exception on onSuccess call: {}", flowletContext, t);
            } finally {
                enqueueEntry();
            }
        }

        @Override
        public void onFailure(Object inputObject, InputContext inputContext, FailureReason reason, InputAcknowledger inputAcknowledger) {
            LOG.warn("Process failure: {}, {}, input: {}", flowletContext, reason.getMessage(), input, reason.getCause());
            FailurePolicy failurePolicy;
            try {
                flowletContext.getProgramMetrics().increment("process.errors", 1);
                failurePolicy = txCallback.onFailure(inputObject, inputContext, reason);
                if (failurePolicy == null) {
                    failurePolicy = FailurePolicy.RETRY;
                    LOG.info("Callback returns null for failure policy. Default to {}.", failurePolicy);
                }
            } catch (Throwable t) {
                LOG.error("Exception on onFailure call: {}", flowletContext, t);
                failurePolicy = FailurePolicy.RETRY;
            }
            if (input.getRetry() >= processEntry.getProcessSpec().getProcessMethod().getMaxRetries()) {
                LOG.info("Too many retries, ignores the input: {}", input);
                failurePolicy = FailurePolicy.IGNORE;
            }
            if (failurePolicy == FailurePolicy.RETRY) {
                FlowletProcessEntry retryEntry = processEntry.isRetry() ? processEntry : FlowletProcessEntry.create(processEntry.getProcessSpec(), new ProcessSpecification<>(new SingleItemQueueReader<>(input), processEntry.getProcessSpec().getProcessMethod(), null));
                processQueue.offer(retryEntry);
            } else if (failurePolicy == FailurePolicy.IGNORE) {
                try {
                    gaugeEventProcessed(input.getQueueName());
                    inputAcknowledger.ack();
                } catch (Throwable t) {
                    LOG.error("Fatal problem, fail to ack an input: {}", flowletContext, t);
                } finally {
                    enqueueEntry();
                }
            }
        }

        private void enqueueEntry() {
            processQueue.offer(processEntry.resetRetry());
        }

        private void gaugeEventProcessed(QueueName inputQueueName) {
            if (processEntry.isTick()) {
                flowletContext.getProgramMetrics().increment("process.ticks.processed", processedCount);
            } else if (inputQueueName == null) {
                flowletContext.getProgramMetrics().increment("process.events.processed", processedCount);
            } else {
                queueMetricsCollectors.getUnchecked(inputQueueName.getSimpleName()).increment("process.events.processed", processedCount);
            }
        }
    };
}