Search in sources :

Example 1 with RollbackOnFailure

use of org.apache.nifi.processor.util.pattern.RollbackOnFailure in project nifi by apache.

the class PutSQL method onBatchUpdateError.

private ExceptionHandler.OnError<FunctionContext, StatementFlowFileEnclosure> onBatchUpdateError(final ProcessContext context, final ProcessSession session, final RoutingResult result) {
    return RollbackOnFailure.createOnError((c, enclosure, r, e) -> {
        // If rollbackOnFailure is enabled, the error will be thrown as ProcessException instead.
        if (e instanceof BatchUpdateException && !c.isRollbackOnFailure()) {
            // If we get a BatchUpdateException, then we want to determine which FlowFile caused the failure,
            // and route that FlowFile to failure while routing those that finished processing to success and those
            // that have not yet been executed to retry.
            // Currently fragmented transaction does not use batch update.
            final int[] updateCounts = ((BatchUpdateException) e).getUpdateCounts();
            final List<FlowFile> batchFlowFiles = enclosure.getFlowFiles();
            // In the presence of a BatchUpdateException, the driver has the option of either stopping when an error
            // occurs, or continuing. If it continues, then it must account for all statements in the batch and for
            // those that fail return a Statement.EXECUTE_FAILED for the number of rows updated.
            // So we will iterate over all of the update counts returned. If any is equal to Statement.EXECUTE_FAILED,
            // we will route the corresponding FlowFile to failure. Otherwise, the FlowFile will go to success
            // unless it has not yet been processed (its index in the List > updateCounts.length).
            int failureCount = 0;
            int successCount = 0;
            int retryCount = 0;
            for (int i = 0; i < updateCounts.length; i++) {
                final int updateCount = updateCounts[i];
                final FlowFile flowFile = batchFlowFiles.get(i);
                if (updateCount == Statement.EXECUTE_FAILED) {
                    result.routeTo(flowFile, REL_FAILURE);
                    failureCount++;
                } else {
                    result.routeTo(flowFile, REL_SUCCESS);
                    successCount++;
                }
            }
            if (failureCount == 0) {
                // if no failures found, the driver decided not to execute the statements after the
                // failure, so route the last one to failure.
                final FlowFile failedFlowFile = batchFlowFiles.get(updateCounts.length);
                result.routeTo(failedFlowFile, REL_FAILURE);
                failureCount++;
            }
            if (updateCounts.length < batchFlowFiles.size()) {
                final List<FlowFile> unexecuted = batchFlowFiles.subList(updateCounts.length + 1, batchFlowFiles.size());
                for (final FlowFile flowFile : unexecuted) {
                    result.routeTo(flowFile, REL_RETRY);
                    retryCount++;
                }
            }
            getLogger().error("Failed to update database due to a failed batch update, {}. There were a total of {} FlowFiles that failed, {} that succeeded, " + "and {} that were not execute and will be routed to retry; ", new Object[] { e, failureCount, successCount, retryCount }, e);
            return;
        }
        // Apply default error handling and logging for other Exceptions.
        ExceptionHandler.OnError<RollbackOnFailure, FlowFileGroup> onGroupError = ExceptionHandler.createOnGroupError(context, session, result, REL_FAILURE, REL_RETRY);
        onGroupError = onGroupError.andThen((cl, il, rl, el) -> {
            switch(r.destination()) {
                case Failure:
                    getLogger().error("Failed to update database for {} due to {}; routing to failure", new Object[] { il.getFlowFiles(), e }, e);
                    break;
                case Retry:
                    getLogger().error("Failed to update database for {} due to {}; it is possible that retrying the operation will succeed, so routing to retry", new Object[] { il.getFlowFiles(), e }, e);
                    break;
            }
        });
        onGroupError.apply(c, enclosure, r, e);
    });
}
Also used : ExceptionHandler(org.apache.nifi.processor.util.pattern.ExceptionHandler) StandardValidators(org.apache.nifi.processor.util.StandardValidators) FetchFlowFiles(org.apache.nifi.processor.util.pattern.PartialFunctions.FetchFlowFiles) FlowFileFilter(org.apache.nifi.processor.FlowFileFilter) SQLNonTransientException(java.sql.SQLNonTransientException) Connection(java.sql.Connection) BiFunction(java.util.function.BiFunction) ExceptionHandler.createOnError(org.apache.nifi.processor.util.pattern.ExceptionHandler.createOnError) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ErrorTypes(org.apache.nifi.processor.util.pattern.ErrorTypes) RoutingResult(org.apache.nifi.processor.util.pattern.RoutingResult) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) ResultSet(java.sql.ResultSet) Map(java.util.Map) ReadsAttributes(org.apache.nifi.annotation.behavior.ReadsAttributes) ExceptionHandler(org.apache.nifi.processor.util.pattern.ExceptionHandler) InputStreamCallback(org.apache.nifi.processor.io.InputStreamCallback) PutGroup(org.apache.nifi.processor.util.pattern.PutGroup) FlowFile(org.apache.nifi.flowfile.FlowFile) FragmentAttributes(org.apache.nifi.flowfile.attributes.FragmentAttributes) Set(java.util.Set) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) PreparedStatement(java.sql.PreparedStatement) StandardCharsets(java.nio.charset.StandardCharsets) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) List(java.util.List) JdbcCommon(org.apache.nifi.processors.standard.util.JdbcCommon) Tags(org.apache.nifi.annotation.documentation.Tags) DBCPService(org.apache.nifi.dbcp.DBCPService) ReadsAttribute(org.apache.nifi.annotation.behavior.ReadsAttribute) RollbackOnFailure(org.apache.nifi.processor.util.pattern.RollbackOnFailure) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) BatchUpdateException(java.sql.BatchUpdateException) HashMap(java.util.HashMap) ProcessException(org.apache.nifi.processor.exception.ProcessException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) SQLException(java.sql.SQLException) Relationship(org.apache.nifi.processor.Relationship) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) AbstractSessionFactoryProcessor(org.apache.nifi.processor.AbstractSessionFactoryProcessor) PartialFunctions(org.apache.nifi.processor.util.pattern.PartialFunctions) FlowFileGroup(org.apache.nifi.processor.util.pattern.PartialFunctions.FlowFileGroup) ProcessContext(org.apache.nifi.processor.ProcessContext) ProcessSession(org.apache.nifi.processor.ProcessSession) IOException(java.io.IOException) SeeAlso(org.apache.nifi.annotation.documentation.SeeAlso) ProcessSessionFactory(org.apache.nifi.processor.ProcessSessionFactory) TimeUnit(java.util.concurrent.TimeUnit) OnScheduled(org.apache.nifi.annotation.lifecycle.OnScheduled) SupportsBatching(org.apache.nifi.annotation.behavior.SupportsBatching) StreamUtils(org.apache.nifi.stream.io.StreamUtils) Statement(java.sql.Statement) BitSet(java.util.BitSet) Comparator(java.util.Comparator) InputStream(java.io.InputStream) FlowFile(org.apache.nifi.flowfile.FlowFile) FlowFileGroup(org.apache.nifi.processor.util.pattern.PartialFunctions.FlowFileGroup) RollbackOnFailure(org.apache.nifi.processor.util.pattern.RollbackOnFailure) BatchUpdateException(java.sql.BatchUpdateException)

Aggregations

IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 StandardCharsets (java.nio.charset.StandardCharsets)1 BatchUpdateException (java.sql.BatchUpdateException)1 Connection (java.sql.Connection)1 PreparedStatement (java.sql.PreparedStatement)1 ResultSet (java.sql.ResultSet)1 SQLException (java.sql.SQLException)1 SQLNonTransientException (java.sql.SQLNonTransientException)1 Statement (java.sql.Statement)1 ArrayList (java.util.ArrayList)1 BitSet (java.util.BitSet)1 Comparator (java.util.Comparator)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 TimeUnit (java.util.concurrent.TimeUnit)1 BiFunction (java.util.function.BiFunction)1