Examples with FlowFileAccessException org.apache.nifi.processor.exception.FlowFileAccessException used on opensource projects

Example 6 with FlowFileAccessException

use of org.apache.nifi.processor.exception.FlowFileAccessException in project nifi by apache.

the class StandardProcessSession method read.

@Override
public void read(FlowFile source, boolean allowSessionStreamManagement, InputStreamCallback reader) {
    verifyTaskActive();
    source = validateRecordState(source, true);
    final StandardRepositoryRecord record = records.get(source);
    try {
        ensureNotAppending(record.getCurrentClaim());
        claimCache.flush(record.getCurrentClaim());
    } catch (final IOException e) {
        throw new FlowFileAccessException("Failed to access ContentClaim for " + source.toString(), e);
    }
    try (final InputStream rawIn = getInputStream(source, record.getCurrentClaim(), record.getCurrentClaimOffset(), true);
        final InputStream limitedIn = new LimitedInputStream(rawIn, source.getSize());
        final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
        final ByteCountingInputStream countingStream = new ByteCountingInputStream(disableOnCloseIn, this.bytesRead)) {
        // We want to differentiate between IOExceptions thrown by the repository and IOExceptions thrown from
        // Processor code. As a result, as have the FlowFileAccessInputStream that catches IOException from the repository
        // and translates into either FlowFileAccessException or ContentNotFoundException. We keep track of any
        // ContentNotFoundException because if it is thrown, the Processor code may catch it and do something else with it
        // but in reality, if it is thrown, we want to know about it and handle it, even if the Processor code catches it.
        final FlowFileAccessInputStream ffais = new FlowFileAccessInputStream(countingStream, source, record.getCurrentClaim());
        boolean cnfeThrown = false;
        try {
            incrementReadCount(source);
            reader.process(createTaskTerminationStream(ffais));
            // Allow processors to close the file after reading to avoid too many files open or do smart session stream management.
            if (this.currentReadClaimStream != null && !allowSessionStreamManagement) {
                currentReadClaimStream.close();
                currentReadClaimStream = null;
            }
        } catch (final ContentNotFoundException cnfe) {
            cnfeThrown = true;
            throw cnfe;
        } finally {
            decrementReadCount(source);
            bytesRead += countingStream.getBytesRead();
            // if cnfeThrown is true, we don't need to re-thrown the Exception; it will propagate.
            if (!cnfeThrown && ffais.getContentNotFoundException() != null) {
                throw ffais.getContentNotFoundException();
            }
        }
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final IOException ex) {
        throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ex.toString(), ex);
    }
}

Example 7 with FlowFileAccessException

use of org.apache.nifi.processor.exception.FlowFileAccessException in project nifi by apache.

the class StandardProcessSession method write.

@Override
public OutputStream write(FlowFile source) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    ContentClaim newClaim = null;
    try {
        newClaim = claimCache.getContentClaim();
        claimLog.debug("Creating ContentClaim {} for 'write' for {}", newClaim, source);
        ensureNotAppending(newClaim);
        final OutputStream rawStream = claimCache.write(newClaim);
        final OutputStream disableOnClose = new DisableOnCloseOutputStream(rawStream);
        final ByteCountingOutputStream countingOut = new ByteCountingOutputStream(disableOnClose);
        final FlowFile sourceFlowFile = source;
        final ContentClaim updatedClaim = newClaim;
        final OutputStream errorHandlingOutputStream = new OutputStream() {

            private boolean closed = false;

            @Override
            public void write(final int b) throws IOException {
                try {
                    countingOut.write(b);
                } catch (final IOException ioe) {
                    LOG.error("Failed to write content to " + sourceFlowFile + "; rolling back session", ioe);
                    rollback(true);
                    close();
                    throw new FlowFileAccessException("Failed to write to Content Repository for " + sourceFlowFile, ioe);
                }
            }

            @Override
            public void write(final byte[] b) throws IOException {
                try {
                    countingOut.write(b);
                } catch (final IOException ioe) {
                    LOG.error("Failed to write content to " + sourceFlowFile + "; rolling back session", ioe);
                    rollback(true);
                    close();
                    throw new FlowFileAccessException("Failed to write to Content Repository for " + sourceFlowFile, ioe);
                }
            }

            @Override
            public void write(final byte[] b, final int off, final int len) throws IOException {
                try {
                    countingOut.write(b, off, len);
                } catch (final IOException ioe) {
                    LOG.error("Failed to write content to " + sourceFlowFile + "; rolling back session", ioe);
                    rollback(true);
                    close();
                    throw new FlowFileAccessException("Failed to write to Content Repository for " + sourceFlowFile, ioe);
                }
            }

            @Override
            public void flush() throws IOException {
                try {
                    countingOut.flush();
                } catch (final IOException ioe) {
                    LOG.error("Failed to write content to " + sourceFlowFile + "; rolling back session", ioe);
                    rollback(true);
                    close();
                    throw new FlowFileAccessException("Failed to write to Content Repository for " + sourceFlowFile, ioe);
                }
            }

            @Override
            public void close() throws IOException {
                if (closed) {
                    return;
                }
                closed = true;
                writeRecursionSet.remove(sourceFlowFile);
                final long bytesWritten = countingOut.getBytesWritten();
                StandardProcessSession.this.bytesWritten += bytesWritten;
                final OutputStream removed = openOutputStreams.remove(sourceFlowFile);
                if (removed == null) {
                    LOG.error("Closed Session's OutputStream but there was no entry for it in the map; sourceFlowFile={}; map={}", sourceFlowFile, openOutputStreams);
                }
                flush();
                removeTemporaryClaim(record);
                final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(updatedClaim).contentClaimOffset(Math.max(0, updatedClaim.getLength() - bytesWritten)).size(bytesWritten).build();
                record.setWorking(newFile);
            }
        };
        writeRecursionSet.add(source);
        openOutputStreams.put(source, errorHandlingOutputStream);
        return createTaskTerminationStream(errorHandlingOutputStream);
    } catch (final ContentNotFoundException nfe) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        destroyContent(newClaim);
        handleContentNotFound(nfe, record);
        throw nfe;
    } catch (final FlowFileAccessException ffae) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        destroyContent(newClaim);
        throw ffae;
    } catch (final IOException ioe) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        destroyContent(newClaim);
        throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
    } catch (final Throwable t) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        destroyContent(newClaim);
        throw t;
    }
}

Example 8 with FlowFileAccessException

use of org.apache.nifi.processor.exception.FlowFileAccessException in project nifi by apache.

the class StandardProcessSession method commit.

@SuppressWarnings({ "unchecked", "rawtypes" })
private void commit(final Checkpoint checkpoint) {
    try {
        final long commitStartNanos = System.nanoTime();
        resetReadClaim();
        try {
            claimCache.flush();
        } finally {
            claimCache.reset();
        }
        final long updateProvenanceStart = System.nanoTime();
        updateProvenanceRepo(checkpoint);
        final long claimRemovalStart = System.nanoTime();
        final long updateProvenanceNanos = claimRemovalStart - updateProvenanceStart;
        /**
         * Figure out which content claims can be released. At this point,
         * we will decrement the Claimant Count for the claims via the
         * Content Repository. We do not actually destroy the content
         * because otherwise, we could remove the Original Claim and
         * crash/restart before the FlowFileRepository is updated. This will
         * result in the FlowFile being restored such that the content claim
         * points to the Original Claim -- which has already been removed!
         */
        for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : checkpoint.records.entrySet()) {
            final FlowFile flowFile = entry.getKey();
            final StandardRepositoryRecord record = entry.getValue();
            if (record.isMarkedForDelete()) {
                // if the working claim is not the same as the original claim, we can immediately destroy the working claim
                // because it was created in this session and is to be deleted. We don't need to wait for the FlowFile Repo to sync.
                decrementClaimCount(record.getWorkingClaim());
                if (record.getOriginalClaim() != null && !record.getOriginalClaim().equals(record.getWorkingClaim())) {
                    // if working & original claim are same, don't remove twice; we only want to remove the original
                    // if it's different from the working. Otherwise, we remove two claimant counts. This causes
                    // an issue if we only updated the FlowFile attributes.
                    decrementClaimCount(record.getOriginalClaim());
                }
                final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate();
                final Connectable connectable = context.getConnectable();
                final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable;
                LOG.info("{} terminated by {}; life of FlowFile = {} ms", new Object[] { flowFile, terminator, flowFileLife });
            } else if (record.isWorking() && record.getWorkingClaim() != record.getOriginalClaim()) {
                // records which have been updated - remove original if exists
                decrementClaimCount(record.getOriginalClaim());
            }
        }
        final long claimRemovalFinishNanos = System.nanoTime();
        final long claimRemovalNanos = claimRemovalFinishNanos - claimRemovalStart;
        // Update the FlowFile Repository
        try {
            final Collection<StandardRepositoryRecord> repoRecords = checkpoint.records.values();
            context.getFlowFileRepository().updateRepository((Collection) repoRecords);
        } catch (final IOException ioe) {
            // if we fail to commit the session, we need to roll back
            // the checkpoints as well because none of the checkpoints
            // were ever committed.
            rollback(false, true);
            throw new ProcessException("FlowFile Repository failed to update", ioe);
        }
        final long flowFileRepoUpdateFinishNanos = System.nanoTime();
        final long flowFileRepoUpdateNanos = flowFileRepoUpdateFinishNanos - claimRemovalFinishNanos;
        updateEventRepository(checkpoint);
        final long updateEventRepositoryFinishNanos = System.nanoTime();
        final long updateEventRepositoryNanos = updateEventRepositoryFinishNanos - flowFileRepoUpdateFinishNanos;
        // transfer the flowfiles to the connections' queues.
        final Map<FlowFileQueue, Collection<FlowFileRecord>> recordMap = new HashMap<>();
        for (final StandardRepositoryRecord record : checkpoint.records.values()) {
            if (record.isMarkedForAbort() || record.isMarkedForDelete()) {
                // these don't need to be transferred
                continue;
            }
            // in this case, we just ignore it, and it will be cleaned up by clearing the records map.
            if (record.getCurrent() != null) {
                Collection<FlowFileRecord> collection = recordMap.get(record.getDestination());
                if (collection == null) {
                    collection = new ArrayList<>();
                    recordMap.put(record.getDestination(), collection);
                }
                collection.add(record.getCurrent());
            }
        }
        for (final Map.Entry<FlowFileQueue, Collection<FlowFileRecord>> entry : recordMap.entrySet()) {
            entry.getKey().putAll(entry.getValue());
        }
        final long enqueueFlowFileFinishNanos = System.nanoTime();
        final long enqueueFlowFileNanos = enqueueFlowFileFinishNanos - updateEventRepositoryFinishNanos;
        // Delete any files from disk that need to be removed.
        for (final Path path : checkpoint.deleteOnCommit.values()) {
            try {
                Files.deleteIfExists(path);
            } catch (final IOException e) {
                throw new FlowFileAccessException("Unable to delete " + path.toFile().getAbsolutePath(), e);
            }
        }
        checkpoint.deleteOnCommit.clear();
        if (LOG.isInfoEnabled()) {
            final String sessionSummary = summarizeEvents(checkpoint);
            if (!sessionSummary.isEmpty()) {
                LOG.info("{} for {}, committed the following events: {}", new Object[] { this, connectableDescription, sessionSummary });
            }
        }
        for (final Map.Entry<String, Long> entry : checkpoint.countersOnCommit.entrySet()) {
            context.adjustCounter(entry.getKey(), entry.getValue());
        }
        acknowledgeRecords();
        resetState();
        if (LOG.isDebugEnabled()) {
            final StringBuilder timingInfo = new StringBuilder();
            timingInfo.append("Session commit for ").append(this).append(" [").append(connectableDescription).append("]").append(" took ");
            final long commitNanos = System.nanoTime() - commitStartNanos;
            formatNanos(commitNanos, timingInfo);
            timingInfo.append("; FlowFile Repository Update took ");
            formatNanos(flowFileRepoUpdateNanos, timingInfo);
            timingInfo.append("; Claim Removal took ");
            formatNanos(claimRemovalNanos, timingInfo);
            timingInfo.append("; FlowFile Event Update took ");
            formatNanos(updateEventRepositoryNanos, timingInfo);
            timingInfo.append("; Enqueuing FlowFiles took ");
            formatNanos(enqueueFlowFileNanos, timingInfo);
            timingInfo.append("; Updating Provenance Event Repository took ");
            formatNanos(updateProvenanceNanos, timingInfo);
            LOG.debug(timingInfo.toString());
        }
    } catch (final Exception e) {
        try {
            // if we fail to commit the session, we need to roll back
            // the checkpoints as well because none of the checkpoints
            // were ever committed.
            rollback(false, true);
        } catch (final Exception e1) {
            e.addSuppressed(e1);
        }
        if (e instanceof RuntimeException) {
            throw (RuntimeException) e;
        } else {
            throw new ProcessException(e);
        }
    }
}

Example 9 with FlowFileAccessException

use of org.apache.nifi.processor.exception.FlowFileAccessException in project nifi by apache.

the class StandardProcessSession method getInputStream.

private InputStream getInputStream(final FlowFile flowFile, final ContentClaim claim, final long offset, final boolean allowCachingOfStream) throws ContentNotFoundException {
    // that there is no actual content.
    if (flowFile.getSize() == 0L) {
        return new ByteArrayInputStream(new byte[0]);
    }
    try {
        // callback for reading FlowFile 1 and if we used the same stream we'd be destroying the ability to read from FlowFile 1.
        if (allowCachingOfStream && readRecursionSet.isEmpty() && writeRecursionSet.isEmpty()) {
            if (currentReadClaim == claim) {
                if (currentReadClaimStream != null && currentReadClaimStream.getBytesConsumed() <= offset) {
                    final long bytesToSkip = offset - currentReadClaimStream.getBytesConsumed();
                    if (bytesToSkip > 0) {
                        StreamUtils.skip(currentReadClaimStream, bytesToSkip);
                    }
                    return new DisableOnCloseInputStream(currentReadClaimStream);
                }
            }
            claimCache.flush(claim);
            final InputStream rawInStream = context.getContentRepository().read(claim);
            if (currentReadClaimStream != null) {
                currentReadClaimStream.close();
            }
            currentReadClaim = claim;
            currentReadClaimStream = new ByteCountingInputStream(rawInStream);
            StreamUtils.skip(currentReadClaimStream, offset);
            // reuse the same InputStream for the next FlowFile
            return new DisableOnCloseInputStream(currentReadClaimStream);
        } else {
            claimCache.flush(claim);
            final InputStream rawInStream = context.getContentRepository().read(claim);
            try {
                StreamUtils.skip(rawInStream, offset);
            } catch (IOException ioe) {
                try {
                    rawInStream.close();
                } catch (final Exception e) {
                    ioe.addSuppressed(ioe);
                }
                throw ioe;
            }
            return rawInStream;
        }
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final EOFException eof) {
        throw new ContentNotFoundException(claim, eof);
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Failed to read content of " + flowFile, ioe);
    }
}

Example 10 with FlowFileAccessException

use of org.apache.nifi.processor.exception.FlowFileAccessException in project nifi by apache.

the class StandardProcessSession method exportTo.

@Override
public void exportTo(FlowFile source, final Path destination, final boolean append) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    try {
        ensureNotAppending(record.getCurrentClaim());
        claimCache.flush(record.getCurrentClaim());
        final long copyCount = context.getContentRepository().exportTo(record.getCurrentClaim(), destination, append, record.getCurrentClaimOffset(), source.getSize());
        bytesRead += copyCount;
        bytesWritten += copyCount;
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final Throwable t) {
        throw new FlowFileAccessException("Failed to export " + source + " to " + destination + " due to " + t.toString(), t);
    }
}