Examples with ContentClaim org.apache.nifi.controller.repository.claim.ContentClaim used on opensource projects

Example 61 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class StandardProcessSession method write.

@Override
public FlowFile write(FlowFile source, final StreamCallback writer) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    final ContentClaim currClaim = record.getCurrentClaim();
    long writtenToFlowFile = 0L;
    ContentClaim newClaim = null;
    try {
        newClaim = claimCache.getContentClaim();
        claimLog.debug("Creating ContentClaim {} for 'write' for {}", newClaim, source);
        ensureNotAppending(newClaim);
        if (currClaim != null) {
            claimCache.flush(currClaim.getResourceClaim());
        }
        try (final InputStream is = getInputStream(source, currClaim, record.getCurrentClaimOffset(), true);
            final InputStream limitedIn = new LimitedInputStream(is, source.getSize());
            final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
            final ByteCountingInputStream countingIn = new ByteCountingInputStream(disableOnCloseIn, bytesRead);
            final OutputStream os = claimCache.write(newClaim);
            final OutputStream disableOnCloseOut = new DisableOnCloseOutputStream(os);
            final ByteCountingOutputStream countingOut = new ByteCountingOutputStream(disableOnCloseOut)) {
            writeRecursionSet.add(source);
            // We want to differentiate between IOExceptions thrown by the repository and IOExceptions thrown from
            // Processor code. As a result, as have the FlowFileAccessInputStream that catches IOException from the repository
            // and translates into either FlowFileAccessException or ContentNotFoundException. We keep track of any
            // ContentNotFoundException because if it is thrown, the Processor code may catch it and do something else with it
            // but in reality, if it is thrown, we want to know about it and handle it, even if the Processor code catches it.
            final FlowFileAccessInputStream ffais = new FlowFileAccessInputStream(countingIn, source, currClaim);
            final FlowFileAccessOutputStream ffaos = new FlowFileAccessOutputStream(countingOut, source);
            boolean cnfeThrown = false;
            try {
                writer.process(createTaskTerminationStream(ffais), createTaskTerminationStream(ffaos));
            } catch (final ContentNotFoundException cnfe) {
                cnfeThrown = true;
                throw cnfe;
            } finally {
                writtenToFlowFile = countingOut.getBytesWritten();
                this.bytesWritten += writtenToFlowFile;
                this.bytesRead += countingIn.getBytesRead();
                writeRecursionSet.remove(source);
                // if cnfeThrown is true, we don't need to re-thrown the Exception; it will propagate.
                if (!cnfeThrown && ffais.getContentNotFoundException() != null) {
                    throw ffais.getContentNotFoundException();
                }
            }
        }
    } catch (final ContentNotFoundException nfe) {
        destroyContent(newClaim);
        handleContentNotFound(nfe, record);
    } catch (final IOException ioe) {
        destroyContent(newClaim);
        throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
    } catch (final FlowFileAccessException ffae) {
        destroyContent(newClaim);
        throw ffae;
    } catch (final Throwable t) {
        destroyContent(newClaim);
        throw t;
    }
    removeTemporaryClaim(record);
    final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(Math.max(0L, newClaim.getLength() - writtenToFlowFile)).size(writtenToFlowFile).build();
    record.setWorking(newFile);
    return newFile;
}

Example 62 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class StandardProcessSession method append.

@Override
public FlowFile append(FlowFile source, final OutputStreamCallback writer) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    long newSize = 0L;
    // Get the current Content Claim from the record and see if we already have
    // an OutputStream that we can append to.
    final ContentClaim oldClaim = record.getCurrentClaim();
    ByteCountingOutputStream outStream = oldClaim == null ? null : appendableStreams.get(oldClaim);
    long originalByteWrittenCount = 0;
    ContentClaim newClaim = null;
    try {
        if (outStream == null) {
            claimCache.flush(oldClaim);
            try (final InputStream oldClaimIn = context.getContentRepository().read(oldClaim)) {
                newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant());
                claimLog.debug("Creating ContentClaim {} for 'append' for {}", newClaim, source);
                final OutputStream rawOutStream = context.getContentRepository().write(newClaim);
                final OutputStream bufferedOutStream = new BufferedOutputStream(rawOutStream);
                outStream = new ByteCountingOutputStream(bufferedOutStream);
                originalByteWrittenCount = 0;
                appendableStreams.put(newClaim, outStream);
                // We need to copy all of the data from the old claim to the new claim
                StreamUtils.copy(oldClaimIn, outStream);
                // wrap our OutputStreams so that the processor cannot close it
                try (final OutputStream disableOnClose = new DisableOnCloseOutputStream(outStream)) {
                    writeRecursionSet.add(source);
                    writer.process(new FlowFileAccessOutputStream(disableOnClose, source));
                } finally {
                    writeRecursionSet.remove(source);
                }
            }
        } else {
            newClaim = oldClaim;
            originalByteWrittenCount = outStream.getBytesWritten();
            // wrap our OutputStreams so that the processor cannot close it
            try (final OutputStream disableOnClose = new DisableOnCloseOutputStream(outStream);
                final OutputStream flowFileAccessOutStream = new FlowFileAccessOutputStream(disableOnClose, source)) {
                writeRecursionSet.add(source);
                writer.process(flowFileAccessOutStream);
            } finally {
                writeRecursionSet.remove(source);
            }
        }
        // update the newSize to reflect the number of bytes written
        newSize = outStream.getBytesWritten();
    } catch (final ContentNotFoundException nfe) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        // it here also, we would be decrementing the claimant count twice!
        if (newClaim != oldClaim) {
            destroyContent(newClaim);
        }
        handleContentNotFound(nfe, record);
    } catch (final IOException ioe) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        // See above explanation for why this is done only if newClaim != oldClaim
        if (newClaim != oldClaim) {
            destroyContent(newClaim);
        }
        throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
    } catch (final Throwable t) {
        // need to reset write claim before we can remove the claim
        resetWriteClaims();
        // See above explanation for why this is done only if newClaim != oldClaim
        if (newClaim != oldClaim) {
            destroyContent(newClaim);
        }
        throw t;
    } finally {
        if (outStream != null) {
            final long bytesWrittenThisIteration = outStream.getBytesWritten() - originalByteWrittenCount;
            bytesWritten += bytesWrittenThisIteration;
        }
    }
    // the FlowFile was written to, via #write() and then append() was called.
    if (newClaim != oldClaim) {
        removeTemporaryClaim(record);
    }
    final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(0).size(newSize).build();
    record.setWorking(newFile);
    return newFile;
}

Example 63 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class StandardProcessSession method removeExpired.

private void removeExpired(final Set<FlowFileRecord> flowFiles, final Connection connection) {
    if (flowFiles.isEmpty()) {
        return;
    }
    LOG.info("{} {} FlowFiles have expired and will be removed", new Object[] { this, flowFiles.size() });
    final List<RepositoryRecord> expiredRecords = new ArrayList<>(flowFiles.size());
    final Connectable connectable = context.getConnectable();
    final String processorType = connectable.getComponentType();
    final StandardProvenanceReporter expiredReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(), processorType, context.getProvenanceRepository(), this);
    final Map<String, FlowFileRecord> recordIdMap = new HashMap<>();
    for (final FlowFileRecord flowFile : flowFiles) {
        recordIdMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile);
        final StandardRepositoryRecord record = new StandardRepositoryRecord(connection.getFlowFileQueue(), flowFile);
        record.markForDelete();
        expiredRecords.add(record);
        expiredReporter.expire(flowFile, "Expiration Threshold = " + connection.getFlowFileQueue().getFlowFileExpiration());
        decrementClaimCount(flowFile.getContentClaim());
        final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate();
        final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable;
        LOG.info("{} terminated by {} due to FlowFile expiration; life of FlowFile = {} ms", new Object[] { flowFile, terminator, flowFileLife });
    }
    try {
        final Iterable<ProvenanceEventRecord> iterable = new Iterable<ProvenanceEventRecord>() {

            @Override
            public Iterator<ProvenanceEventRecord> iterator() {
                final Iterator<ProvenanceEventRecord> expiredEventIterator = expiredReporter.getEvents().iterator();
                final Iterator<ProvenanceEventRecord> enrichingIterator = new Iterator<ProvenanceEventRecord>() {

                    @Override
                    public boolean hasNext() {
                        return expiredEventIterator.hasNext();
                    }

                    @Override
                    public ProvenanceEventRecord next() {
                        final ProvenanceEventRecord event = expiredEventIterator.next();
                        final StandardProvenanceEventRecord.Builder enriched = new StandardProvenanceEventRecord.Builder().fromEvent(event);
                        final FlowFileRecord record = recordIdMap.get(event.getFlowFileUuid());
                        if (record == null) {
                            return null;
                        }
                        final ContentClaim claim = record.getContentClaim();
                        if (claim != null) {
                            final ResourceClaim resourceClaim = claim.getResourceClaim();
                            enriched.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), record.getContentClaimOffset() + claim.getOffset(), record.getSize());
                            enriched.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), record.getContentClaimOffset() + claim.getOffset(), record.getSize());
                        }
                        enriched.setAttributes(record.getAttributes(), Collections.<String, String>emptyMap());
                        return enriched.build();
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException();
                    }
                };
                return enrichingIterator;
            }
        };
        context.getProvenanceRepository().registerEvents(iterable);
        context.getFlowFileRepository().updateRepository(expiredRecords);
    } catch (final IOException e) {
        LOG.error("Failed to update FlowFile Repository to record expired records due to {}", e);
    }
}

Example 64 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class StandardProcessSession method importFrom.

@Override
public FlowFile importFrom(final InputStream source, FlowFile destination) {
    verifyTaskActive();
    destination = validateRecordState(destination);
    final StandardRepositoryRecord record = records.get(destination);
    ContentClaim newClaim = null;
    final long claimOffset = 0L;
    final long newSize;
    try {
        try {
            newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant());
            claimLog.debug("Creating ContentClaim {} for 'importFrom' for {}", newClaim, destination);
            newSize = context.getContentRepository().importFrom(createTaskTerminationStream(source), newClaim);
            bytesWritten += newSize;
        } catch (final IOException e) {
            throw new FlowFileAccessException("Unable to create ContentClaim due to " + e.toString(), e);
        }
    } catch (final Throwable t) {
        if (newClaim != null) {
            destroyContent(newClaim);
        }
        throw new FlowFileAccessException("Failed to import data from " + source + " for " + destination + " due to " + t.toString(), t);
    }
    removeTemporaryClaim(record);
    final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(claimOffset).size(newSize).build();
    record.setWorking(newFile);
    return newFile;
}

Example 65 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class StandardProcessSession method enrich.

private StandardProvenanceEventRecord enrich(final ProvenanceEventRecord rawEvent, final Map<String, FlowFileRecord> flowFileRecordMap, final Map<FlowFileRecord, StandardRepositoryRecord> records, final boolean updateAttributes) {
    final StandardProvenanceEventRecord.Builder recordBuilder = new StandardProvenanceEventRecord.Builder().fromEvent(rawEvent);
    final FlowFileRecord eventFlowFile = flowFileRecordMap.get(rawEvent.getFlowFileUuid());
    if (eventFlowFile != null) {
        final StandardRepositoryRecord repoRecord = records.get(eventFlowFile);
        if (repoRecord.getCurrent() != null && repoRecord.getCurrentClaim() != null) {
            final ContentClaim currentClaim = repoRecord.getCurrentClaim();
            final long currentOffset = repoRecord.getCurrentClaimOffset();
            final long size = eventFlowFile.getSize();
            final ResourceClaim resourceClaim = currentClaim.getResourceClaim();
            recordBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), currentOffset + currentClaim.getOffset(), size);
        }
        if (repoRecord.getOriginal() != null && repoRecord.getOriginalClaim() != null) {
            final ContentClaim originalClaim = repoRecord.getOriginalClaim();
            final long originalOffset = repoRecord.getOriginal().getContentClaimOffset();
            final long originalSize = repoRecord.getOriginal().getSize();
            final ResourceClaim resourceClaim = originalClaim.getResourceClaim();
            recordBuilder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), originalOffset + originalClaim.getOffset(), originalSize);
        }
        final FlowFileQueue originalQueue = repoRecord.getOriginalQueue();
        if (originalQueue != null) {
            recordBuilder.setSourceQueueIdentifier(originalQueue.getIdentifier());
        }
    }
    if (updateAttributes) {
        final FlowFileRecord flowFileRecord = flowFileRecordMap.get(rawEvent.getFlowFileUuid());
        if (flowFileRecord != null) {
            final StandardRepositoryRecord record = records.get(flowFileRecord);
            if (record != null) {
                recordBuilder.setAttributes(record.getOriginalAttributes(), record.getUpdatedAttributes());
            }
        }
    }
    return recordBuilder.build();
}