Examples with ResourceClaim org.apache.nifi.controller.repository.claim.ResourceClaim used on opensource projects

Example 16 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FlowController method initializeFlow.

public void initializeFlow() throws IOException {
    writeLock.lock();
    try {
        // get all connections/queues and recover from swap files.
        final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
        long maxIdFromSwapFiles = -1L;
        if (flowFileRepository.isVolatile()) {
            for (final Connection connection : connections) {
                final FlowFileQueue queue = connection.getFlowFileQueue();
                queue.purgeSwapFiles();
            }
        } else {
            for (final Connection connection : connections) {
                final FlowFileQueue queue = connection.getFlowFileQueue();
                final SwapSummary swapSummary = queue.recoverSwappedFlowFiles();
                if (swapSummary != null) {
                    final Long maxFlowFileId = swapSummary.getMaxFlowFileId();
                    if (maxFlowFileId != null && maxFlowFileId > maxIdFromSwapFiles) {
                        maxIdFromSwapFiles = maxFlowFileId;
                    }
                    for (final ResourceClaim resourceClaim : swapSummary.getResourceClaims()) {
                        resourceClaimManager.incrementClaimantCount(resourceClaim);
                    }
                }
            }
        }
        flowFileRepository.loadFlowFiles(this, maxIdFromSwapFiles + 1);
        // Begin expiring FlowFiles that are old
        final RepositoryContextFactory contextFactory = new RepositoryContextFactory(contentRepository, flowFileRepository, flowFileEventRepository, counterRepositoryRef.get(), provenanceRepository);
        processScheduler.scheduleFrameworkTask(new ExpireFlowFiles(this, contextFactory), "Expire FlowFiles", 30L, 30L, TimeUnit.SECONDS);
        // now that we've loaded the FlowFiles, this has restored our ContentClaims' states, so we can tell the
        // ContentRepository to purge superfluous files
        contentRepository.cleanup();
        for (final RemoteSiteListener listener : externalSiteListeners) {
            listener.start();
        }
        notifyComponentsConfigurationRestored();
        timerDrivenEngineRef.get().scheduleWithFixedDelay(new Runnable() {

            @Override
            public void run() {
                try {
                    updateRemoteProcessGroups();
                } catch (final Throwable t) {
                    LOG.warn("Unable to update Remote Process Groups due to " + t);
                    if (LOG.isDebugEnabled()) {
                        LOG.warn("", t);
                    }
                }
            }
        }, 0L, 30L, TimeUnit.SECONDS);
        timerDrivenEngineRef.get().scheduleWithFixedDelay(new Runnable() {

            @Override
            public void run() {
                final ProcessGroup rootGroup = getRootGroup();
                final List<ProcessGroup> allGroups = rootGroup.findAllProcessGroups();
                allGroups.add(rootGroup);
                for (final ProcessGroup group : allGroups) {
                    try {
                        group.synchronizeWithFlowRegistry(flowRegistryClient);
                    } catch (final Exception e) {
                        LOG.error("Failed to synchronize {} with Flow Registry", group, e);
                    }
                }
            }
        }, 5, 60, TimeUnit.SECONDS);
        initialized.set(true);
    } finally {
        writeLock.unlock();
    }
}

Example 17 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FlowController method getReplayFailureReason.

private String getReplayFailureReason(final ProvenanceEventRecord event) {
    // Check that the event is a valid type.
    final ProvenanceEventType type = event.getEventType();
    if (type == ProvenanceEventType.JOIN) {
        return "Cannot replay events that are created from multiple parents";
    }
    // Make sure event has the Content Claim info
    final Long contentSize = event.getPreviousFileSize();
    final String contentClaimId = event.getPreviousContentClaimIdentifier();
    final String contentClaimSection = event.getPreviousContentClaimSection();
    final String contentClaimContainer = event.getPreviousContentClaimContainer();
    if (contentSize == null || contentClaimId == null || contentClaimSection == null || contentClaimContainer == null) {
        return "Cannot replay data from Provenance Event because the event does not contain the required Content Claim";
    }
    try {
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(contentClaimContainer, contentClaimSection, contentClaimId, false, false);
        final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, event.getPreviousContentClaimOffset());
        if (!contentRepository.isAccessible(contentClaim)) {
            return "Content is no longer available in Content Repository";
        }
    } catch (final IOException ioe) {
        return "Failed to determine whether or not content was available in Content Repository due to " + ioe.toString();
    }
    // Make sure that the source queue exists
    if (event.getSourceQueueIdentifier() == null) {
        return "Cannot replay data from Provenance Event because the event does not specify the Source FlowFile Queue";
    }
    final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
    FlowFileQueue queue = null;
    for (final Connection connection : connections) {
        if (event.getSourceQueueIdentifier().equals(connection.getIdentifier())) {
            queue = connection.getFlowFileQueue();
            break;
        }
    }
    if (queue == null) {
        return "Cannot replay data from Provenance Event because the Source FlowFile Queue with ID " + event.getSourceQueueIdentifier() + " no longer exists";
    }
    return null;
}

Example 18 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FlowController method replayFlowFile.

public ProvenanceEventRecord replayFlowFile(final ProvenanceEventRecord event, final NiFiUser user) throws IOException {
    if (event == null) {
        throw new NullPointerException();
    }
    // Check that the event is a valid type.
    final ProvenanceEventType type = event.getEventType();
    if (type == ProvenanceEventType.JOIN) {
        throw new IllegalArgumentException("Cannot replay events that are created from multiple parents");
    }
    // Make sure event has the Content Claim info
    final Long contentSize = event.getPreviousFileSize();
    final String contentClaimId = event.getPreviousContentClaimIdentifier();
    final String contentClaimSection = event.getPreviousContentClaimSection();
    final String contentClaimContainer = event.getPreviousContentClaimContainer();
    if (contentSize == null || contentClaimId == null || contentClaimSection == null || contentClaimContainer == null) {
        throw new IllegalArgumentException("Cannot replay data from Provenance Event because the event does not contain the required Content Claim");
    }
    // Make sure that the source queue exists
    if (event.getSourceQueueIdentifier() == null) {
        throw new IllegalArgumentException("Cannot replay data from Provenance Event because the event does not specify the Source FlowFile Queue");
    }
    final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
    FlowFileQueue queue = null;
    for (final Connection connection : connections) {
        if (event.getSourceQueueIdentifier().equals(connection.getIdentifier())) {
            queue = connection.getFlowFileQueue();
            break;
        }
    }
    if (queue == null) {
        throw new IllegalStateException("Cannot replay data from Provenance Event because the Source FlowFile Queue with ID " + event.getSourceQueueIdentifier() + " no longer exists");
    }
    // Create the ContentClaim. To do so, we first need the appropriate Resource Claim. Because we don't know whether or
    // not the Resource Claim is still active, we first call ResourceClaimManager.getResourceClaim. If this returns
    // null, then we know that the Resource Claim is no longer active and can just create a new one that is not writable.
    // It's critical though that we first call getResourceClaim because otherwise, if the Resource Claim is active and we
    // create a new one that is not writable, we could end up archiving or destroying the Resource Claim while it's still
    // being written to by the Content Repository. This is important only because we are creating a FlowFile with this Resource
    // Claim. If, for instance, we are simply creating the claim to request its content, as in #getContentAvailability, etc.
    // then this is not necessary.
    ResourceClaim resourceClaim = resourceClaimManager.getResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier());
    if (resourceClaim == null) {
        resourceClaim = resourceClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false, false);
    }
    // Increment Claimant Count, since we will now be referencing the Content Claim
    resourceClaimManager.incrementClaimantCount(resourceClaim);
    final long claimOffset = event.getPreviousContentClaimOffset() == null ? 0L : event.getPreviousContentClaimOffset().longValue();
    final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, claimOffset);
    contentClaim.setLength(event.getPreviousFileSize() == null ? -1L : event.getPreviousFileSize());
    if (!contentRepository.isAccessible(contentClaim)) {
        resourceClaimManager.decrementClaimantCount(resourceClaim);
        throw new IllegalStateException("Cannot replay data from Provenance Event because the data is no longer available in the Content Repository");
    }
    final String parentUUID = event.getFlowFileUuid();
    final String newFlowFileUUID = UUID.randomUUID().toString();
    // We need to create a new FlowFile by populating it with information from the
    // Provenance Event. Particularly of note here is that we are setting the FlowFile's
    // contentClaimOffset to 0. This is done for backward compatibility reasons. ContentClaim
    // used to not have a concept of an offset, and the offset was tied only to the FlowFile. This
    // was later refactored, so that the offset was part of the ContentClaim. If we set the offset
    // in both places, we'll end up skipping over that many bytes twice instead of once (once to get
    // to the beginning of the Content Claim and again to get to the offset within that Content Claim).
    // To avoid this, we just always set the offset in the Content Claim itself and set the
    // FlowFileRecord's contentClaimOffset to 0.
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttributes(event.getPreviousAttributes()).contentClaim(contentClaim).contentClaimOffset(// use 0 because we used the content claim offset in the Content Claim itself
    0L).entryDate(System.currentTimeMillis()).id(flowFileRepository.getNextFlowFileSequence()).lineageStart(event.getLineageStartDate(), 0L).size(contentSize.longValue()).addAttribute("flowfile.replay", "true").addAttribute("flowfile.replay.timestamp", String.valueOf(new Date())).addAttribute(CoreAttributes.UUID.key(), newFlowFileUUID).removeAttributes(CoreAttributes.DISCARD_REASON.key(), CoreAttributes.ALTERNATE_IDENTIFIER.key()).build();
    // Register a Provenance Event to indicate that we replayed the data.
    final ProvenanceEventRecord replayEvent = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.REPLAY).addChildUuid(newFlowFileUUID).addParentUuid(parentUUID).setFlowFileUUID(parentUUID).setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes()).setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize()).setDetails("Replay requested by " + user.getIdentity()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setLineageStartDate(event.getLineageStartDate()).setComponentType(event.getComponentType()).setComponentId(event.getComponentId()).build();
    provenanceRepository.registerEvent(replayEvent);
    // Update the FlowFile Repository to indicate that we have added the FlowFile to the flow
    final StandardRepositoryRecord record = new StandardRepositoryRecord(queue);
    record.setWorking(flowFileRecord);
    record.setDestination(queue);
    flowFileRepository.updateRepository(Collections.singleton(record));
    // Enqueue the data
    queue.put(flowFileRecord);
    return replayEvent;
}

Example 19 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FlowController method getContent.

public InputStream getContent(final FlowFileRecord flowFile, final String requestor, final String requestUri) throws IOException {
    requireNonNull(flowFile);
    requireNonNull(requestor);
    requireNonNull(requestUri);
    InputStream stream;
    final ResourceClaim resourceClaim;
    final ContentClaim contentClaim = flowFile.getContentClaim();
    if (contentClaim == null) {
        resourceClaim = null;
        stream = new ByteArrayInputStream(new byte[0]);
    } else {
        resourceClaim = flowFile.getContentClaim().getResourceClaim();
        stream = contentRepository.read(flowFile.getContentClaim());
        final long contentClaimOffset = flowFile.getContentClaimOffset();
        if (contentClaimOffset > 0L) {
            StreamUtils.skip(stream, contentClaimOffset);
        }
        stream = new LimitingInputStream(stream, flowFile.getSize());
    }
    // Register a Provenance Event to indicate that we replayed the data.
    final StandardProvenanceEventRecord.Builder sendEventBuilder = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.DOWNLOAD).setFlowFileUUID(flowFile.getAttribute(CoreAttributes.UUID.key())).setAttributes(flowFile.getAttributes(), Collections.emptyMap()).setTransitUri(requestUri).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(flowFile.getEntryDate()).setLineageStartDate(flowFile.getLineageStartDate()).setComponentType(getName()).setComponentId(getRootGroupId()).setDetails("Download of Content requested by " + requestor + " for " + flowFile);
    if (contentClaim != null) {
        sendEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
    }
    final ProvenanceEventRecord sendEvent = sendEventBuilder.build();
    provenanceRepository.registerEvent(sendEvent);
    return stream;
}

Example 20 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class StandardProcessSession method removeExpired.

private void removeExpired(final Set<FlowFileRecord> flowFiles, final Connection connection) {
    if (flowFiles.isEmpty()) {
        return;
    }
    LOG.info("{} {} FlowFiles have expired and will be removed", new Object[] { this, flowFiles.size() });
    final List<RepositoryRecord> expiredRecords = new ArrayList<>(flowFiles.size());
    final Connectable connectable = context.getConnectable();
    final String processorType = connectable.getComponentType();
    final StandardProvenanceReporter expiredReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(), processorType, context.getProvenanceRepository(), this);
    final Map<String, FlowFileRecord> recordIdMap = new HashMap<>();
    for (final FlowFileRecord flowFile : flowFiles) {
        recordIdMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile);
        final StandardRepositoryRecord record = new StandardRepositoryRecord(connection.getFlowFileQueue(), flowFile);
        record.markForDelete();
        expiredRecords.add(record);
        expiredReporter.expire(flowFile, "Expiration Threshold = " + connection.getFlowFileQueue().getFlowFileExpiration());
        decrementClaimCount(flowFile.getContentClaim());
        final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate();
        final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable;
        LOG.info("{} terminated by {} due to FlowFile expiration; life of FlowFile = {} ms", new Object[] { flowFile, terminator, flowFileLife });
    }
    try {
        final Iterable<ProvenanceEventRecord> iterable = new Iterable<ProvenanceEventRecord>() {

            @Override
            public Iterator<ProvenanceEventRecord> iterator() {
                final Iterator<ProvenanceEventRecord> expiredEventIterator = expiredReporter.getEvents().iterator();
                final Iterator<ProvenanceEventRecord> enrichingIterator = new Iterator<ProvenanceEventRecord>() {

                    @Override
                    public boolean hasNext() {
                        return expiredEventIterator.hasNext();
                    }

                    @Override
                    public ProvenanceEventRecord next() {
                        final ProvenanceEventRecord event = expiredEventIterator.next();
                        final StandardProvenanceEventRecord.Builder enriched = new StandardProvenanceEventRecord.Builder().fromEvent(event);
                        final FlowFileRecord record = recordIdMap.get(event.getFlowFileUuid());
                        if (record == null) {
                            return null;
                        }
                        final ContentClaim claim = record.getContentClaim();
                        if (claim != null) {
                            final ResourceClaim resourceClaim = claim.getResourceClaim();
                            enriched.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), record.getContentClaimOffset() + claim.getOffset(), record.getSize());
                            enriched.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), record.getContentClaimOffset() + claim.getOffset(), record.getSize());
                        }
                        enriched.setAttributes(record.getAttributes(), Collections.<String, String>emptyMap());
                        return enriched.build();
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException();
                    }
                };
                return enrichingIterator;
            }
        };
        context.getProvenanceRepository().registerEvents(iterable);
        context.getFlowFileRepository().updateRepository(expiredRecords);
    } catch (final IOException e) {
        LOG.error("Failed to update FlowFile Repository to record expired records due to {}", e);
    }
}