Examples with FlowFileRecord org.apache.nifi.controller.repository.FlowFileRecord used on opensource projects

Example 31 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method drop.

private QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException {
    // Create a Provenance Event and a FlowFile Repository record for each FlowFile
    final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
    final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
    for (final FlowFileRecord flowFile : flowFiles) {
        provenanceEvents.add(createDropEvent(flowFile, requestor));
        flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
    }
    long dropContentSize = 0L;
    for (final FlowFileRecord flowFile : flowFiles) {
        dropContentSize += flowFile.getSize();
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim == null) {
            continue;
        }
        final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
        if (resourceClaim == null) {
            continue;
        }
        resourceClaimManager.decrementClaimantCount(resourceClaim);
    }
    provRepository.registerEvents(provenanceEvents);
    flowFileRepository.updateRepository(flowFileRepoRecords);
    return new QueueSize(flowFiles.size(), dropContentSize);
}

Example 32 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method doPoll.

private void doPoll(final List<FlowFileRecord> records, int maxResults, final Set<FlowFileRecord> expiredRecords) {
    migrateSwapToActive();
    final long bytesDrained = drainQueue(activeQueue, records, maxResults, expiredRecords);
    long expiredBytes = 0L;
    for (final FlowFileRecord record : expiredRecords) {
        expiredBytes += record.getSize();
    }
    incrementActiveQueueSize(-(expiredRecords.size() + records.size()), -bytesDrained);
    incrementUnacknowledgedQueueSize(records.size(), bytesDrained - expiredBytes);
}

Example 33 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method writeSwapFilesIfNecessary.

/**
 * This method MUST be called with the write lock held
 */
private void writeSwapFilesIfNecessary() {
    if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
        return;
    }
    migrateSwapToActive();
    final int numSwapFiles = swapQueue.size() / SWAP_RECORD_POLL_SIZE;
    int originalSwapQueueCount = swapQueue.size();
    long originalSwapQueueBytes = 0L;
    for (final FlowFileRecord flowFile : swapQueue) {
        originalSwapQueueBytes += flowFile.getSize();
    }
    // Create a new Priority queue with the prioritizers that are set, but reverse the
    // prioritizers because we want to pull the lowest-priority FlowFiles to swap out
    final PriorityQueue<FlowFileRecord> tempQueue = new PriorityQueue<>(activeQueue.size() + swapQueue.size(), Collections.reverseOrder(new Prioritizer(priorities)));
    tempQueue.addAll(activeQueue);
    tempQueue.addAll(swapQueue);
    long bytesSwappedOut = 0L;
    int flowFilesSwappedOut = 0;
    final List<String> swapLocations = new ArrayList<>(numSwapFiles);
    for (int i = 0; i < numSwapFiles; i++) {
        // Create a new swap file for the next SWAP_RECORD_POLL_SIZE records
        final List<FlowFileRecord> toSwap = new ArrayList<>(SWAP_RECORD_POLL_SIZE);
        for (int j = 0; j < SWAP_RECORD_POLL_SIZE; j++) {
            final FlowFileRecord flowFile = tempQueue.poll();
            toSwap.add(flowFile);
            bytesSwappedOut += flowFile.getSize();
            flowFilesSwappedOut++;
        }
        try {
            // currently ordered in reverse priority order based on the ordering of the temp queue.
            Collections.reverse(toSwap);
            final String swapLocation = swapManager.swapOut(toSwap, this);
            swapLocations.add(swapLocation);
        } catch (final IOException ioe) {
            // if we failed, we must add the FlowFiles back to the queue.
            tempQueue.addAll(toSwap);
            logger.error("FlowFile Queue with identifier {} has {} FlowFiles queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting " + "the Java heap space but failed to write information to disk due to {}", getIdentifier(), getQueueSize().getObjectCount(), ioe.toString());
            logger.error("", ioe);
            if (eventReporter != null) {
                eventReporter.reportEvent(Severity.ERROR, "Failed to Overflow to Disk", "Flowfile Queue with identifier " + getIdentifier() + " has " + getQueueSize().getObjectCount() + " queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting the Java heap space but failed to write information to disk. " + "See logs for more information.");
            }
            break;
        }
    }
    // Pull any records off of the temp queue that won't fit back on the active queue, and add those to the
    // swap queue. Then add the records back to the active queue.
    swapQueue.clear();
    long updatedSwapQueueBytes = 0L;
    while (tempQueue.size() > swapThreshold) {
        final FlowFileRecord record = tempQueue.poll();
        swapQueue.add(record);
        updatedSwapQueueBytes += record.getSize();
    }
    // currently ordered in reverse priority order based on the ordering of the temp queue
    Collections.reverse(swapQueue);
    // replace the contents of the active queue, since we've merged it with the swap queue.
    activeQueue.clear();
    FlowFileRecord toRequeue;
    long activeQueueBytes = 0L;
    while ((toRequeue = tempQueue.poll()) != null) {
        activeQueue.offer(toRequeue);
        activeQueueBytes += toRequeue.getSize();
    }
    boolean updated = false;
    while (!updated) {
        final FlowFileQueueSize originalSize = size.get();
        final int addedSwapRecords = swapQueue.size() - originalSwapQueueCount;
        final long addedSwapBytes = updatedSwapQueueBytes - originalSwapQueueBytes;
        final FlowFileQueueSize newSize = new FlowFileQueueSize(activeQueue.size(), activeQueueBytes, originalSize.swappedCount + addedSwapRecords + flowFilesSwappedOut, originalSize.swappedBytes + addedSwapBytes + bytesSwappedOut, originalSize.swapFiles + numSwapFiles, originalSize.unacknowledgedCount, originalSize.unacknowledgedBytes);
        updated = size.compareAndSet(originalSize, newSize);
    }
    this.swapLocations.addAll(swapLocations);
}

Example 34 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardConnectionDAO method getFlowFile.

@Override
public FlowFileRecord getFlowFile(String id, String flowFileUuid) {
    try {
        final Connection connection = locateConnection(id);
        final FlowFileQueue queue = connection.getFlowFileQueue();
        final FlowFileRecord flowFile = queue.getFlowFile(flowFileUuid);
        if (flowFile == null) {
            throw new ResourceNotFoundException(String.format("The FlowFile with UUID %s is no longer in the active queue.", flowFileUuid));
        }
        // get the attributes and ensure appropriate access
        final Map<String, String> attributes = flowFile.getAttributes();
        final Authorizable dataAuthorizable = new DataAuthorizable(connection.getSourceAuthorizable());
        dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser(), attributes);
        return flowFile;
    } catch (final IOException ioe) {
        logger.error(String.format("Unable to get the flowfile (%s) at this time.", flowFileUuid), ioe);
        throw new IllegalStateException("Unable to get the FlowFile at this time.");
    }
}