Examples with QueueSize org.apache.nifi.controller.queue.QueueSize used on opensource projects

Example 6 with QueueSize

use of org.apache.nifi.controller.queue.QueueSize in project nifi by apache.

the class StandardFlowFileQueue method dropFlowFiles.

@Override
public DropFlowFileStatus dropFlowFiles(final String requestIdentifier, final String requestor) {
    logger.info("Initiating drop of FlowFiles from {} on behalf of {} (request identifier={})", this, requestor, requestIdentifier);
    // purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
    if (dropRequestMap.size() > 10) {
        final List<String> toDrop = new ArrayList<>();
        for (final Map.Entry<String, DropFlowFileRequest> entry : dropRequestMap.entrySet()) {
            final DropFlowFileRequest request = entry.getValue();
            final boolean completed = request.getState() == DropFlowFileState.COMPLETE || request.getState() == DropFlowFileState.FAILURE;
            if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
                toDrop.add(entry.getKey());
            }
        }
        for (final String requestId : toDrop) {
            dropRequestMap.remove(requestId);
        }
    }
    final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
    final QueueSize originalSize = getQueueSize();
    dropRequest.setCurrentSize(originalSize);
    dropRequest.setOriginalSize(originalSize);
    if (originalSize.getObjectCount() == 0) {
        dropRequest.setDroppedSize(originalSize);
        dropRequest.setState(DropFlowFileState.COMPLETE);
        dropRequestMap.put(requestIdentifier, dropRequest);
        return dropRequest;
    }
    final Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            writeLock.lock();
            try {
                dropRequest.setState(DropFlowFileState.DROPPING_FLOWFILES);
                logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, getQueueSize());
                try {
                    final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
                    QueueSize droppedSize;
                    try {
                        if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                            logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                            return;
                        }
                        droppedSize = drop(activeQueueRecords, requestor);
                        logger.debug("For DropFlowFileRequest {}, Dropped {} from active queue", requestIdentifier, droppedSize);
                    } catch (final IOException ioe) {
                        logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                        logger.error("", ioe);
                        dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
                        return;
                    }
                    activeQueue.clear();
                    incrementActiveQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount());
                    dropRequest.setCurrentSize(getQueueSize());
                    dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                    final QueueSize swapSize = size.get().swapQueueSize();
                    logger.debug("For DropFlowFileRequest {}, Swap Queue has {} elements, Swapped Record Count = {}, Swapped Content Size = {}", requestIdentifier, swapQueue.size(), swapSize.getObjectCount(), swapSize.getByteCount());
                    if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                        logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                        return;
                    }
                    try {
                        droppedSize = drop(swapQueue, requestor);
                    } catch (final IOException ioe) {
                        logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                        logger.error("", ioe);
                        dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
                        return;
                    }
                    swapQueue.clear();
                    dropRequest.setCurrentSize(getQueueSize());
                    dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                    swapMode = false;
                    incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount(), 0);
                    logger.debug("For DropFlowFileRequest {}, dropped {} from Swap Queue", requestIdentifier, droppedSize);
                    final int swapFileCount = swapLocations.size();
                    final Iterator<String> swapLocationItr = swapLocations.iterator();
                    while (swapLocationItr.hasNext()) {
                        final String swapLocation = swapLocationItr.next();
                        SwapContents swapContents = null;
                        try {
                            if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                                logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                                return;
                            }
                            swapContents = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
                            droppedSize = drop(swapContents.getFlowFiles(), requestor);
                        } catch (final IncompleteSwapFileException isfe) {
                            swapContents = isfe.getPartialContents();
                            final String warnMsg = "Failed to swap in FlowFiles from Swap File " + swapLocation + " because the file was corrupt. " + "Some FlowFiles may not be dropped from the queue until NiFi is restarted.";
                            logger.warn(warnMsg);
                            if (eventReporter != null) {
                                eventReporter.reportEvent(Severity.WARNING, "Drop FlowFiles", warnMsg);
                            }
                        } catch (final IOException ioe) {
                            logger.error("Failed to swap in FlowFiles from Swap File {} in order to drop the FlowFiles for Connection {} due to {}", swapLocation, StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                            logger.error("", ioe);
                            if (eventReporter != null) {
                                eventReporter.reportEvent(Severity.ERROR, "Drop FlowFiles", "Failed to swap in FlowFiles from Swap File " + swapLocation + ". The FlowFiles contained in this Swap File will not be dropped from the queue");
                            }
                            dropRequest.setState(DropFlowFileState.FAILURE, "Failed to swap in FlowFiles from Swap File " + swapLocation + " due to " + ioe.toString());
                            if (swapContents != null) {
                                // ensure that we don't lose the FlowFiles from our queue.
                                activeQueue.addAll(swapContents.getFlowFiles());
                            }
                            return;
                        }
                        dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                        incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount(), -1);
                        dropRequest.setCurrentSize(getQueueSize());
                        swapLocationItr.remove();
                        logger.debug("For DropFlowFileRequest {}, dropped {} for Swap File {}", requestIdentifier, droppedSize, swapLocation);
                    }
                    logger.debug("Dropped FlowFiles from {} Swap Files", swapFileCount);
                    logger.info("Successfully dropped {} FlowFiles ({} bytes) from Connection with ID {} on behalf of {}", dropRequest.getDroppedSize().getObjectCount(), dropRequest.getDroppedSize().getByteCount(), StandardFlowFileQueue.this.getIdentifier(), requestor);
                    dropRequest.setState(DropFlowFileState.COMPLETE);
                } catch (final Exception e) {
                    logger.error("Failed to drop FlowFiles from Connection with ID {} due to {}", StandardFlowFileQueue.this.getIdentifier(), e.toString());
                    logger.error("", e);
                    dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + e.toString());
                }
            } finally {
                writeLock.unlock("Drop FlowFiles");
            }
        }
    }, "Drop FlowFiles for Connection " + getIdentifier());
    t.setDaemon(true);
    t.start();
    dropRequestMap.put(requestIdentifier, dropRequest);
    return dropRequest;
}

Example 7 with QueueSize

use of org.apache.nifi.controller.queue.QueueSize in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}

Example 8 with QueueSize

use of org.apache.nifi.controller.queue.QueueSize in project nifi by apache.

the class SwapSummaryFieldMap method getSwapSummary.

@SuppressWarnings("unchecked")
public static SwapSummary getSwapSummary(final Record record, final ResourceClaimManager claimManager) {
    final int flowFileCount = (Integer) record.getFieldValue(SwapSchema.FLOWFILE_COUNT);
    final long flowFileSize = (Long) record.getFieldValue(SwapSchema.FLOWFILE_SIZE);
    final QueueSize queueSize = new QueueSize(flowFileCount, flowFileSize);
    final long maxFlowFileId = (Long) record.getFieldValue(SwapSchema.MAX_RECORD_ID);
    final Map<Record, Integer> resourceClaimRecords = (Map<Record, Integer>) record.getFieldValue(SwapSchema.RESOURCE_CLAIMS);
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final Map.Entry<Record, Integer> entry : resourceClaimRecords.entrySet()) {
        final Record resourceClaimRecord = entry.getKey();
        final ResourceClaim claim = ResourceClaimFieldMap.getResourceClaim(resourceClaimRecord, claimManager);
        for (int i = 0; i < entry.getValue(); i++) {
            resourceClaims.add(claim);
        }
    }
    return new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
}

Example 9 with QueueSize

use of org.apache.nifi.controller.queue.QueueSize in project nifi by apache.

the class FlowController method getTotalFlowFileCount.

// 
// Access to controller status
// 
public QueueSize getTotalFlowFileCount(final ProcessGroup group) {
    int count = 0;
    long contentSize = 0L;
    for (final Connection connection : group.getConnections()) {
        final QueueSize size = connection.getFlowFileQueue().size();
        count += size.getObjectCount();
        contentSize += size.getByteCount();
    }
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        final QueueSize size = getTotalFlowFileCount(childGroup);
        count += size.getObjectCount();
        contentSize += size.getByteCount();
    }
    return new QueueSize(count, contentSize);
}

Example 10 with QueueSize

use of org.apache.nifi.controller.queue.QueueSize in project nifi by apache.

the class TestStandardFlowFileQueue method testExpire.

@Test
public void testExpire() {
    queue.setFlowFileExpiration("1 ms");
    for (int i = 0; i < 100; i++) {
        queue.put(new TestFlowFile());
    }
    // just make sure that the flowfiles have time to expire.
    try {
        Thread.sleep(100L);
    } catch (final InterruptedException ie) {
    }
    final Set<FlowFileRecord> expiredRecords = new HashSet<>(100);
    final FlowFileRecord pulled = queue.poll(expiredRecords);
    assertNull(pulled);
    assertEquals(100, expiredRecords.size());
    final QueueSize activeSize = queue.getActiveQueueSize();
    assertEquals(0, activeSize.getObjectCount());
    assertEquals(0L, activeSize.getByteCount());
    final QueueSize unackSize = queue.getUnacknowledgedQueueSize();
    assertEquals(0, unackSize.getObjectCount());
    assertEquals(0L, unackSize.getByteCount());
}