Examples with FlowFileRecord org.apache.nifi.controller.repository.FlowFileRecord used on opensource projects

Example 11 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method listFlowFiles.

@Override
public ListFlowFileStatus listFlowFiles(final String requestIdentifier, final int maxResults) {
    // purge any old requests from the map just to keep it clean. But if there are very few requests, which is usually the case, then don't bother
    if (listRequestMap.size() > 10) {
        final List<String> toDrop = new ArrayList<>();
        for (final Map.Entry<String, ListFlowFileRequest> entry : listRequestMap.entrySet()) {
            final ListFlowFileRequest request = entry.getValue();
            final boolean completed = request.getState() == ListFlowFileState.COMPLETE || request.getState() == ListFlowFileState.FAILURE;
            if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
                toDrop.add(entry.getKey());
            }
        }
        for (final String requestId : toDrop) {
            listRequestMap.remove(requestId);
        }
    }
    // numSteps = 1 for each swap location + 1 for active queue + 1 for swap queue.
    final ListFlowFileRequest listRequest = new ListFlowFileRequest(requestIdentifier, maxResults, size());
    final Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int position = 0;
            int resultCount = 0;
            final List<FlowFileSummary> summaries = new ArrayList<>();
            // Create an ArrayList that contains all of the contents of the active queue.
            // We do this so that we don't have to hold the lock any longer than absolutely necessary.
            // We cannot simply pull the first 'maxResults' records from the queue, however, because the
            // Iterator provided by PriorityQueue does not return records in order. So we would have to either
            // use a writeLock and 'pop' the first 'maxResults' records off the queue or use a read lock and
            // do a shallow copy of the queue. The shallow copy is generally quicker because it doesn't have to do
            // the sorting to put the records back. So even though this has an expensive of Java Heap to create the
            // extra collection, we are making this trade-off to avoid locking the queue any longer than required.
            final List<FlowFileRecord> allFlowFiles;
            final Prioritizer prioritizer;
            readLock.lock();
            try {
                logger.debug("{} Acquired lock to perform listing of FlowFiles", StandardFlowFileQueue.this);
                allFlowFiles = new ArrayList<>(activeQueue);
                prioritizer = new Prioritizer(StandardFlowFileQueue.this.priorities);
            } finally {
                readLock.unlock("List FlowFiles");
            }
            listRequest.setState(ListFlowFileState.CALCULATING_LIST);
            // sort the FlowFileRecords so that we have the list in the same order as on the queue.
            Collections.sort(allFlowFiles, prioritizer);
            for (final FlowFileRecord flowFile : allFlowFiles) {
                summaries.add(summarize(flowFile, ++position));
                if (summaries.size() >= maxResults) {
                    break;
                }
            }
            logger.debug("{} Finished listing FlowFiles for active queue with a total of {} results", StandardFlowFileQueue.this, resultCount);
            listRequest.setFlowFileSummaries(summaries);
            listRequest.setState(ListFlowFileState.COMPLETE);
        }
    }, "List FlowFiles for Connection " + getIdentifier());
    t.setDaemon(true);
    t.start();
    listRequestMap.put(requestIdentifier, listRequest);
    return listRequest;
}

Example 12 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method acknowledge.

@Override
public void acknowledge(final Collection<FlowFileRecord> flowFiles) {
    long totalSize = 0L;
    for (final FlowFileRecord flowFile : flowFiles) {
        totalSize += flowFile.getSize();
    }
    incrementUnacknowledgedQueueSize(-flowFiles.size(), -totalSize);
    if (connection.getSource().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
        // it's possible that queue was full but no longer is. Notify that the source may now be available to run,
        // because of back pressure caused by this queue.
        scheduler.registerEvent(connection.getSource());
    }
}

Example 13 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}

Example 14 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class SimpleSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream destination) throws IOException {
    if (toSwap == null || toSwap.isEmpty()) {
        return;
    }
    long contentSize = 0L;
    for (final FlowFileRecord record : toSwap) {
        contentSize += record.getSize();
    }
    // persist record to disk via the swap file
    final DataOutputStream out = new DataOutputStream(destination);
    try {
        out.writeInt(SWAP_ENCODING_VERSION);
        out.writeUTF(queue.getIdentifier());
        out.writeInt(toSwap.size());
        out.writeLong(contentSize);
        // get the max record id and write that out so that we know it quickly for restoration
        long maxRecordId = 0L;
        for (final FlowFileRecord flowFile : toSwap) {
            if (flowFile.getId() > maxRecordId) {
                maxRecordId = flowFile.getId();
            }
        }
        out.writeLong(maxRecordId);
        for (final FlowFileRecord flowFile : toSwap) {
            out.writeLong(flowFile.getId());
            out.writeLong(flowFile.getEntryDate());
            out.writeLong(flowFile.getLineageStartDate());
            out.writeLong(flowFile.getLineageStartIndex());
            out.writeLong(flowFile.getLastQueueDate());
            out.writeLong(flowFile.getQueueDateIndex());
            out.writeLong(flowFile.getSize());
            final ContentClaim claim = flowFile.getContentClaim();
            if (claim == null) {
                out.writeBoolean(false);
            } else {
                out.writeBoolean(true);
                final ResourceClaim resourceClaim = claim.getResourceClaim();
                out.writeUTF(resourceClaim.getId());
                out.writeUTF(resourceClaim.getContainer());
                out.writeUTF(resourceClaim.getSection());
                out.writeLong(claim.getOffset());
                out.writeLong(claim.getLength());
                out.writeLong(flowFile.getContentClaimOffset());
                out.writeBoolean(resourceClaim.isLossTolerant());
            }
            final Map<String, String> attributes = flowFile.getAttributes();
            out.writeInt(attributes.size());
            for (final Map.Entry<String, String> entry : attributes.entrySet()) {
                writeString(entry.getKey(), out);
                writeString(entry.getValue(), out);
            }
        }
    } finally {
        out.flush();
    }
    logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", toSwap.size(), queue, swapLocation);
}

Example 15 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class FlowFileRecordFieldMap method getFlowFile.

@SuppressWarnings("unchecked")
public static FlowFileRecord getFlowFile(final Record record, final ResourceClaimManager claimManager) {
    final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder();
    builder.id((Long) record.getFieldValue(FlowFileSchema.RECORD_ID));
    builder.entryDate((Long) record.getFieldValue(FlowFileSchema.ENTRY_DATE));
    builder.size((Long) record.getFieldValue(FlowFileSchema.FLOWFILE_SIZE));
    builder.addAttributes((Map<String, String>) record.getFieldValue(FlowFileSchema.ATTRIBUTES));
    builder.lineageStart((Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_DATE), (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_INDEX));
    builder.lastQueued((Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE), (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE_INDEX));
    final Record contentClaimRecord = (Record) record.getFieldValue(FlowFileSchema.CONTENT_CLAIM);
    if (contentClaimRecord != null) {
        final ContentClaim claim = ContentClaimFieldMap.getContentClaim(contentClaimRecord, claimManager);
        builder.contentClaim(claim);
        final Long offset = ContentClaimFieldMap.getContentClaimOffset(contentClaimRecord);
        if (offset != null) {
            builder.contentClaimOffset(offset);
        }
    }
    return builder.build();
}