Examples with FlowFileQueue org.apache.nifi.controller.queue.FlowFileQueue used on opensource projects

Example 41 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class WriteAheadRepositoryRecordSerde method deserializeRecord.

@Override
public StandardRepositoryRecord deserializeRecord(final DataInputStream in, final int version) throws IOException {
    final int action = in.read();
    if (action == -1) {
        return null;
    }
    final long recordId = in.readLong();
    if (action == ACTION_DELETE) {
        final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder().id(recordId);
        if (version > 4) {
            deserializeClaim(in, version, ffBuilder);
        }
        final FlowFileRecord flowFileRecord = ffBuilder.build();
        final StandardRepositoryRecord record = new StandardRepositoryRecord((FlowFileQueue) null, flowFileRecord);
        record.markForDelete();
        return record;
    }
    // if action was not delete, it must be create/swap in
    final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
    final long entryDate = in.readLong();
    if (version > 1) {
        // read the lineage identifiers and lineage start date, which were added in version 2.
        if (version < 9) {
            final int numLineageIds = in.readInt();
            for (int i = 0; i < numLineageIds; i++) {
                // skip identifiers
                in.readUTF();
            }
        }
        final long lineageStartDate = in.readLong();
        final long lineageStartIndex;
        if (version > 7) {
            lineageStartIndex = in.readLong();
        } else {
            lineageStartIndex = 0L;
        }
        ffBuilder.lineageStart(lineageStartDate, lineageStartIndex);
        if (version > 5) {
            final long lastQueueDate = in.readLong();
            final long queueDateIndex;
            if (version > 7) {
                queueDateIndex = in.readLong();
            } else {
                queueDateIndex = 0L;
            }
            ffBuilder.lastQueued(lastQueueDate, queueDateIndex);
        }
    }
    final long size = in.readLong();
    final String connectionId = readString(in);
    logger.debug("{} -> {}", new Object[] { recordId, connectionId });
    ffBuilder.id(recordId);
    ffBuilder.entryDate(entryDate);
    ffBuilder.size(size);
    deserializeClaim(in, version, ffBuilder);
    final int attributesChanged = in.read();
    if (attributesChanged == 1) {
        final int numAttributes = in.readInt();
        final Map<String, String> attributes = new HashMap<>();
        for (int i = 0; i < numAttributes; i++) {
            final String key = readString(in);
            final String value = readString(in);
            attributes.put(key, value);
        }
        ffBuilder.addAttributes(attributes);
    } else if (attributesChanged == -1) {
        throw new EOFException();
    } else if (attributesChanged != 0) {
        throw new IOException("Attribute Change Qualifier not found in stream; found value: " + attributesChanged + " after successfully restoring " + recordsRestored + " records");
    }
    final FlowFileRecord flowFile = ffBuilder.build();
    String swapLocation = null;
    if (action == ACTION_SWAPPED_IN) {
        swapLocation = in.readUTF();
    }
    final StandardRepositoryRecord record;
    final FlowFileQueue queue = getFlowFileQueue(connectionId);
    record = new StandardRepositoryRecord(queue, flowFile);
    if (swapLocation != null) {
        record.setSwapLocation(swapLocation);
    }
    if (connectionId.isEmpty()) {
        logger.warn("{} does not have a FlowFile Queue associated with it; this record will be discarded", flowFile);
        record.markForAbort();
    } else if (queue == null) {
        logger.warn("{} maps to unknown FlowFile Queue {}; this record will be discarded", flowFile, connectionId);
        record.markForAbort();
    }
    recordsRestored++;
    return record;
}

Example 42 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class DtoFactory method createConnectionDiagnosticsDto.

private ConnectionDiagnosticsDTO createConnectionDiagnosticsDto(final Connection connection) {
    final ConnectionDiagnosticsDTO dto = new ConnectionDiagnosticsDTO();
    dto.setConnection(createConnectionDto(connection));
    final FlowFileQueue queue = connection.getFlowFileQueue();
    final QueueSize totalSize = queue.size();
    dto.setTotalByteCount(totalSize.getByteCount());
    dto.setTotalFlowFileCount(totalSize.getObjectCount());
    final QueueSize activeSize = queue.getActiveQueueSize();
    dto.setActiveQueueByteCount(activeSize.getByteCount());
    dto.setActiveQueueFlowFileCount(activeSize.getObjectCount());
    final QueueSize inFlightSize = queue.getUnacknowledgedQueueSize();
    dto.setInFlightByteCount(inFlightSize.getByteCount());
    dto.setInFlightFlowFileCount(inFlightSize.getObjectCount());
    final QueueSize swapSize = queue.getSwapQueueSize();
    dto.setSwapByteCount(swapSize.getByteCount());
    dto.setSwapFlowFileCount(swapSize.getObjectCount());
    dto.setSwapFiles(queue.getSwapFileCount());
    dto.setAllActiveQueueFlowFilesPenalized(queue.isAllActiveFlowFilesPenalized());
    dto.setAnyActiveQueueFlowFilesPenalized(queue.isAnyActiveFlowFilePenalized());
    return dto;
}

Example 43 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class StandardConnectionDAO method verifyList.

@Override
public void verifyList(String id) {
    final Connection connection = locateConnection(id);
    final FlowFileQueue queue = connection.getFlowFileQueue();
    verifyList(queue);
}

Example 44 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class StandardConnectionDAO method createFlowFileListingRequest.

@Override
public ListFlowFileStatus createFlowFileListingRequest(String id, String listingRequestId) {
    final Connection connection = locateConnection(id);
    final FlowFileQueue queue = connection.getFlowFileQueue();
    // ensure we can list
    verifyList(queue);
    return queue.listFlowFiles(listingRequestId, 100);
}

Example 45 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class StandardConnectionDAO method getFlowFile.

@Override
public FlowFileRecord getFlowFile(String id, String flowFileUuid) {
    try {
        final Connection connection = locateConnection(id);
        final FlowFileQueue queue = connection.getFlowFileQueue();
        final FlowFileRecord flowFile = queue.getFlowFile(flowFileUuid);
        if (flowFile == null) {
            throw new ResourceNotFoundException(String.format("The FlowFile with UUID %s is no longer in the active queue.", flowFileUuid));
        }
        // get the attributes and ensure appropriate access
        final Map<String, String> attributes = flowFile.getAttributes();
        final Authorizable dataAuthorizable = new DataAuthorizable(connection.getSourceAuthorizable());
        dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser(), attributes);
        return flowFile;
    } catch (final IOException ioe) {
        logger.error(String.format("Unable to get the flowfile (%s) at this time.", flowFileUuid), ioe);
        throw new IllegalStateException("Unable to get the FlowFile at this time.");
    }
}