Examples with Connection org.apache.nifi.connectable.Connection used on opensource projects

Example 66 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardProcessSession method getQueueSize.

@Override
public QueueSize getQueueSize() {
    verifyTaskActive();
    int flowFileCount = 0;
    long byteCount = 0L;
    for (final Connection conn : context.getPollableConnections()) {
        final QueueSize queueSize = conn.getFlowFileQueue().size();
        flowFileCount += queueSize.getObjectCount();
        byteCount += queueSize.getByteCount();
    }
    return new QueueSize(flowFileCount, byteCount);
}

Example 67 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardProcessSession method expireFlowFiles.

public void expireFlowFiles() {
    final Set<FlowFileRecord> expired = new HashSet<>();
    final FlowFileFilter filter = new FlowFileFilter() {

        @Override
        public FlowFileFilterResult filter(final FlowFile flowFile) {
            return FlowFileFilterResult.REJECT_AND_CONTINUE;
        }
    };
    for (final Connection conn : context.getConnectable().getIncomingConnections()) {
        do {
            expired.clear();
            conn.getFlowFileQueue().poll(filter, expired);
            removeExpired(expired, conn);
        } while (!expired.isEmpty());
    }
}

Example 68 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardProcessSession method get.

private List<FlowFile> get(final ConnectionPoller poller, final boolean lockAllQueues) {
    final List<Connection> connections = context.getPollableConnections();
    if (lockAllQueues) {
        for (final Connection connection : connections) {
            connection.lock();
        }
    }
    final int startIndex = context.getNextIncomingConnectionIndex();
    try {
        for (int i = 0; i < connections.size(); i++) {
            final int connectionIndex = (startIndex + i) % connections.size();
            final Connection conn = connections.get(connectionIndex);
            final Set<FlowFileRecord> expired = new HashSet<>();
            final List<FlowFileRecord> newlySelected = poller.poll(conn, expired);
            removeExpired(expired, conn);
            if (newlySelected.isEmpty() && expired.isEmpty()) {
                continue;
            }
            for (final FlowFileRecord flowFile : newlySelected) {
                registerDequeuedRecord(flowFile, conn);
            }
            return new ArrayList<>(newlySelected);
        }
        return new ArrayList<>();
    } finally {
        if (lockAllQueues) {
            for (final Connection connection : connections) {
                connection.unlock();
            }
        }
    }
}

Example 69 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardProcessSession method checkpoint.

public void checkpoint() {
    verifyTaskActive();
    resetWriteClaims(false);
    closeStreams(openInputStreams, "committed", "input");
    closeStreams(openOutputStreams, "committed", "output");
    if (!readRecursionSet.isEmpty()) {
        throw new IllegalStateException();
    }
    if (!writeRecursionSet.isEmpty()) {
        throw new IllegalStateException();
    }
    if (this.checkpoint == null) {
        this.checkpoint = new Checkpoint();
    }
    if (records.isEmpty()) {
        LOG.trace("{} checkpointed, but no events were performed by this ProcessSession", this);
        return;
    }
    // any drop event that is the result of an auto-terminate should happen at the very end, so we keep the
    // records in a separate List so that they can be persisted to the Provenance Repo after all of the
    // Processor-reported events.
    List<ProvenanceEventRecord> autoTerminatedEvents = null;
    // validate that all records have a transfer relationship for them and if so determine the destination node and clone as necessary
    final Map<FlowFileRecord, StandardRepositoryRecord> toAdd = new HashMap<>();
    for (final StandardRepositoryRecord record : records.values()) {
        if (record.isMarkedForDelete()) {
            continue;
        }
        final Relationship relationship = record.getTransferRelationship();
        if (relationship == null) {
            rollback();
            throw new FlowFileHandlingException(record.getCurrent() + " transfer relationship not specified");
        }
        final List<Connection> destinations = new ArrayList<>(context.getConnections(relationship));
        if (destinations.isEmpty() && !context.getConnectable().isAutoTerminated(relationship)) {
            if (relationship != Relationship.SELF) {
                rollback();
                throw new FlowFileHandlingException(relationship + " does not have any destinations for " + context.getConnectable());
            }
        }
        if (destinations.isEmpty() && relationship == Relationship.SELF) {
            record.setDestination(record.getOriginalQueue());
        } else if (destinations.isEmpty()) {
            record.markForDelete();
            if (autoTerminatedEvents == null) {
                autoTerminatedEvents = new ArrayList<>();
            }
            final ProvenanceEventRecord dropEvent;
            try {
                dropEvent = provenanceReporter.generateDropEvent(record.getCurrent(), "Auto-Terminated by " + relationship.getName() + " Relationship");
                autoTerminatedEvents.add(dropEvent);
            } catch (final Exception e) {
                LOG.warn("Unable to generate Provenance Event for {} on behalf of {} due to {}", record.getCurrent(), connectableDescription, e);
                if (LOG.isDebugEnabled()) {
                    LOG.warn("", e);
                }
            }
        } else {
            // remove last element
            final Connection finalDestination = destinations.remove(destinations.size() - 1);
            record.setDestination(finalDestination.getFlowFileQueue());
            incrementConnectionInputCounts(finalDestination, record);
            for (final Connection destination : destinations) {
                // iterate over remaining destinations and "clone" as needed
                incrementConnectionInputCounts(destination, record);
                final FlowFileRecord currRec = record.getCurrent();
                final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec);
                builder.id(context.getNextFlowFileSequence());
                final String newUuid = UUID.randomUUID().toString();
                builder.addAttribute(CoreAttributes.UUID.key(), newUuid);
                final FlowFileRecord clone = builder.build();
                final StandardRepositoryRecord newRecord = new StandardRepositoryRecord(destination.getFlowFileQueue());
                provenanceReporter.clone(currRec, clone, false);
                final ContentClaim claim = clone.getContentClaim();
                if (claim != null) {
                    context.getContentRepository().incrementClaimaintCount(claim);
                }
                newRecord.setWorking(clone, Collections.<String, String>emptyMap());
                newRecord.setDestination(destination.getFlowFileQueue());
                newRecord.setTransferRelationship(record.getTransferRelationship());
                // put the mapping into toAdd because adding to records now will cause a ConcurrentModificationException
                toAdd.put(clone, newRecord);
            }
        }
    }
    records.putAll(toAdd);
    toAdd.clear();
    checkpoint.checkpoint(this, autoTerminatedEvents);
    resetState();
}

Example 70 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardProcessSession method isSpuriousRouteEvent.

/**
 * Checks if the given event is a spurious ROUTE, meaning that the ROUTE
 * indicates that a FlowFile was routed to a relationship with only 1
 * connection and that Connection is the Connection from which the FlowFile
 * was pulled. I.e., the FlowFile was really routed nowhere.
 *
 * @param event event
 * @param records records
 * @return true if spurious route
 */
private boolean isSpuriousRouteEvent(final ProvenanceEventRecord event, final Map<FlowFileRecord, StandardRepositoryRecord> records) {
    if (event.getEventType() == ProvenanceEventType.ROUTE) {
        final String relationshipName = event.getRelationship();
        final Relationship relationship = new Relationship.Builder().name(relationshipName).build();
        final Collection<Connection> connectionsForRelationship = this.context.getConnections(relationship);
        // as it may be cloning the FlowFile and adding to multiple connections.
        if (connectionsForRelationship.size() == 1) {
            for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : records.entrySet()) {
                final FlowFileRecord flowFileRecord = entry.getKey();
                if (event.getFlowFileUuid().equals(flowFileRecord.getAttribute(CoreAttributes.UUID.key()))) {
                    final StandardRepositoryRecord repoRecord = entry.getValue();
                    if (repoRecord.getOriginalQueue() == null) {
                        return false;
                    }
                    final String originalQueueId = repoRecord.getOriginalQueue().getIdentifier();
                    final Connection destinationConnection = connectionsForRelationship.iterator().next();
                    final String destinationQueueId = destinationConnection.getFlowFileQueue().getIdentifier();
                    return originalQueueId.equals(destinationQueueId);
                }
            }
        }
    }
    return false;
}