Examples with Relationship org.apache.nifi.processor.Relationship used on opensource projects

Example 61 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class TestStandardProcessSession method testUpdateAttributesThenJoin.

@Test
public void testUpdateAttributesThenJoin() throws IOException {
    final FlowFileRecord flowFileRecord1 = new StandardFlowFileRecord.Builder().id(1L).addAttribute("uuid", "11111111-1111-1111-1111-111111111111").entryDate(System.currentTimeMillis()).build();
    final FlowFileRecord flowFileRecord2 = new StandardFlowFileRecord.Builder().id(2L).addAttribute("uuid", "22222222-2222-2222-2222-222222222222").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord1);
    flowFileQueue.put(flowFileRecord2);
    FlowFile ff1 = session.get();
    FlowFile ff2 = session.get();
    ff1 = session.putAttribute(ff1, "index", "1");
    ff2 = session.putAttribute(ff2, "index", "2");
    final List<FlowFile> parents = new ArrayList<>(2);
    parents.add(ff1);
    parents.add(ff2);
    final FlowFile child = session.create(parents);
    final Relationship rel = new Relationship.Builder().name("A").build();
    session.transfer(ff1, rel);
    session.transfer(ff2, rel);
    session.transfer(child, rel);
    session.commit();
    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 1000);
    // We should have a JOIN and 2 ATTRIBUTE_MODIFIED's
    assertEquals(3, events.size());
    int joinCount = 0;
    int ff1UpdateCount = 0;
    int ff2UpdateCount = 0;
    for (final ProvenanceEventRecord event : events) {
        switch(event.getEventType()) {
            case JOIN:
                assertEquals(child.getAttribute("uuid"), event.getFlowFileUuid());
                joinCount++;
                break;
            case ATTRIBUTES_MODIFIED:
                if (event.getFlowFileUuid().equals(ff1.getAttribute("uuid"))) {
                    ff1UpdateCount++;
                } else if (event.getFlowFileUuid().equals(ff2.getAttribute("uuid"))) {
                    ff2UpdateCount++;
                } else {
                    Assert.fail("Got ATTRIBUTE_MODIFIED for wrong FlowFile: " + event.getFlowFileUuid());
                }
                break;
            default:
                Assert.fail("Unexpected event type: " + event);
        }
    }
    assertEquals(1, joinCount);
    assertEquals(1, ff1UpdateCount);
    assertEquals(1, ff2UpdateCount);
    assertEquals(1, joinCount);
}

Example 62 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class NiFiRegistryFlowMapper method mapProcessor.

public VersionedProcessor mapProcessor(final ProcessorNode procNode, final ControllerServiceProvider serviceProvider) {
    final VersionedProcessor processor = new InstantiatedVersionedProcessor(procNode.getIdentifier(), procNode.getProcessGroupIdentifier());
    processor.setIdentifier(getId(procNode.getVersionedComponentId(), procNode.getIdentifier()));
    processor.setGroupIdentifier(getGroupId(procNode.getProcessGroupIdentifier()));
    processor.setType(procNode.getCanonicalClassName());
    processor.setAnnotationData(procNode.getAnnotationData());
    processor.setAutoTerminatedRelationships(procNode.getAutoTerminatedRelationships().stream().map(Relationship::getName).collect(Collectors.toSet()));
    processor.setBulletinLevel(procNode.getBulletinLevel().name());
    processor.setBundle(mapBundle(procNode.getBundleCoordinate()));
    processor.setComments(procNode.getComments());
    processor.setConcurrentlySchedulableTaskCount(procNode.getMaxConcurrentTasks());
    processor.setExecutionNode(procNode.getExecutionNode().name());
    processor.setName(procNode.getName());
    processor.setPenaltyDuration(procNode.getPenalizationPeriod());
    processor.setPosition(mapPosition(procNode.getPosition()));
    processor.setProperties(mapProperties(procNode, serviceProvider));
    processor.setPropertyDescriptors(mapPropertyDescriptors(procNode));
    processor.setRunDurationMillis(procNode.getRunDuration(TimeUnit.MILLISECONDS));
    processor.setSchedulingPeriod(procNode.getSchedulingPeriod());
    processor.setSchedulingStrategy(procNode.getSchedulingStrategy().name());
    processor.setStyle(procNode.getStyle());
    processor.setYieldDuration(procNode.getYieldPeriod());
    return processor;
}

Example 63 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class TestHttpFlowFileServerProtocol method setupMockProcessSession.

private void setupMockProcessSession() {
    // Construct a RootGroupPort as a processor to use NiFi mock library.
    final Processor rootGroupPort = mock(Processor.class);
    final Set<Relationship> relationships = new HashSet<>();
    relationships.add(Relationship.ANONYMOUS);
    when(rootGroupPort.getRelationships()).thenReturn(relationships);
    when(rootGroupPort.getIdentifier()).thenReturn("root-group-port-id");
    sessionState = new SharedSessionState(rootGroupPort, new AtomicLong(0));
    processSession = new MockProcessSession(sessionState, rootGroupPort);
    processContext = new MockProcessContext(rootGroupPort);
}

Example 64 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class StandardProcessSession method checkpoint.

public void checkpoint() {
    verifyTaskActive();
    resetWriteClaims(false);
    closeStreams(openInputStreams, "committed", "input");
    closeStreams(openOutputStreams, "committed", "output");
    if (!readRecursionSet.isEmpty()) {
        throw new IllegalStateException();
    }
    if (!writeRecursionSet.isEmpty()) {
        throw new IllegalStateException();
    }
    if (this.checkpoint == null) {
        this.checkpoint = new Checkpoint();
    }
    if (records.isEmpty()) {
        LOG.trace("{} checkpointed, but no events were performed by this ProcessSession", this);
        return;
    }
    // any drop event that is the result of an auto-terminate should happen at the very end, so we keep the
    // records in a separate List so that they can be persisted to the Provenance Repo after all of the
    // Processor-reported events.
    List<ProvenanceEventRecord> autoTerminatedEvents = null;
    // validate that all records have a transfer relationship for them and if so determine the destination node and clone as necessary
    final Map<FlowFileRecord, StandardRepositoryRecord> toAdd = new HashMap<>();
    for (final StandardRepositoryRecord record : records.values()) {
        if (record.isMarkedForDelete()) {
            continue;
        }
        final Relationship relationship = record.getTransferRelationship();
        if (relationship == null) {
            rollback();
            throw new FlowFileHandlingException(record.getCurrent() + " transfer relationship not specified");
        }
        final List<Connection> destinations = new ArrayList<>(context.getConnections(relationship));
        if (destinations.isEmpty() && !context.getConnectable().isAutoTerminated(relationship)) {
            if (relationship != Relationship.SELF) {
                rollback();
                throw new FlowFileHandlingException(relationship + " does not have any destinations for " + context.getConnectable());
            }
        }
        if (destinations.isEmpty() && relationship == Relationship.SELF) {
            record.setDestination(record.getOriginalQueue());
        } else if (destinations.isEmpty()) {
            record.markForDelete();
            if (autoTerminatedEvents == null) {
                autoTerminatedEvents = new ArrayList<>();
            }
            final ProvenanceEventRecord dropEvent;
            try {
                dropEvent = provenanceReporter.generateDropEvent(record.getCurrent(), "Auto-Terminated by " + relationship.getName() + " Relationship");
                autoTerminatedEvents.add(dropEvent);
            } catch (final Exception e) {
                LOG.warn("Unable to generate Provenance Event for {} on behalf of {} due to {}", record.getCurrent(), connectableDescription, e);
                if (LOG.isDebugEnabled()) {
                    LOG.warn("", e);
                }
            }
        } else {
            // remove last element
            final Connection finalDestination = destinations.remove(destinations.size() - 1);
            record.setDestination(finalDestination.getFlowFileQueue());
            incrementConnectionInputCounts(finalDestination, record);
            for (final Connection destination : destinations) {
                // iterate over remaining destinations and "clone" as needed
                incrementConnectionInputCounts(destination, record);
                final FlowFileRecord currRec = record.getCurrent();
                final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec);
                builder.id(context.getNextFlowFileSequence());
                final String newUuid = UUID.randomUUID().toString();
                builder.addAttribute(CoreAttributes.UUID.key(), newUuid);
                final FlowFileRecord clone = builder.build();
                final StandardRepositoryRecord newRecord = new StandardRepositoryRecord(destination.getFlowFileQueue());
                provenanceReporter.clone(currRec, clone, false);
                final ContentClaim claim = clone.getContentClaim();
                if (claim != null) {
                    context.getContentRepository().incrementClaimaintCount(claim);
                }
                newRecord.setWorking(clone, Collections.<String, String>emptyMap());
                newRecord.setDestination(destination.getFlowFileQueue());
                newRecord.setTransferRelationship(record.getTransferRelationship());
                // put the mapping into toAdd because adding to records now will cause a ConcurrentModificationException
                toAdd.put(clone, newRecord);
            }
        }
    }
    records.putAll(toAdd);
    toAdd.clear();
    checkpoint.checkpoint(this, autoTerminatedEvents);
    resetState();
}

Example 65 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class StandardProcessSession method isSpuriousRouteEvent.

/**
 * Checks if the given event is a spurious ROUTE, meaning that the ROUTE
 * indicates that a FlowFile was routed to a relationship with only 1
 * connection and that Connection is the Connection from which the FlowFile
 * was pulled. I.e., the FlowFile was really routed nowhere.
 *
 * @param event event
 * @param records records
 * @return true if spurious route
 */
private boolean isSpuriousRouteEvent(final ProvenanceEventRecord event, final Map<FlowFileRecord, StandardRepositoryRecord> records) {
    if (event.getEventType() == ProvenanceEventType.ROUTE) {
        final String relationshipName = event.getRelationship();
        final Relationship relationship = new Relationship.Builder().name(relationshipName).build();
        final Collection<Connection> connectionsForRelationship = this.context.getConnections(relationship);
        // as it may be cloning the FlowFile and adding to multiple connections.
        if (connectionsForRelationship.size() == 1) {
            for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : records.entrySet()) {
                final FlowFileRecord flowFileRecord = entry.getKey();
                if (event.getFlowFileUuid().equals(flowFileRecord.getAttribute(CoreAttributes.UUID.key()))) {
                    final StandardRepositoryRecord repoRecord = entry.getValue();
                    if (repoRecord.getOriginalQueue() == null) {
                        return false;
                    }
                    final String originalQueueId = repoRecord.getOriginalQueue().getIdentifier();
                    final Connection destinationConnection = connectionsForRelationship.iterator().next();
                    final String destinationQueueId = destinationConnection.getFlowFileQueue().getIdentifier();
                    return originalQueueId.equals(destinationQueueId);
                }
            }
        }
    }
    return false;
}