Examples with ConnectionStatus org.apache.nifi.controller.status.ConnectionStatus used on opensource projects

Example 16 with ConnectionStatus

use of org.apache.nifi.controller.status.ConnectionStatus in project nifi by apache.

the class NiFiFlowAnalyzer method getNextProcessComponent.

private List<String> getNextProcessComponent(NiFiFlow nifiFlow, NiFiFlowPath path, String componentId) {
    final List<ConnectionStatus> outs = nifiFlow.getOutgoingConnections(componentId);
    if (outs == null || outs.isEmpty()) {
        return Collections.emptyList();
    }
    final List<String> nextProcessComponent = new ArrayList<>();
    for (ConnectionStatus out : outs) {
        final String destinationId = out.getDestinationId();
        if (path.getProcessComponentIds().contains(destinationId)) {
            // If the connection is pointing back to current path, then skip it to avoid loop.
            continue;
        }
        if (nifiFlow.isProcessComponent(destinationId)) {
            nextProcessComponent.add(destinationId);
        } else {
            nextProcessComponent.addAll(getNextProcessComponent(nifiFlow, path, destinationId));
        }
    }
    return nextProcessComponent;
}

Example 17 with ConnectionStatus

use of org.apache.nifi.controller.status.ConnectionStatus in project nifi by apache.

the class NiFiFlowAnalyzer method traverse.

private void traverse(NiFiFlow nifiFlow, NiFiFlowPath path, String componentId) {
    // E.g InputPort -> MergeContent, GenerateFlowFile -> InputPort.
    if (path.getProcessComponentIds().size() > 0 && nifiFlow.isRootInputPort(componentId)) {
        return;
    }
    // Add known inputs/outputs to/from this processor, such as RootGroupIn/Output port
    if (nifiFlow.isProcessComponent(componentId)) {
        path.addProcessor(componentId);
    }
    final List<ConnectionStatus> outs = nifiFlow.getOutgoingConnections(componentId);
    if (outs == null || outs.isEmpty()) {
        return;
    }
    // Analyze destination process components.
    final List<String> nextProcessComponents = getNextProcessComponent(nifiFlow, path, componentId);
    nextProcessComponents.forEach(destPid -> {
        if (path.getProcessComponentIds().contains(destPid)) {
            // Avoid loop to it self.
            return;
        }
        // If the destination has more than one inputs, or there are multiple destinations,
        // then it should be treated as a separate flow path.
        final boolean createJointPoint = nextProcessComponents.size() > 1 || getIncomingProcessorsIds(nifiFlow, nifiFlow.getIncomingConnections(destPid)).size() > 1;
        if (createJointPoint) {
            final boolean alreadyTraversed = nifiFlow.isTraversedPath(destPid);
            // Create an input queue DataSet because Atlas doesn't show lineage if it doesn't have in and out.
            // This DataSet is also useful to link flowPaths together on Atlas lineage graph.
            final Tuple<AtlasObjectId, AtlasEntity> queueTuple = nifiFlow.getOrCreateQueue(destPid);
            final AtlasObjectId queueId = queueTuple.getKey();
            path.getOutputs().add(queueId);
            // If the destination is already traversed once, it doesn't have to be visited again.
            if (alreadyTraversed) {
                return;
            }
            // Get existing or create new one.
            final NiFiFlowPath jointPoint = nifiFlow.getOrCreateFlowPath(destPid);
            jointPoint.getInputs().add(queueId);
            // Start traversing as a new joint point.
            traverse(nifiFlow, jointPoint, destPid);
        } else {
            // Normal relation, continue digging.
            traverse(nifiFlow, path, destPid);
        }
    });
}

Example 18 with ConnectionStatus

use of org.apache.nifi.controller.status.ConnectionStatus in project nifi by apache.

the class NiFiRootGroupPort method analyze.

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    if (!ProvenanceEventType.SEND.equals(event.getEventType()) && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) {
        return null;
    }
    final boolean isInputPort = event.getComponentType().equals("Input Port");
    final String type = isInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT;
    final String rootPortId = event.getComponentId();
    final S2STransitUrl s2sUrl = parseTransitURL(event.getTransitUri(), context.getClusterResolver());
    // Find connections connecting to/from the remote port.
    final List<ConnectionStatus> connections = isInputPort ? context.findConnectionFrom(rootPortId) : context.findConnectionTo(rootPortId);
    if (connections == null || connections.isEmpty()) {
        logger.warn("Connection was not found: {}", new Object[] { event });
        return null;
    }
    // The name of the port can be retrieved from any connection, use the first one.
    final ConnectionStatus connection = connections.get(0);
    final Referenceable ref = new Referenceable(type);
    ref.set(ATTR_NAME, isInputPort ? connection.getSourceName() : connection.getDestinationName());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2sUrl.clusterName, rootPortId));
    return singleDataSetRef(event.getComponentId(), event.getEventType(), ref);
}

Example 19 with ConnectionStatus

use of org.apache.nifi.controller.status.ConnectionStatus in project nifi by apache.

the class SimpleFlowPathLineage method processRemotePortEvent.

/**
 * Create a flow_path entity corresponding to the target RemoteGroupPort when a SEND/RECEIVE event are received.
 * Because such entity can not be created in advance while analyzing flow statically,
 * as ReportingTask can not determine whether a component id is a RemoteGroupPort,
 * since connectionStatus is the only available information in ReportingContext.
 * ConnectionStatus only knows component id, component type is unknown.
 * For example, there is no difference to tell if a connected component is a funnel or a RemoteGroupPort.
 */
private void processRemotePortEvent(AnalysisContext analysisContext, NiFiFlow nifiFlow, ProvenanceEventRecord event, DataSetRefs analyzedRefs) {
    final boolean isRemoteInputPort = "Remote Input Port".equals(event.getComponentType());
    // Create a RemoteInputPort Process.
    // event.getComponentId returns UUID for RemoteGroupPort as a client of S2S, and it's different from a remote port UUID (portDataSetid).
    // See NIFI-4571 for detail.
    final Referenceable remotePortDataSet = isRemoteInputPort ? analyzedRefs.getOutputs().iterator().next() : analyzedRefs.getInputs().iterator().next();
    final String portProcessId = event.getComponentId();
    final NiFiFlowPath remotePortProcess = new NiFiFlowPath(portProcessId);
    remotePortProcess.setName(event.getComponentType());
    remotePortProcess.addProcessor(portProcessId);
    // That is only possible by calling lineage API.
    if (isRemoteInputPort) {
        final ProvenanceEventRecord previousEvent = findPreviousProvenanceEvent(analysisContext, event);
        if (previousEvent == null) {
            logger.warn("Previous event was not found: {}", new Object[] { event });
            return;
        }
        // Set groupId from incoming connection if available.
        final List<ConnectionStatus> incomingConnections = nifiFlow.getIncomingConnections(portProcessId);
        if (incomingConnections == null || incomingConnections.isEmpty()) {
            logger.warn("Incoming relationship was not found: {}", new Object[] { event });
            return;
        }
        final ConnectionStatus connection = incomingConnections.get(0);
        remotePortProcess.setGroupId(connection.getGroupId());
        final Referenceable remotePortProcessRef = toReferenceable(remotePortProcess, nifiFlow);
        createEntity(remotePortProcessRef);
        // Create a queue.
        Referenceable queueFromStaticFlowPathToRemotePortProcess = new Referenceable(TYPE_NIFI_QUEUE);
        queueFromStaticFlowPathToRemotePortProcess.set(ATTR_NAME, "queue");
        queueFromStaticFlowPathToRemotePortProcess.set(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(portProcessId));
        // Create lineage: Static flow_path -> queue
        DataSetRefs staticFlowPathRefs = new DataSetRefs(previousEvent.getComponentId());
        staticFlowPathRefs.addOutput(queueFromStaticFlowPathToRemotePortProcess);
        addDataSetRefs(nifiFlow, staticFlowPathRefs);
        // Create lineage: Queue -> RemoteInputPort process -> RemoteInputPort dataSet
        DataSetRefs remotePortRefs = new DataSetRefs(portProcessId);
        remotePortRefs.addInput(queueFromStaticFlowPathToRemotePortProcess);
        remotePortRefs.addOutput(remotePortDataSet);
        addDataSetRefs(remotePortRefs, remotePortProcessRef);
    } else {
        // For RemoteOutputPort, it's possible that multiple processors are connected.
        // In that case, the received FlowFile is cloned and passed to each connection.
        // So we need to create multiple DataSetRefs.
        final List<ConnectionStatus> connections = nifiFlow.getOutgoingConnections(portProcessId);
        if (connections == null || connections.isEmpty()) {
            logger.warn("Incoming connection was not found: {}", new Object[] { event });
            return;
        }
        // Set group id from outgoing connection if available.
        remotePortProcess.setGroupId(connections.get(0).getGroupId());
        final Referenceable remotePortProcessRef = toReferenceable(remotePortProcess, nifiFlow);
        createEntity(remotePortProcessRef);
        // Create lineage: RemoteOutputPort dataSet -> RemoteOutputPort process
        DataSetRefs remotePortRefs = new DataSetRefs(portProcessId);
        remotePortRefs.addInput(remotePortDataSet);
        addDataSetRefs(remotePortRefs, remotePortProcessRef);
        for (ConnectionStatus connection : connections) {
            final String destinationId = connection.getDestinationId();
            final NiFiFlowPath destFlowPath = nifiFlow.findPath(destinationId);
            if (destFlowPath == null) {
                // as a queue will be created by the connected RemoteInputPort to connect this RemoteOutputPort.
                continue;
            }
            // Create a queue.
            Referenceable queueFromRemotePortProcessToStaticFlowPath = new Referenceable(TYPE_NIFI_QUEUE);
            queueFromRemotePortProcessToStaticFlowPath.set(ATTR_NAME, "queue");
            queueFromRemotePortProcessToStaticFlowPath.set(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(destinationId));
            // Create lineage: Queue -> Static flow_path
            DataSetRefs staticFlowPathRefs = new DataSetRefs(destinationId);
            staticFlowPathRefs.addInput(queueFromRemotePortProcessToStaticFlowPath);
            addDataSetRefs(nifiFlow, staticFlowPathRefs);
            // Create lineage: RemoteOutputPort dataSet -> RemoteOutputPort process -> Queue
            remotePortRefs.addOutput(queueFromRemotePortProcessToStaticFlowPath);
            addDataSetRefs(remotePortRefs, remotePortProcessRef);
        }
        // Add RemoteOutputPort process, so that it can be found even if it is connected to RemoteInputPort directory without any processor in between.
        nifiFlow.getFlowPaths().put(remotePortProcess.getId(), remotePortProcess);
    }
}

Example 20 with ConnectionStatus

use of org.apache.nifi.controller.status.ConnectionStatus in project nifi by apache.

the class TestNiFiRemotePort method testRemoteOutputPort.

@Test
public void testRemoteOutputPort() {
    final String componentType = "Remote Output Port";
    final String transitUri = "http://0.example.com:8080/nifi-api/data-transfer/output-ports/port-guid/transactions/tx-guid/flow-files";
    final ProvenanceEventRecord record = Mockito.mock(ProvenanceEventRecord.class);
    when(record.getComponentId()).thenReturn("port-guid");
    when(record.getComponentType()).thenReturn(componentType);
    when(record.getTransitUri()).thenReturn(transitUri);
    when(record.getEventType()).thenReturn(ProvenanceEventType.RECEIVE);
    final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
    when(clusterResolvers.fromHostNames(matches(".+\\.example\\.com"))).thenReturn("cluster1");
    final List<ConnectionStatus> connections = new ArrayList<>();
    final ConnectionStatus connection = new ConnectionStatus();
    connection.setSourceId("port-guid");
    connection.setSourceName("outputPortA");
    connections.add(connection);
    final AnalysisContext context = Mockito.mock(AnalysisContext.class);
    when(context.getClusterResolver()).thenReturn(clusterResolvers);
    when(context.findConnectionFrom(matches("port-guid"))).thenReturn(connections);
    final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(componentType, transitUri, record.getEventType());
    assertNotNull(analyzer);
    final DataSetRefs refs = analyzer.analyze(context, record);
    assertEquals(1, refs.getInputs().size());
    assertEquals(0, refs.getOutputs().size());
    Referenceable ref = refs.getInputs().iterator().next();
    assertEquals(TYPE_NIFI_OUTPUT_PORT, ref.getTypeName());
    assertEquals("outputPortA", ref.get(ATTR_NAME));
    assertEquals("port-guid@cluster1", ref.get(ATTR_QUALIFIED_NAME));
}