Examples with ConnectableDTO org.apache.nifi.web.api.dto.ConnectableDTO used on opensource projects

Example 11 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 12 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class ImportReusableTemplate method createRemoteInputPorts.

private boolean createRemoteInputPorts(UploadProgressMessage remoteInputPortsMessage) {
    ImportComponentOption remoteProcessGroupOption = importTemplateOptions.findImportComponentOption(ImportComponent.REMOTE_INPUT_PORT);
    String rootProcessGroupId = templateConnectionUtil.getRootProcessGroup().getId();
    String reusableTemplateProcessGroupId = templateConnectionUtil.getReusableTemplateProcessGroupId();
    Map<String, PortDTO> reusableTemplateCategoryPorts = getReusableTemplateCategoryProcessGroup().getContents().getInputPorts().stream().collect(Collectors.toMap(p -> p.getName(), p -> p));
    StringBuffer connectedStr = new StringBuffer("");
    remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> r.isSelected() && !r.isExisting()).forEach(r -> {
        // check/create the port at the parent canvas
        PortDTO portDTO = new PortDTO();
        portDTO.setName(r.getInputPortName());
        portDTO.setType(NifiConstants.INPUT_PORT);
        portDTO.setState(NifiProcessUtil.PROCESS_STATE.STOPPED.name());
        PortDTO newInputPort = nifiRestClient.getNiFiRestClient().processGroups().createInputPort(rootProcessGroupId, portDTO);
        getItemsCreated().addCreatedRemoteInputPort(newInputPort);
        PortDTO reusableTemplatePort = reusableTemplateCategoryPorts.get(r.getInputPortName());
        // connect this to the Reusable Template input port with the same name
        ConnectableDTO source = new ConnectableDTO();
        source.setGroupId(rootProcessGroupId);
        source.setId(newInputPort.getId());
        source.setName(newInputPort.getName());
        source.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
        ConnectableDTO dest = new ConnectableDTO();
        dest.setGroupId(reusableTemplateProcessGroupId);
        dest.setName(r.getInputPortName());
        dest.setId(reusableTemplatePort.getId());
        dest.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
        ConnectionDTO connectionDTO = nifiRestClient.getNiFiRestClient().processGroups().createConnection(rootProcessGroupId, source, dest);
        getItemsCreated().addCreatedRemoteInputPortConnection(connectionDTO);
        if (connectedStr.length() != 0) {
            connectedStr.append(",");
        } else {
            connectedStr.append("Created ");
        }
        connectedStr.append(r.getInputPortName());
        remoteInputPortsMessage.update(connectedStr.toString());
    });
    if (connectedStr.length() != 0) {
        connectedStr.append(" as remote input ports");
    }
    if (connectedStr.length() > 0) {
        remoteInputPortsMessage.update(connectedStr.toString(), true);
    } else {
        remoteInputPortsMessage.complete(true);
    }
    return true;
}

Example 13 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class ImportReusableTemplate method recreateOutputPortConnections.

private void recreateOutputPortConnections(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    VersionedProcessGroup versionedProcessGroup = newTemplateInstance.getVersionedProcessGroup();
    String templateName = importTemplate.getTemplateName();
    // Dest == input port in versioned off process group
    if (versionedProcessGroup != null) {
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ConnectionDTO connectionDTO : versionedProcessGroup.getDeletedInputPortConnections()) {
                if (connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)) {
                    // connect
                    PortDTO destPort = newTemplateInstance.getProcessGroupEntity().getContents().getInputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionDTO.getDestination().getName()) && connectionDTO.getDestination().getGroupId().equalsIgnoreCase(newTemplateInstance.getVersionedProcessGroup().getProcessGroupPriorToVersioning().getId())).findFirst().orElse(null);
                    if (destPort != null) {
                        // make the connection now from the output port to the 'connectionToUse' destination
                        ConnectableDTO source = NifiConnectionUtil.asNewConnectable(connectionDTO.getSource());
                        ConnectableDTO dest = NifiConnectionUtil.asConnectable(destPort);
                        ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                        connections.add(newConnection);
                        // possibly store the ports too?
                        log.info("Reconnected output port {} ({}) to this new process group input port:  {} {{}) ", source.getName(), source.getId(), dest.getName(), dest.getId());
                    } else {
                        // ERROR cant recreate previous connections that were going into this reusable template
                        String msg = "Unable to recreate the connection for template: " + templateName + " that was previously connected to this template prior to the update. The following connection is missing:  Connecting ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "'].";
                        log.error(msg);
                        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, msg, "");
                        importStatusMessage.update("Unable to establish prior connection for reusable template: " + templateName + ".  Connection:  ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "']", false);
                        break;
                    }
                }
            }
        }
    }
}

Example 14 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project nifi-minifi by apache.

the class ConnectionSchemaTest method testNoSourceId.

@Test
public void testNoSourceId() {
    dto.setSource(new ConnectableDTO());
    map.remove(ConnectionSchema.SOURCE_ID_KEY);
    assertDtoAndMapConstructorAreSame(1);
}

Example 15 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project nifi-minifi by apache.

the class FlowSnippetDTOEnricher method enrich.

public void enrich(FlowSnippetDTO flowSnippetDTO, final String encodingVersion) {
    List<FlowSnippetDTO> allFlowSnippets = getAllFlowSnippets(flowSnippetDTO);
    Set<RemoteProcessGroupDTO> remoteProcessGroups = getAll(allFlowSnippets, FlowSnippetDTO::getRemoteProcessGroups).collect(Collectors.toSet());
    Map<String, String> connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName));
    Map<String, String> rpgIdToTargetIdMap = new HashMap<>();
    for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) {
        final RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents();
        final Set<RemoteProcessGroupPortDTO> rpgInputPortDtos = nullToEmpty(contents.getInputPorts());
        final Set<RemoteProcessGroupPortDTO> rpgOutputPortDtos = nullToEmpty(contents.getOutputPorts());
        switch(encodingVersion) {
            case "1.2":
                // Map all port DTOs to their respective targetIds
                rpgIdToTargetIdMap.putAll(Stream.concat(rpgInputPortDtos.stream(), rpgOutputPortDtos.stream()).collect(Collectors.toMap(RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getTargetId)));
                break;
            default:
                break;
        }
        addConnectables(connectableNameMap, rpgInputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
        addConnectables(connectableNameMap, rpgOutputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
    }
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getInputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getOutputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    final Set<ConnectionDTO> connections = getAll(allFlowSnippets, FlowSnippetDTO::getConnections).collect(Collectors.toSet());
    // Enrich connection endpoints using known names and overriding with targetIds for remote ports
    for (ConnectionDTO connection : connections) {
        setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap);
        setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap);
    }
    // Override any ids that are for Remote Ports to use their target Ids where available
    connections.stream().flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())).filter(connectable -> connectable.getType().equals(ConnectableType.REMOTE_OUTPUT_PORT.toString()) || connectable.getType().equals(ConnectableType.REMOTE_INPUT_PORT.toString())).forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId())));
    // Establish unique names for connections
    for (ConnectionDTO connection : connections) {
        if (StringUtil.isNullOrEmpty(connection.getName())) {
            StringBuilder name = new StringBuilder();
            ConnectableDTO connectionSource = connection.getSource();
            name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap));
            name.append("/");
            if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) {
                name.append(connection.getSelectedRelationships().iterator().next());
            }
            name.append("/");
            ConnectableDTO connectionDestination = connection.getDestination();
            name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap));
            connection.setName(name.toString());
        }
    }
    nullToEmpty(flowSnippetDTO.getProcessGroups()).stream().map(ProcessGroupDTO::getContents).forEach(snippetDTO -> enrich(snippetDTO, encodingVersion));
}