Examples with ConnectionDTO org.apache.nifi.web.api.dto.ConnectionDTO used on opensource projects

Example 21 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class LegacyNifiRestClient method startProcessGroupAndParentInputPorts.

public void startProcessGroupAndParentInputPorts(ProcessGroupDTO entity) {
    // 1 startAll
    try {
        startAll(entity.getId(), entity.getParentGroupId());
    } catch (NifiClientRuntimeException e) {
        log.error("Error trying to mark connection ports Running for {}", entity.getName());
    }
    Set<PortDTO> ports = null;
    try {
        ports = getPortsForProcessGroup(entity.getParentGroupId());
    } catch (NifiClientRuntimeException e) {
        log.error("Error getPortsForProcessGroup {}", entity.getName());
    }
    if (ports != null && !ports.isEmpty()) {
        Map<String, PortDTO> portsById = ports.stream().collect(Collectors.toMap(port -> port.getId(), port -> port));
        ProcessGroupFlowDTO flow = getNiFiRestClient().processGroups().flow(entity.getParentGroupId());
        if (flow != null) {
            Set<PortDTO> matchingPorts = new HashSet<>();
            flow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).forEach(connectionDTO -> {
                // map input ports
                if (connectionDTO.getDestination().getGroupId().equalsIgnoreCase(entity.getId()) && portsById.containsKey(connectionDTO.getSource().getId())) {
                    matchingPorts.add(portsById.get(connectionDTO.getSource().getId()));
                }
                // map output ports
                if (connectionDTO.getSource().getGroupId().equalsIgnoreCase(entity.getId()) && portsById.containsKey(connectionDTO.getDestination().getId())) {
                    matchingPorts.add(portsById.get(connectionDTO.getDestination().getId()));
                }
            });
            for (PortDTO port : matchingPorts) {
                port.setState(NifiProcessUtil.PROCESS_STATE.RUNNING.name());
                if (port.getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name())) {
                    try {
                        if (port.getState().equalsIgnoreCase(NifiProcessUtil.PROCESS_STATE.DISABLED.name())) {
                            PortDTO stoppedPort = stopInputPort(entity.getParentGroupId(), port.getId());
                            if (stoppedPort.getValidationErrors() != null && !stoppedPort.getValidationErrors().isEmpty()) {
                                startInputPort(entity.getParentGroupId(), port.getId());
                            } else {
                                String validationErrors = stoppedPort.getValidationErrors() != null ? ("Validation Errors: " + stoppedPort.getValidationErrors().stream().collect(Collectors.joining(","))) : "";
                                log.warn("Unable to start input port [" + port.getId() + "] with name: [" + port.getName() + "] within process group: [" + port.getParentGroupId() + "]. " + validationErrors);
                            }
                        } else {
                            startInputPort(entity.getParentGroupId(), port.getId());
                        }
                    } catch (NifiClientRuntimeException e) {
                        log.error("Error starting Input Port {} for process group {}", port.getName(), entity.getName());
                    }
                } else if (port.getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name())) {
                    try {
                        if (port.getState().equalsIgnoreCase(NifiProcessUtil.PROCESS_STATE.DISABLED.name())) {
                            PortDTO stoppedPort = stopOutputPort(entity.getParentGroupId(), port.getId());
                            if (stoppedPort.getValidationErrors() != null && !stoppedPort.getValidationErrors().isEmpty()) {
                                startOutputPort(entity.getParentGroupId(), port.getId());
                            } else {
                                String validationErrors = stoppedPort.getValidationErrors() != null ? ("Validation Errors: " + stoppedPort.getValidationErrors().stream().collect(Collectors.joining(","))) : "";
                                log.warn("Unable to start output port [" + port.getId() + "] with name: [" + port.getName() + "] within process group: [" + port.getParentGroupId() + "]. " + validationErrors);
                            }
                        } else {
                            startOutputPort(entity.getParentGroupId(), port.getId());
                        }
                    } catch (NifiClientRuntimeException e) {
                        log.error("Error starting Output Port {} for process group {}", port.getName(), entity.getName());
                    }
                }
            }
        }
    }
}

Example 22 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class AlignProcessGroupComponents method createLayoutGroups.

/**
 * Group the items together to create the various LayoutGroups needed for different Rendering
 */
private void createLayoutGroups() {
    Map<String, Set<ProcessGroupDTO>> outputPortIdToGroup = new HashMap<String, Set<ProcessGroupDTO>>();
    groupIdToOutputPorts = new HashMap<>();
    Map<String, Set<ProcessGroupDTO>> inputPortIdToGroup = new HashMap<String, Set<ProcessGroupDTO>>();
    groupIdToInputPorts = new HashMap<>();
    Map<String, Set<String>> groupIdToGroup = new HashMap<>();
    List<ProcessGroupDTO> connectedGroups = new LinkedList<>();
    parentProcessGroup.getContents().getConnections().stream().filter(connectionDTO -> (isOutputPortToGroupConnection(connectionDTO) || isGroupToGroupConnection(connectionDTO) || isInputPortToGroupConnection(connectionDTO))).forEach(connectionDTO -> {
        PortDTO outputPort = outputPortMap.get(connectionDTO.getDestination().getId()) == null ? outputPortMap.get(connectionDTO.getSource().getId()) : outputPortMap.get(connectionDTO.getDestination().getId());
        PortDTO inputPort = inputPortMap.get(connectionDTO.getSource().getId()) == null ? inputPortMap.get(connectionDTO.getDestination().getId()) : inputPortMap.get(connectionDTO.getSource().getId());
        ProcessGroupDTO destinationGroup = processGroupDTOMap.get(connectionDTO.getDestination().getGroupId());
        ProcessGroupDTO sourceGroup = processGroupDTOMap.get(connectionDTO.getSource().getGroupId());
        if (outputPort != null) {
            ProcessGroupDTO processGroup = destinationGroup == null ? sourceGroup : destinationGroup;
            outputPortIdToGroup.computeIfAbsent(outputPort.getId(), (key) -> new HashSet<ProcessGroupDTO>()).add(processGroup);
            groupIdToOutputPorts.computeIfAbsent(processGroup.getId(), (key) -> new HashSet<PortDTO>()).add(outputPort);
            if (processGroupWithConnectionsMap.containsKey(processGroup.getId())) {
                processGroupWithConnectionsMap.get(processGroup.getId()).addConnection(connectionDTO).addPort(outputPort);
            }
        }
        if (inputPort != null) {
            ProcessGroupDTO processGroup = destinationGroup == null ? sourceGroup : destinationGroup;
            inputPortIdToGroup.computeIfAbsent(inputPort.getId(), (key) -> new HashSet<ProcessGroupDTO>()).add(processGroup);
            groupIdToInputPorts.computeIfAbsent(processGroup.getId(), (key) -> new HashSet<PortDTO>()).add(inputPort);
            if (processGroupWithConnectionsMap.containsKey(processGroup.getId())) {
                processGroupWithConnectionsMap.get(processGroup.getId()).addConnection(connectionDTO).addPort(outputPort);
            }
        } else if (destinationGroup != null && sourceGroup != null) {
            groupIdToGroup.computeIfAbsent(sourceGroup.getId(), (key) -> new HashSet<String>()).add(destinationGroup.getId());
        }
    });
    // group port connections together
    groupIdToOutputPorts.entrySet().stream().forEach(entry -> {
        String processGroupId = entry.getKey();
        String portKey = entry.getValue().stream().map(portDTO -> portDTO.getId()).sorted().collect(Collectors.joining(","));
        portKey = "AAA" + portKey;
        layoutGroups.computeIfAbsent(portKey, (key) -> new ProcessGroupToOutputPort(entry.getValue())).add(processGroupDTOMap.get(processGroupId));
    });
    // group port connections together
    groupIdToInputPorts.entrySet().stream().forEach(entry -> {
        String processGroupId = entry.getKey();
        String portKey = entry.getValue().stream().map(portDTO -> portDTO.getId()).sorted().collect(Collectors.joining(","));
        portKey = "BBB" + portKey;
        layoutGroups.computeIfAbsent(portKey, (key) -> new InputPortToProcessGroup(entry.getValue())).add(processGroupDTOMap.get(processGroupId));
    });
    groupIdToGroup.entrySet().stream().forEach(entry -> {
        String sourceGroupId = entry.getKey();
        String processGroupKey = entry.getValue().stream().sorted().collect(Collectors.joining(","));
        processGroupKey = "CCC" + processGroupKey;
        layoutGroups.computeIfAbsent(processGroupKey, (key) -> new ProcessGroupToProcessGroup(entry.getValue())).add(processGroupDTOMap.get(entry.getKey()));
    });
    // add in any groups that dont have connections to ports
    processGroupDTOMap.values().stream().filter(processGroupDTO -> !groupIdToGroup.values().stream().flatMap(set -> set.stream()).collect(Collectors.toSet()).contains(processGroupDTO.getId()) && !groupIdToInputPorts.containsKey(processGroupDTO.getId()) && !groupIdToOutputPorts.containsKey(processGroupDTO.getId()) && !groupIdToGroup.containsKey(processGroupDTO.getId())).forEach(group -> {
        layoutGroups.computeIfAbsent("NO_PORTS", (key) -> new ProcessGroupWithoutConnections()).add(group);
    });
    // identify the sequence of processgroups if they are connected to each other
    List<String> startingProcessorIds = groupIdToGroup.keySet().stream().filter(id -> !groupIdToGroup.values().stream().anyMatch(ids -> ids.contains(id))).collect(Collectors.toList());
    connectedProcessGroups = new ArrayList<>();
    // start with these and attempt to create flows
    startingProcessorIds.forEach(id -> {
        LayoutOrder layoutOrder = new LayoutOrder(0, 0, processGroupDTOMap.get(id), new LinkedList<>());
        addPorts(layoutOrder, id);
        connectedProcessGroups.add(layoutOrder);
        buildLayoutOrder(layoutOrder, processGroupDTOMap.get(id), groupIdToGroup);
        ConnectedProcessGroup connectedProcessGroup = new ConnectedProcessGroup(layoutOrder);
        connectedProcessGroupsLayouts.add(connectedProcessGroup);
    });
}

Example 23 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class NifiConnectionOrderVisitor method searchConnectionMatchingSource.

private ConnectionDTO searchConnectionMatchingSource(String parentGroupId, String destinationId) {
    // search up to find the connection that matches this dest id
    try {
        ProcessGroupDTO parent = null;
        try {
            log.debug("fetch ProcessGroup for searchConnectionMatchingSource {} ", parentGroupId);
            parent = getGroup(parentGroupId);
        } catch (NifiComponentNotFoundException e) {
            log.debug("Exception searching Connection matching the source. Parent Group ID: " + parentGroupId + ", and destinationId of  " + destinationId);
        }
        if (parent != null) {
            // processGroup.getDto().setParent(parentParent.getProcessGroup());
            // get Contents of this parent
            NifiVisitableProcessGroup visitableProcessGroup = new NifiVisitableProcessGroup(parent);
            ConnectionDTO conn = visitableProcessGroup.getConnectionMatchingSourceId(destinationId);
            if (conn != null) {
                return conn;
            }
            if (conn == null && parent.getParentGroupId() != null) {
                return searchConnectionMatchingSource(parent.getParentGroupId(), destinationId);
            }
        }
    } catch (Exception e) {
        log.error("Exception searching Connection matching the source.  Parent Group ID: " + parentGroupId + ", and destinationId of  " + destinationId);
    }
    return null;
}

Example 24 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class NifiVisitableProcessGroup method accept.

/**
 * Visit this process group using the supplied visitor
 *
 * @param nifiVisitor the visitor
 */
@Override
public void accept(NifiFlowVisitor nifiVisitor) {
    if (dto.getContents() != null) {
        if (dto.getContents().getProcessors() != null) {
            for (ProcessorDTO processorDTO : dto.getContents().getProcessors()) {
                NifiVisitableProcessor processor = getOrCreateProcessor(nifiVisitor, processorDTO);
                nifiVisitor.visitProcessor(processor);
            }
        }
        if (dto.getContents().getProcessGroups() != null) {
            for (ProcessGroupDTO processGroupDTO : dto.getContents().getProcessGroups()) {
                if (processGroupDTO != null) {
                    nifiVisitor.visitProcessGroup(getOrCreateProcessGroup(nifiVisitor, processGroupDTO));
                }
            }
        }
        if (dto.getContents().getConnections() != null) {
            for (ConnectionDTO connectionDTO : dto.getContents().getConnections()) {
                nifiVisitor.visitConnection(new NifiVisitableConnection(this, connectionDTO));
            }
        }
        populateStartingAndEndingProcessors();
    }
}

Example 25 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class NiFiConnectionsRestClientV1 method update.

@Override
public Optional<ConnectionDTO> update(@Nonnull ConnectionDTO connectionDTO) {
    Optional<ConnectionEntity> current = findEntityById(connectionDTO.getId());
    if (current.isPresent()) {
        ConnectionEntity connectionEntity = new ConnectionEntity();
        ConnectionDTO updateDto = new ConnectionDTO();
        connectionEntity.setComponent(updateDto);
        updateDto.setId(connectionDTO.getId());
        updateDto.setSelectedRelationships(connectionDTO.getSelectedRelationships());
        if (connectionDTO.getSource() != null) {
            updateDto.setSource(new ConnectableDTO());
            updateDto.getSource().setGroupId(connectionDTO.getSource().getGroupId());
            updateDto.getSource().setId(connectionDTO.getSource().getId());
            updateDto.getSource().setType(connectionDTO.getSource().getType());
        }
        if (connectionDTO.getDestination() != null) {
            updateDto.setDestination(new ConnectableDTO());
            updateDto.getDestination().setGroupId(connectionDTO.getDestination().getGroupId());
            updateDto.getDestination().setId(connectionDTO.getDestination().getId());
            updateDto.getDestination().setType(connectionDTO.getDestination().getType());
        }
        final RevisionDTO revision = new RevisionDTO();
        revision.setVersion(current.get().getRevision().getVersion());
        connectionEntity.setRevision(revision);
        try {
            return Optional.of(client.put(CONNECTION_PATH + connectionDTO.getId(), connectionEntity, ConnectionEntity.class).getComponent());
        } catch (Exception e) {
            log.error("Error updating connection entry info for connection: {} {} ", connectionDTO, e.getMessage());
            return Optional.empty();
        }
    }
    return Optional.empty();
}