Examples with ProcessorDTO org.apache.nifi.web.api.dto.ProcessorDTO used on opensource projects

Example 41 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project nifi-minifi by apache.

the class FlowSnippetDTOEnricher method enrich.

public void enrich(FlowSnippetDTO flowSnippetDTO, final String encodingVersion) {
    List<FlowSnippetDTO> allFlowSnippets = getAllFlowSnippets(flowSnippetDTO);
    Set<RemoteProcessGroupDTO> remoteProcessGroups = getAll(allFlowSnippets, FlowSnippetDTO::getRemoteProcessGroups).collect(Collectors.toSet());
    Map<String, String> connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName));
    Map<String, String> rpgIdToTargetIdMap = new HashMap<>();
    for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) {
        final RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents();
        final Set<RemoteProcessGroupPortDTO> rpgInputPortDtos = nullToEmpty(contents.getInputPorts());
        final Set<RemoteProcessGroupPortDTO> rpgOutputPortDtos = nullToEmpty(contents.getOutputPorts());
        switch(encodingVersion) {
            case "1.2":
                // Map all port DTOs to their respective targetIds
                rpgIdToTargetIdMap.putAll(Stream.concat(rpgInputPortDtos.stream(), rpgOutputPortDtos.stream()).collect(Collectors.toMap(RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getTargetId)));
                break;
            default:
                break;
        }
        addConnectables(connectableNameMap, rpgInputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
        addConnectables(connectableNameMap, rpgOutputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
    }
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getInputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getOutputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    final Set<ConnectionDTO> connections = getAll(allFlowSnippets, FlowSnippetDTO::getConnections).collect(Collectors.toSet());
    // Enrich connection endpoints using known names and overriding with targetIds for remote ports
    for (ConnectionDTO connection : connections) {
        setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap);
        setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap);
    }
    // Override any ids that are for Remote Ports to use their target Ids where available
    connections.stream().flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())).filter(connectable -> connectable.getType().equals(ConnectableType.REMOTE_OUTPUT_PORT.toString()) || connectable.getType().equals(ConnectableType.REMOTE_INPUT_PORT.toString())).forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId())));
    // Establish unique names for connections
    for (ConnectionDTO connection : connections) {
        if (StringUtil.isNullOrEmpty(connection.getName())) {
            StringBuilder name = new StringBuilder();
            ConnectableDTO connectionSource = connection.getSource();
            name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap));
            name.append("/");
            if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) {
                name.append(connection.getSelectedRelationships().iterator().next());
            }
            name.append("/");
            ConnectableDTO connectionDestination = connection.getDestination();
            name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap));
            connection.setName(name.toString());
        }
    }
    nullToEmpty(flowSnippetDTO.getProcessGroups()).stream().map(ProcessGroupDTO::getContents).forEach(snippetDTO -> enrich(snippetDTO, encodingVersion));
}

Example 42 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class NifiConnectionOrderVisitor method getConnectionProcessor.

private NifiVisitableProcessor getConnectionProcessor(String groupId, String id) {
    NifiVisitableProcessor processor = visitedProcessors.get(id);
    if (processor == null) {
        if (!this.processorsMap.containsKey(id)) {
            // if the current group is not related to this processgroup then attempt to walk this processors processgroup
            try {
                log.debug("fetch ProcessGroup for getConnectionProcessor {} ", groupId);
                ProcessGroupDTO processGroupEntity = getGroup(groupId);
                if (processGroupEntity != null) {
                    ProcessorDTO processorDTO = NifiProcessUtil.findFirstProcessorsById(processGroupEntity.getContents().getProcessors(), id);
                    if (processorDTO != null) {
                        this.processorsMap.put(id, processorDTO);
                    }
                    if (processGroup.getDto().getId() != groupId && !visitedProcessGroups.containsKey(processGroupEntity.getId())) {
                        visitProcessGroup(new NifiVisitableProcessGroup(processGroupEntity));
                    }
                } else {
                    log.error("getConnectionProcessor error.  Unable to find Process Group for process group id: {}, and processor: {} {} ", groupId, id);
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        // 
        processor = visitedProcessors.get(id);
        if (processor == null) {
            processor = new NifiVisitableProcessor(this.processorsMap.get(id));
            // visit the group?
            processor.accept(this);
        }
    }
    return processor;
}

Example 43 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class TemplateCreationHelperTest method updateControllerServiceReferencesWithEnabled.

/**
 * Verify preferring enabled controller services over disabled controller services when updating processor properties.
 */
@Test
public void updateControllerServiceReferencesWithEnabled() {
    final AtomicReference<NifiProperty> updateProperty = new AtomicReference<>();
    // Mock NiFi client
    final LegacyNifiRestClient restClient = Mockito.mock(LegacyNifiRestClient.class);
    Mockito.when(restClient.getPropertyDescriptorTransform()).thenReturn(new MockNiFiPropertyDescriptorTransform());
    Mockito.doAnswer(invocation -> {
        updateProperty.set(invocation.getArgumentAt(2, NifiProperty.class));
        return null;
    }).when(restClient).updateProcessorProperty(Mockito.isNull(String.class), Mockito.eq("P1"), Mockito.any());
    final ControllerServiceDTO service1 = new ControllerServiceDTO();
    service1.setId("S1");
    service1.setName("Service1");
    service1.setProperties(Collections.emptyMap());
    service1.setState("DISABLED");
    final ControllerServiceDTO service2 = new ControllerServiceDTO();
    service2.setId("S2");
    service2.setName("Service2");
    service2.setProperties(Collections.emptyMap());
    service2.setState("ENABLED");
    Mockito.when(restClient.getControllerServices()).thenReturn(ImmutableSet.of(service1, service2));
    // Mock processors
    final ProcessorConfigDTO config = new ProcessorConfigDTO();
    config.setDescriptors(Collections.singletonMap("service", newPropertyDescriptor("service", "com.example.Service", "S1", "S2")));
    config.setProperties(Collections.singletonMap("service", "invalid"));
    final ProcessorDTO processor = new ProcessorDTO();
    processor.setId("P1");
    processor.setName("Processor1");
    processor.setConfig(config);
    // Update processors
    final TemplateCreationHelper helper = new TemplateCreationHelper(restClient);
    helper.snapshotControllerServiceReferences();
    helper.identifyNewlyCreatedControllerServiceReferences();
    helper.updateControllerServiceReferences(Collections.singletonList(processor));
    // Verify new processor properties
    Assert.assertNotNull("Property 'Service' not set on processor 'Processor1'.", updateProperty.get());
    Assert.assertEquals("S2", updateProperty.get().getValue());
}

Example 44 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class NifiProcessUtil method getProcessors.

/**
 * Return a set of processors in a template, optionally allowing the framework to traverse into a templates input ports to get the connecting process groups
 *
 * @param template      a template to parse
 * @param excludeInputs {@code true} will traverse down into the input ports and gather processors in the conencting groups, {@code false} will traverse input ports and their respective process
 *                      groups
 * @return return a set of processors
 */
public static Set<ProcessorDTO> getProcessors(TemplateDTO template, boolean excludeInputs) {
    Set<ProcessorDTO> processors = new HashSet<>();
    for (ProcessorDTO processorDTO : template.getSnippet().getProcessors()) {
        processors.add(processorDTO);
    }
    if (template.getSnippet().getProcessGroups() != null) {
        for (ProcessGroupDTO groupDTO : template.getSnippet().getProcessGroups()) {
            processors.addAll(getProcessors(groupDTO));
        }
    }
    if (excludeInputs) {
        final List<ProcessorDTO> inputs = NifiTemplateUtil.getInputProcessorsForTemplate(template);
        Iterables.removeIf(processors, new Predicate<ProcessorDTO>() {

            @Override
            public boolean apply(ProcessorDTO processorDTO) {
                return (inputs.contains(processorDTO));
            }
        });
    }
    return processors;
}

Example 45 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class TestFlowBuilder method processorDTO.

private ProcessorDTO processorDTO() {
    ProcessorDTO processorDTO = new ProcessorDTO();
    processorDTO.setName("Processor " + processorCounter.incrementAndGet());
    processorDTO.setId(UUID.randomUUID().toString());
    return processorDTO;
}