Examples with ProcessorDTO org.apache.nifi.web.api.dto.ProcessorDTO used on opensource projects

Example 46 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class TemplateCreationHelper method reassignControllerServiceIds.

private List<ProcessorDTO> reassignControllerServiceIds(List<ProcessorDTO> processors, TemplateInstance instance) {
    Set<ProcessorDTO> updatedProcessors = new HashSet<>();
    if (processors != null) {
        processors.stream().forEach(processorDTO -> {
            Map<String, String> updatedProcessorProperties = new HashMap<>();
            processorDTO.getConfig().getDescriptors().forEach((k, v) -> {
                if (v.getIdentifiesControllerService() != null) {
                    boolean idsMatch = getMergedControllerServices().keySet().stream().anyMatch(id -> id.equalsIgnoreCase(processorDTO.getConfig().getProperties().get(k)));
                    if (!idsMatch && templateProperties != null && !templateProperties.isEmpty()) {
                        NifiProperty matchingProperty = templateProperties.stream().filter(p -> p.getKey().equalsIgnoreCase(k) && p.getProcessorName().equalsIgnoreCase(processorDTO.getName()) && v.getIdentifiesControllerService().equalsIgnoreCase(p.getPropertyDescriptor().getIdentifiesControllerService())).findFirst().orElse(null);
                        if (matchingProperty != null && matchingProperty.getPropertyDescriptor() != null && matchingProperty.getPropertyDescriptor().getAllowableValues() != null) {
                            NiFiAllowableValue matchingValue = matchingProperty.getPropertyDescriptor().getAllowableValues().stream().filter(niFiAllowableValue -> niFiAllowableValue.getValue().equalsIgnoreCase(matchingProperty.getValue())).findFirst().orElse(null);
                            if (matchingValue != null) {
                                String name = matchingValue.getDisplayName();
                                String validControllerServiceId = hasMatchingService(enabledServiceNameMap, name) ? enabledServiceNameMap.get(name).get(0).getId() : hasMatchingService(serviceNameMap, name) ? serviceNameMap.get(name).get(0).getId() : null;
                                if (StringUtils.isNotBlank(validControllerServiceId) && (v.isRequired() || !v.isRequired() && StringUtils.isNotBlank(processorDTO.getConfig().getProperties().get(k)))) {
                                    processorDTO.getConfig().getProperties().put(k, validControllerServiceId);
                                    updatedProcessorProperties.put(k, validControllerServiceId);
                                    if (!updatedProcessors.contains(processorDTO)) {
                                        updatedProcessors.add(processorDTO);
                                    }
                                }
                            }
                        }
                    }
                    // if we havent made a match attempt to see if the cs was removed
                    if (!updatedProcessorProperties.containsKey(k) && !idsMatch && instance != null) {
                        String value = processorDTO.getConfig().getProperties().get(k);
                        // find the correct reference from that was removed due to a matching service
                        ControllerServiceDTO controllerServiceDTO = instance.findMatchingControllerServoce(value);
                        if (controllerServiceDTO != null) {
                            updatedProcessorProperties.put(k, controllerServiceDTO.getId());
                        }
                    }
                }
            });
            if (!updatedProcessorProperties.isEmpty()) {
                ProcessorDTO updatedProcessor = new ProcessorDTO();
                updatedProcessor.setId(processorDTO.getId());
                updatedProcessor.setConfig(new ProcessorConfigDTO());
                updatedProcessor.getConfig().setProperties(updatedProcessorProperties);
                // update the processor
                ProcessorDTO updated = restClient.updateProcessor(updatedProcessor);
                updatedProcessors.add(updated);
            }
        });
    }
    // update the data back in the processors list
    if (!updatedProcessors.isEmpty()) {
        Map<String, ProcessorDTO> updatedMap = updatedProcessors.stream().collect(Collectors.toMap(p -> p.getId(), p -> p));
        return processors.stream().map(p -> updatedMap.containsKey(p.getId()) ? updatedMap.get(p.getId()) : p).collect(Collectors.toList());
    }
    return processors;
}

Example 47 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class TemplateInstanceCreator method createTemplate.

public NifiProcessGroup createTemplate() throws TemplateCreationException {
    try {
        NifiProcessGroup newProcessGroup = null;
        TemplateDTO template = restClient.getTemplateById(templateId);
        VersionedProcessGroup versionedProcessGroup = null;
        if (template != null) {
            TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
            templateCreationHelper.setTemplateProperties(templateProperties);
            String processGroupId = null;
            ProcessGroupDTO group = null;
            if (isCreateReusableFlow()) {
                log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
                // 1 get/create the parent "reusable_templates" processgroup
                ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                if (reusableParentGroup == null) {
                    reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                }
                ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
                if (thisGroup != null) {
                    // version the group
                    log.info("A previous Process group of with this name {} was found.  Versioning it before continuing", thisGroup.getName());
                    versionedProcessGroup = templateCreationHelper.versionProcessGroup(thisGroup, this.getVersionIdentifier());
                }
                group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
            } else {
                String tmpName = template.getName() + "_" + System.currentTimeMillis();
                group = restClient.createProcessGroup(tmpName);
                log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
            }
            processGroupId = group.getId();
            if (StringUtils.isNotBlank(processGroupId)) {
                // snapshot the existing controller services
                templateCreationHelper.snapshotControllerServiceReferences();
                log.info("Successfully Snapshot of controller services");
                // create the flow from the template
                TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
                FlowSnippetDTO flowSnippetDTO = instance.getFlowSnippetDTO();
                log.info("Successfully created the temp flow");
                if (this.createReusableFlow) {
                    ensureInputPortsForReuseableTemplate(processGroupId);
                    log.info("Reusable flow, input ports created successfully.");
                }
                // mark the new services that were created as a result of creating the new flow from the template
                templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
                ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
                // replace static properties and inject values into the flow
                List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
                if (processorProperties != null) {
                    boolean didReplace = false;
                    for (NifiProperty property : processorProperties) {
                        boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
                        if (replaced) {
                            // update the properties that are replaced
                            if (property.getPropertyDescriptor() != null && StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())) {
                                // verify the property is a valid cs property
                                String value = property.getValue();
                                if (templateCreationHelper.getEnabledServiceNameMap().containsKey(value)) {
                                    property.setValue(templateCreationHelper.getEnabledServiceNameMap().get(value).get(0).getId());
                                }
                            }
                            restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
                            didReplace = true;
                        }
                    }
                    // if we replaced any properties, refetch to get the latest data
                    if (didReplace) {
                        entity = restClient.getProcessGroup(processGroupId, true, true);
                    }
                }
                // identify the various processors (first level initial processors)
                List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
                ProcessorDTO input = null;
                List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                // if the input is null attempt to get the first input available on the template
                if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
                    input = inputProcessors.get(0);
                }
                log.info("Attempt to update/validate controller services for template.");
                // update any references to the controller services and try to assign the value to an enabled service if it is not already
                boolean updatedControllerServices = false;
                if (input != null) {
                    log.info("attempt to update controllerservices on {} input processor ", input.getName());
                    List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap, instance);
                    updatedControllerServices = !updatedProperties.isEmpty();
                }
                log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
                List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap, instance);
                if (!updatedControllerServices) {
                    updatedControllerServices = !updatedProperties.isEmpty();
                }
                log.info("Controller service validation complete");
                // refetch processors for updated errors
                entity = restClient.getProcessGroup(processGroupId, true, true);
                nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                inputProcessors = NifiProcessUtil.getInputProcessors(entity);
                if (inputProcessors != null && !inputProcessors.isEmpty()) {
                    input = inputProcessors.get(0);
                }
                // /make the input/output ports in the category group as running
                if (isCreateReusableFlow()) {
                    log.info("Reusable flow, attempt to mark the connection ports as running.");
                    templateCreationHelper.startProcessGroupAndParentInputPorts(entity);
                    // templateCreationHelper.markConnectionPortsAsRunning(entity);
                    log.info("Reusable flow.  Successfully marked the ports as running.");
                }
                newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
                newProcessGroup.setVersionedProcessGroup(versionedProcessGroup);
                newProcessGroup.setReusableFlowInstance(isCreateReusableFlow());
                if (isCreateReusableFlow()) {
                    // call listeners notify of before mark as running  processing
                    if (creationCallback != null) {
                        try {
                            creationCallback.beforeMarkAsRunning(template.getName(), entity);
                        } catch (Exception e) {
                            log.error("Error calling callback beforeMarkAsRunning ", e);
                        }
                    }
                    log.info("Reusable flow, attempt to mark the Processors as running.");
                    templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
                    log.info("Reusable flow.  Successfully marked the Processors as running.");
                    // align items
                    AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
                    alignProcessGroupComponents.autoLayout();
                }
                templateCreationHelper.cleanupControllerServices();
                log.info("Controller service cleanup complete");
                List<NifiError> errors = templateCreationHelper.getErrors();
                // add any global errors to the object
                if (errors != null && !errors.isEmpty()) {
                    for (NifiError error : errors) {
                        newProcessGroup.addError(error);
                    }
                }
                newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
                if (!newProcessGroup.isSuccess()) {
                    log.info("Errors while importing the template. {} errors found. {}", (errors != null ? errors.size() : 0), (errors != null ? " - " + StringUtils.join(errors, ",") : ""));
                } else {
                    log.info("Success.  Finished importing template ");
                }
                return newProcessGroup;
            }
        }
    } catch (final Exception e) {
        if (log.isDebugEnabled() && e instanceof WebApplicationException) {
            final Response response = ((WebApplicationException) e).getResponse();
            log.debug("NiFi server returned error: {}", response.readEntity(String.class), e);
        }
        throw new TemplateCreationException("Unable to create the template for the Id of [" + templateId + "]. " + e.getMessage(), e);
    }
    return null;
}

Example 48 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class AbstractNiFiProcessorsRestClient method genrateWakeSequence.

private List<Function<ProcessorDTO, ProcessorDTO>> genrateWakeSequence(ProcessorDTO originalProc) {
    List<Function<ProcessorDTO, ProcessorDTO>> sequence = new ArrayList<>();
    PROCESS_STATE state = PROCESS_STATE.valueOf(originalProc.getState().toUpperCase());
    SCHEDULE_STRATEGIES strategy = SCHEDULE_STRATEGIES.valueOf(originalProc.getConfig().getSchedulingStrategy().toUpperCase());
    // Stop the processor if it is disabled or running.
    if (state == PROCESS_STATE.DISABLED || state == PROCESS_STATE.RUNNING) {
        sequence.add((proc) -> {
            proc.setState(PROCESS_STATE.STOPPED.name());
            return updateWithRetry(proc, 5, 300, TimeUnit.MILLISECONDS);
        });
    }
    // In all other cases add the sequence: set a long timer -> start -> stop -> reset original scheduling.
    if (state != PROCESS_STATE.RUNNING || strategy != SCHEDULE_STRATEGIES.TIMER_DRIVEN) {
        // Set long timer -> start -> stop -> reset scheduling.
        sequence.add((proc) -> {
            proc.setConfig(createConfig(SCHEDULE_STRATEGIES.TIMER_DRIVEN.name(), TRIGGER_TIMER_PERIOD));
            proc.setState(PROCESS_STATE.RUNNING.name());
            return updateWithRetry(proc, 5, 300, TimeUnit.MILLISECONDS);
        });
        sequence.add((proc) -> {
            delay(2000);
            proc.setState(PROCESS_STATE.STOPPED.name());
            return updateWithRetry(proc, 5, 300, TimeUnit.MILLISECONDS);
        });
    }
    // Disable or start the processor again depending on its original state
    if (state == PROCESS_STATE.DISABLED) {
        sequence.add((proc) -> {
            proc.setConfig(createConfig(originalProc.getConfig().getSchedulingStrategy(), originalProc.getConfig().getSchedulingPeriod()));
            proc.setState(PROCESS_STATE.DISABLED.name());
            return updateWithRetry(proc, 5, 300, TimeUnit.MILLISECONDS);
        });
    } else if (state == PROCESS_STATE.RUNNING) {
        sequence.add((proc) -> {
            proc.setConfig(createConfig(originalProc.getConfig().getSchedulingStrategy(), originalProc.getConfig().getSchedulingPeriod()));
            proc.setState(PROCESS_STATE.RUNNING.name());
            return updateWithRetry(proc, 5, 300, TimeUnit.MILLISECONDS);
        });
    }
    return sequence;
}

Example 49 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class AbstractNiFiProcessorsRestClient method applySchedule.

protected ProcessorDTO applySchedule(NifiProcessorSchedule schedule) {
    if (schedule != null && schedule.getProcessorId() != null) {
        ProcessorDTO input = new ProcessorDTO();
        input.setId(schedule.getProcessorId());
        input.setConfig(new ProcessorConfigDTO());
        input.getConfig().setSchedulingPeriod(schedule.getSchedulingPeriod());
        input.getConfig().setSchedulingStrategy(schedule.getSchedulingStrategy());
        input.getConfig().setConcurrentlySchedulableTaskCount(schedule.getConcurrentTasks());
        return input;
    }
    return null;
}

Example 50 with ProcessorDTO

use of org.apache.nifi.web.api.dto.ProcessorDTO in project kylo by Teradata.

the class LegacyNifiRestClient method disableAllInputProcessors.

/**
 * Disables all inputs for a given process group
 *
 * @param processGroupId
 * @return processorDTO prior to disabling
 * @throws NifiComponentNotFoundException
 */
public List<ProcessorDTO> disableAllInputProcessors(String processGroupId) throws NifiComponentNotFoundException {
    List<ProcessorDTO> processorDTOs = getInputProcessors(processGroupId);
    ProcessorDTO updateDto = new ProcessorDTO();
    if (processorDTOs != null) {
        for (ProcessorDTO dto : processorDTOs) {
            updateDto.setParentGroupId(dto.getParentGroupId());
            updateDto.setId(dto.getId());
            // fetch the processor and update it
            if (NifiProcessUtil.PROCESS_STATE.STOPPED.name().equals(dto.getState())) {
                // if its stopped you can disable it.. otherwise stop it and then disable it
                updateDto.setState(NifiProcessUtil.PROCESS_STATE.DISABLED.name());
                updateProcessorWithRetry(updateDto);
            }
            if (NifiProcessUtil.PROCESS_STATE.RUNNING.name().equals(dto.getState())) {
                updateDto.setState(NifiProcessUtil.PROCESS_STATE.STOPPED.name());
                updateProcessorWithRetry(updateDto);
                updateDto.setState(NifiProcessUtil.PROCESS_STATE.DISABLED.name());
                updateProcessorWithRetry(updateDto);
            }
        }
    }
    return processorDTOs;
}