Examples with ControllerServiceDTO org.apache.nifi.web.api.dto.ControllerServiceDTO used on opensource projects

Example 6 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateCreationHelper method fixControllerServiceReferences.

/**
 * Enables the controller services for the specified properties or changes the property value to an enabled service.
 *
 * @param controllerServiceProperties property overrides for controller services
 * @param enabledServices             map of enabled controller service ids and names to DTOs
 * @param allServices                 map of all controller service ids to DTOs
 * @param properties                  the processor properties to update
 * @return the list of properties that were modified
 */
@Nonnull
private List<NifiProperty> fixControllerServiceReferences(@Nullable final Map<String, String> controllerServiceProperties, @Nonnull final Map<String, ControllerServiceDTO> enabledServices, @Nonnull final Map<String, ControllerServiceDTO> allServices, @Nonnull final List<NifiProperty> properties) {
    return properties.stream().filter(property -> StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())).filter(property -> StringUtils.isNotBlank(property.getValue())).filter(property -> !enabledServices.containsKey(property.getValue())).filter(property -> {
        final Optional<ControllerServiceDTO> controllerService = findControllerServiceForProperty(controllerServiceProperties, enabledServices, allServices, property);
        if (controllerService.isPresent()) {
            if (!controllerService.get().getId().equals(property.getValue())) {
                property.setValue(controllerService.get().getId());
                return true;
            }
        } else if (property.getPropertyDescriptor().isRequired()) {
            final String message = "Unable to find a valid controller service for the '" + property.getKey() + "' property of the '" + property.getProcessorName() + "' " + "processor.";
            errors.add(new NifiError(NifiError.SEVERITY.FATAL, message, NifiProcessGroup.CONTROLLER_SERVICE_CATEGORY));
        }
        return false;
    }).collect(Collectors.toList());
}

Example 7 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateInstance method addDeletedServiceMapping.

public void addDeletedServiceMapping(String deletedServiceId, List<ControllerServiceDTO> rootServices) {
    if (rootServices != null && !rootServices.isEmpty()) {
        ControllerServiceDTO rootService = rootServices.stream().filter(cs -> NifiProcessUtil.SERVICE_STATE.ENABLED.name().equalsIgnoreCase(cs.getState())).findFirst().orElse(rootServices.get(0));
        deletedServicesToMatchingRootService.put(deletedServiceId, rootService);
        if (scopeIdToRootIdMap.containsKey(deletedServiceId)) {
            deletedServicesToMatchingRootService.put(scopeIdToRootIdMap.get(deletedServiceId), rootService);
        }
        if (rootIdToScopeIdMap.containsKey(deletedServiceId)) {
            deletedServicesToMatchingRootService.put(rootIdToScopeIdMap.get(deletedServiceId), rootService);
        }
    }
}

Example 8 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class AbstractNiFiControllerServicesRestClient method updateServiceAndReferencingComponents.

/**
 * Updates a controller service
 * This will first disable the service, stop all referencing components, enable the service, reset the state of the components back to their prior state
 *
 * @param controllerService the service to update with the updated properties
 * @return the updates service
 */
@Override
public ControllerServiceDTO updateServiceAndReferencingComponents(ControllerServiceDTO controllerService) {
    String id = controllerService.getId();
    // Get all references to this controller service.  this will include processors, other controller services, and reporting tasks
    Optional<ControllerServiceReferencingComponentsEntity> references = getReferences(id);
    // state of the processors prior to update
    Map<String, String> previousProcessorState = null;
    // revisions for the processor references
    Map<String, RevisionDTO> processorRevisions = null;
    // revisions for the controller service references
    Map<String, RevisionDTO> controllerServiceRevisions = null;
    // a map of component id to reference entity.  this will include all referencing processors, controller services, and reporting tasks
    Map<String, ControllerServiceReferencingComponentEntity> referencingComponentEntityMap = null;
    // Stop all processor references and also disable any other controller service references
    if (references.isPresent()) {
        // build the reference state and revision maps prior to making this update
        referencingComponentEntityMap = getReferencingComponents(references.get().getControllerServiceReferencingComponents());
        previousProcessorState = referencingComponentEntityMap.values().stream().filter(e -> e.getComponent().getReferenceType().equalsIgnoreCase("PROCESSOR")).map(c -> c.getComponent()).collect(Collectors.toMap(ControllerServiceReferencingComponentDTO::getId, ControllerServiceReferencingComponentDTO::getState));
        processorRevisions = referencingComponentEntityMap.values().stream().filter(e -> e.getComponent().getReferenceType().equalsIgnoreCase("PROCESSOR")).collect(Collectors.toMap(ControllerServiceReferencingComponentEntity::getId, ControllerServiceReferencingComponentEntity::getRevision));
        controllerServiceRevisions = referencingComponentEntityMap.values().stream().filter(e -> e.getComponent().getReferenceType().equalsIgnoreCase("ControllerService")).collect(Collectors.toMap(ControllerServiceReferencingComponentEntity::getId, ControllerServiceReferencingComponentEntity::getRevision));
        // Stop the referencing processors and ensure they are in the stopped state
        if (!processorRevisions.isEmpty()) {
            String state = NifiProcessUtil.PROCESS_STATE.STOPPED.name();
            log.info("Stopping all component references to controller service {} ", controllerService.getName());
            UpdateControllerServiceReferenceRequestEntity stopComponentsRequest = new UpdateControllerServiceReferenceRequestEntity();
            stopComponentsRequest.setState(state);
            stopComponentsRequest.setId(controllerService.getId());
            stopComponentsRequest.setReferencingComponentRevisions(processorRevisions);
            updateReferences(id, stopComponentsRequest);
            boolean updatedReferences = ensureComponentsAreOfState(id, "Processor", state, 5, 500, TimeUnit.MILLISECONDS);
            if (!updatedReferences) {
                // error unable to change the state of the references. ... error
                throw new NifiClientRuntimeException("Unable to stop processor references to this controller service " + controllerService.getName() + " before making the update");
            }
        }
        // disable any controller service references
        if (!controllerServiceRevisions.isEmpty()) {
            UpdateControllerServiceReferenceRequestEntity stopComponentsRequest = new UpdateControllerServiceReferenceRequestEntity();
            stopComponentsRequest.setState("DISABLED");
            stopComponentsRequest.setId(controllerService.getId());
            stopComponentsRequest.setReferencingComponentRevisions(controllerServiceRevisions);
            updateReferences(id, stopComponentsRequest);
            boolean updatedReferences = ensureComponentsAreOfState(id, "ControllerService", "DISABLED", 5, 500, TimeUnit.MILLISECONDS);
            if (!updatedReferences) {
                // error unable to change the state of the references. ... error
                throw new NifiClientRuntimeException("Unable to disable other controller service references to this controller service " + controllerService.getName() + " before making the update");
            }
        }
    }
    // mark this controller service as disabled
    log.info("Disabling the controller service  {} ", controllerService.getName());
    // update the service and mark it disabled.  this will throw a RuntimeException if it is not successful
    ControllerServiceDTO updatedService = updateStateByIdWithRetries(controllerService.getId(), "DISABLED", 5, 500, TimeUnit.MILLISECONDS);
    // Perform the update to this controller service
    log.info("Updating the controller service  {} ", controllerService.getName());
    updatedService = update(controllerService);
    // Enable the service
    updateStateById(controllerService.getId(), NiFiControllerServicesRestClient.State.ENABLED);
    // Enable any controller service references
    if (!controllerServiceRevisions.isEmpty()) {
        log.info("Enabling other controller services referencing this controller service  {} ", controllerService.getName());
        UpdateControllerServiceReferenceRequestEntity enableReferenceServicesRequest = new UpdateControllerServiceReferenceRequestEntity();
        enableReferenceServicesRequest.setId(controllerService.getId());
        enableReferenceServicesRequest.setState(NiFiControllerServicesRestClient.State.ENABLED.name());
        enableReferenceServicesRequest.setReferencingComponentRevisions(controllerServiceRevisions);
        updateReferences(id, enableReferenceServicesRequest);
        boolean updatedReferences = ensureComponentsAreOfState(id, "ControllerService", NiFiControllerServicesRestClient.State.ENABLED.name(), 5, 500, TimeUnit.MILLISECONDS);
        if (!updatedReferences) {
            // error unable to change the state of the references. ... error
            throw new NifiClientRuntimeException("The controller service " + controllerService.getName() + " was updated, but it was unable to enable other controller service references to this controller service.  Please visit NiFi to reconcile and fix any controller service issues. " + controllerService.getName());
        }
    }
    if (references.isPresent() && previousProcessorState != null) {
        // reset the processor state
        Map<String, RevisionDTO> finalComponentRevisions = processorRevisions;
        // if all referencing processors of of the same state we can call the quick rest endpoint to update all of them
        Set<String> distinctStates = previousProcessorState.values().stream().collect(Collectors.toSet());
        if (distinctStates.size() > 0) {
            if (distinctStates.size() == 1) {
                String state = new ArrayList<>(distinctStates).get(0);
                if (!NifiProcessUtil.PROCESS_STATE.STOPPED.name().equals(state)) {
                    UpdateControllerServiceReferenceRequestEntity startComponentsRequest = new UpdateControllerServiceReferenceRequestEntity();
                    startComponentsRequest.setState(state);
                    startComponentsRequest.setReferencingComponentRevisions(finalComponentRevisions);
                    startComponentsRequest.setId(controllerService.getId());
                    updateReferences(id, startComponentsRequest);
                    log.info("Updating all component references to be {} for controller service  {} ", state, controllerService.getName());
                    boolean updatedReferences = ensureComponentsAreOfState(id, "Processor", state, 5, 500, TimeUnit.MILLISECONDS);
                    if (!updatedReferences) {
                        // error unable to change the state of the references. ... error
                        throw new NifiClientRuntimeException("The controller service {} was updated, but it was unable to update the state of the processors as " + state + ".  Please visit NiFi to reconcile and fix any controller service issues. " + controllerService.getName());
                    }
                    log.info("Successfully updated controller service {}", controllerService.getName());
                }
            } else {
                log.info("The controller service component references ({} total) had mixed states prior to updating {}.  Going through each processor/component and setting its state back to what it was prior to the update.", previousProcessorState.size(), distinctStates);
                // update references back to previous states
                List<ProcessorEntity> updatedProcessors = references.get().getControllerServiceReferencingComponents().stream().filter(c -> c.getComponent().getReferenceType().equalsIgnoreCase("PROCESSOR")).filter(c -> !c.getComponent().getState().equals(NifiProcessUtil.PROCESS_STATE.STOPPED.name())).map(referencingComponentEntity -> referencingComponentEntity.getComponent()).map(c -> {
                    ProcessorEntity entity = new ProcessorEntity();
                    entity.setRevision(finalComponentRevisions.get(c.getId()));
                    entity.setId(c.getId());
                    ProcessorDTO processorDTO = new ProcessorDTO();
                    processorDTO.setId(c.getId());
                    processorDTO.setState(c.getState());
                    entity.setComponent(processorDTO);
                    return entity;
                }).collect(Collectors.toList());
                updatedProcessors.stream().forEach(p -> getClient().processors().update(p));
                log.info("Successfully updated controller service {} and updated {} components back to their prior state ", controllerService.getName(), previousProcessorState.size());
            }
        }
    }
    return updatedService;
}

Example 9 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class LegacyNifiRestClient method enableControllerServiceAndSetProperties.

/**
 * Enables the ControllerService and also replaces the properties if they match their keys
 */
public ControllerServiceDTO enableControllerServiceAndSetProperties(String id, Map<String, String> properties) throws NifiClientRuntimeException {
    ControllerServiceDTO entity = getControllerService(null, id);
    ControllerServiceDTO dto = entity;
    // only need to do this if it is not enabled
    if (!dto.getState().equals(NifiProcessUtil.SERVICE_STATE.ENABLED.name())) {
        if (properties != null) {
            boolean changed;
            Map<String, String> resolvedProperties = NifiEnvironmentProperties.getEnvironmentControllerServiceProperties(properties, dto.getName());
            if (resolvedProperties != null && !resolvedProperties.isEmpty()) {
                changed = ConfigurationPropertyReplacer.replaceControllerServiceProperties(dto, resolvedProperties, getPropertyDescriptorTransform());
            } else {
                changed = ConfigurationPropertyReplacer.replaceControllerServiceProperties(dto, properties, getPropertyDescriptorTransform());
            }
            if (changed) {
                // first save the property change
                client.controllerServices().update(dto);
            }
        }
        if (!dto.getState().equals(NifiProcessUtil.SERVICE_STATE.ENABLED.name())) {
            dto.setState(NifiProcessUtil.SERVICE_STATE.ENABLED.name());
            entity = client.controllerServices().update(dto);
        }
    }
    // initially when trying to enable the service the state will be ENABLING
    // This will last until the service is up.
    // if it is in the ENABLING state it needs to wait and try again to get the status.
    dto = entity;
    if (dto.getState().equalsIgnoreCase(NifiProcessUtil.SERVICE_STATE.ENABLING.name())) {
        // attempt to retry x times before returning
        int retryCount = 0;
        int maxRetries = 5;
        while (retryCount <= maxRetries) {
            entity = getControllerService(null, id);
            if (!entity.getState().equals(NifiProcessUtil.SERVICE_STATE.ENABLED.name())) {
                try {
                    Thread.sleep(3000);
                    retryCount++;
                } catch (InterruptedException e2) {
                }
            } else {
                retryCount = maxRetries + 1;
            }
        }
    }
    return entity;
}

Example 10 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class AbstractNiFiControllerServicesRestClientTest method updateStateByIdWithRetries.

/**
 * Verify updating the state of a controller service.
 */
@Test
public void updateStateByIdWithRetries() {
    // Mock controller services
    final ControllerServiceDTO response1 = new ControllerServiceDTO();
    response1.setState("ENABLING");
    final ControllerServiceDTO response2 = new ControllerServiceDTO();
    response2.setState("ENABLED");
    // Mock NiFi Controller Service REST client
    final AbstractNiFiControllerServicesRestClient client = Mockito.mock(AbstractNiFiControllerServicesRestClient.class, Mockito.CALLS_REAL_METHODS);
    Mockito.when(client.update(Mockito.any())).thenReturn(response1);
    Mockito.when(client.findById(Mockito.anyString())).thenReturn(Optional.of(response2));
    // Test updating state
    Assert.assertEquals(response2, client.updateStateByIdWithRetries("MYID", "ENABLED", 1, 0, TimeUnit.NANOSECONDS));
}