Examples with ControllerServiceReferencingComponentEntity org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity used on opensource projects

Example 1 with ControllerServiceReferencingComponentEntity

use of org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity in project nifi by apache.

the class ControllerServiceEntityMerger method mergeControllerServiceReferencingComponent.

private static void mergeControllerServiceReferencingComponent(ControllerServiceReferencingComponentEntity clientEntity, Map<NodeIdentifier, ControllerServiceReferencingComponentEntity> nodeEntities) {
    final Map<String, Map<NodeIdentifier, PropertyDescriptorDTO>> propertyDescriptorMap = new HashMap<>();
    final Map<NodeIdentifier, Set<ControllerServiceReferencingComponentEntity>> nodeReferencingComponentsMap = new HashMap<>();
    // aggregate the property descriptors
    for (Map.Entry<NodeIdentifier, ControllerServiceReferencingComponentEntity> entry : nodeEntities.entrySet()) {
        final NodeIdentifier nodeIdentifier = entry.getKey();
        final ControllerServiceReferencingComponentEntity nodeEntity = entry.getValue();
        nodeEntity.getComponent().getDescriptors().values().stream().forEach(propertyDescriptor -> {
            propertyDescriptorMap.computeIfAbsent(propertyDescriptor.getName(), nodeIdToPropertyDescriptor -> new HashMap<>()).put(nodeIdentifier, propertyDescriptor);
        });
        nodeReferencingComponentsMap.put(nodeIdentifier, nodeEntity.getComponent().getReferencingComponents());
    }
    // merge property descriptors
    for (Map<NodeIdentifier, PropertyDescriptorDTO> propertyDescriptorByNodeId : propertyDescriptorMap.values()) {
        final Collection<PropertyDescriptorDTO> nodePropertyDescriptors = propertyDescriptorByNodeId.values();
        if (!nodePropertyDescriptors.isEmpty()) {
            // get the name of the property descriptor and find that descriptor being returned to the client
            final PropertyDescriptorDTO propertyDescriptor = nodePropertyDescriptors.iterator().next();
            final PropertyDescriptorDTO clientPropertyDescriptor = clientEntity.getComponent().getDescriptors().get(propertyDescriptor.getName());
            PropertyDescriptorDtoMerger.merge(clientPropertyDescriptor, propertyDescriptorByNodeId);
        }
    }
    final Set<ControllerServiceReferencingComponentEntity> referencingComponents = clientEntity.getComponent().getReferencingComponents();
    mergeControllerServiceReferences(referencingComponents, nodeReferencingComponentsMap);
}

Example 2 with ControllerServiceReferencingComponentEntity

use of org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity in project nifi by apache.

the class ControllerServiceEntityMerger method mergeControllerServiceReferences.

public static void mergeControllerServiceReferences(final Set<ControllerServiceReferencingComponentEntity> referencingComponents, final Map<NodeIdentifier, Set<ControllerServiceReferencingComponentEntity>> referencingComponentMap) {
    final Map<String, Integer> activeThreadCounts = new HashMap<>();
    final Map<String, String> states = new HashMap<>();
    final Map<String, PermissionsDTO> canReads = new HashMap<>();
    for (final Map.Entry<NodeIdentifier, Set<ControllerServiceReferencingComponentEntity>> nodeEntry : referencingComponentMap.entrySet()) {
        final Set<ControllerServiceReferencingComponentEntity> nodeReferencingComponents = nodeEntry.getValue();
        // go through all the nodes referencing components
        if (nodeReferencingComponents != null) {
            for (final ControllerServiceReferencingComponentEntity nodeReferencingComponentEntity : nodeReferencingComponents) {
                final ControllerServiceReferencingComponentDTO nodeReferencingComponent = nodeReferencingComponentEntity.getComponent();
                if (nodeReferencingComponentEntity.getPermissions().getCanRead()) {
                    // handle active thread counts
                    if (nodeReferencingComponent.getActiveThreadCount() != null && nodeReferencingComponent.getActiveThreadCount() > 0) {
                        final Integer current = activeThreadCounts.get(nodeReferencingComponent.getId());
                        if (current == null) {
                            activeThreadCounts.put(nodeReferencingComponent.getId(), nodeReferencingComponent.getActiveThreadCount());
                        } else {
                            activeThreadCounts.put(nodeReferencingComponent.getId(), nodeReferencingComponent.getActiveThreadCount() + current);
                        }
                    }
                    // handle controller service state
                    final String state = states.get(nodeReferencingComponent.getId());
                    if (state == null) {
                        if (ControllerServiceState.DISABLING.name().equals(nodeReferencingComponent.getState())) {
                            states.put(nodeReferencingComponent.getId(), ControllerServiceState.DISABLING.name());
                        } else if (ControllerServiceState.ENABLING.name().equals(nodeReferencingComponent.getState())) {
                            states.put(nodeReferencingComponent.getId(), ControllerServiceState.ENABLING.name());
                        }
                    }
                }
                // handle read permissions
                final PermissionsDTO mergedPermissions = canReads.get(nodeReferencingComponentEntity.getId());
                final PermissionsDTO permissions = nodeReferencingComponentEntity.getPermissions();
                if (permissions != null) {
                    if (mergedPermissions == null) {
                        canReads.put(nodeReferencingComponentEntity.getId(), permissions);
                    } else {
                        PermissionsDtoMerger.mergePermissions(mergedPermissions, permissions);
                    }
                }
            }
        }
    }
    // go through each referencing components
    if (referencingComponents != null) {
        for (final ControllerServiceReferencingComponentEntity referencingComponent : referencingComponents) {
            final PermissionsDTO permissions = canReads.get(referencingComponent.getId());
            if (permissions != null && permissions.getCanRead() != null && permissions.getCanRead()) {
                final Integer activeThreadCount = activeThreadCounts.get(referencingComponent.getId());
                if (activeThreadCount != null) {
                    referencingComponent.getComponent().setActiveThreadCount(activeThreadCount);
                }
                final String state = states.get(referencingComponent.getId());
                if (state != null) {
                    referencingComponent.getComponent().setState(state);
                }
                final Map<NodeIdentifier, ControllerServiceReferencingComponentEntity> nodeEntities = new HashMap<>();
                referencingComponentMap.entrySet().forEach(entry -> {
                    final NodeIdentifier nodeIdentifier = entry.getKey();
                    final Set<ControllerServiceReferencingComponentEntity> nodeControllerServiceReferencingComponents = entry.getValue();
                    nodeControllerServiceReferencingComponents.forEach(nodeControllerServiceReferencingComponent -> {
                        if (referencingComponent.getId() != null && referencingComponent.getId().equals(nodeControllerServiceReferencingComponent.getId())) {
                            nodeEntities.put(nodeIdentifier, nodeControllerServiceReferencingComponent);
                        }
                    });
                });
                mergeControllerServiceReferencingComponent(referencingComponent, nodeEntities);
            } else {
                referencingComponent.setPermissions(permissions);
                referencingComponent.setComponent(null);
            }
        }
    }
}

Example 3 with ControllerServiceReferencingComponentEntity

use of org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity in project kylo by Teradata.

the class AbstractNiFiControllerServicesRestClient method updateReferencingServices.

/**
 * Updates the referencing services with the specified state.
 *
 * @param controllerService    controller service
 * @param referencingRevisions revisions of referencing services
 * @param enabled              {@code true} to enable services, or {@code false} to disable
 * @return {@code true} if all services were updated successfully
 */
private boolean updateReferencingServices(@Nonnull final ControllerServiceDTO controllerService, @Nonnull final Map<String, RevisionDTO> referencingRevisions, final boolean enabled) {
    // Issue request to update referencing components
    final UpdateControllerServiceReferenceRequestEntity referenceEntity = new UpdateControllerServiceReferenceRequestEntity();
    referenceEntity.setId(controllerService.getId());
    referenceEntity.setState(enabled ? State.ENABLED.name() : State.DISABLED.name());
    referenceEntity.setReferencingComponentRevisions(referencingRevisions);
    try {
        updateReferences(controllerService.getId(), referenceEntity);
    } catch (final Exception e) {
        log.error("Failed to update referencing services for controller service {} [{}]: {}", controllerService.getId(), controllerService.getName(), e, e);
        return false;
    }
    // Wait for states to change
    final Predicate<ControllerServiceReferencingComponentEntity> predicate;
    if (enabled) {
        predicate = entity -> StringUtils.equalsAny(entity.getComponent().getState(), "ENABLING", State.ENABLED.name());
    } else {
        predicate = entity -> StringUtils.equals(entity.getComponent().getState(), State.DISABLED.name());
    }
    final Predicate<String> typeFilter = REFERENCE_TYPE_CONTROLLER_SERVICE::equals;
    final Set<ControllerServiceReferencingComponentEntity> failed = pollReferencingComponents(controllerService.getId(), typeFilter, predicate, 5, 500, TimeUnit.MILLISECONDS);
    if (failed.isEmpty()) {
        return true;
    } else {
        if (log.isErrorEnabled()) {
            final String referenceIds = failed.stream().map(ControllerServiceReferencingComponentEntity::getId).collect(Collectors.joining(", "));
            log.error("Failed to stop referencing services for controller service {} [{}]: {}", controllerService.getId(), controllerService.getName(), referenceIds);
        }
        return false;
    }
}

Example 4 with ControllerServiceReferencingComponentEntity

use of org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity in project kylo by Teradata.

the class AbstractNiFiControllerServicesRestClient method updateServiceAndReferencingComponents.

/**
 * Updates a controller service.
 *
 * <p>This will first stop all referencing components, disable the service, update the service, enable the service, then reset the state of the components back to their prior state.</p>
 *
 * @param controllerService the service to update with the updated properties
 * @return the updated service
 */
@Nonnull
@Override
public ControllerServiceDTO updateServiceAndReferencingComponents(@Nonnull final ControllerServiceDTO controllerService) {
    // Recursively get all references to this controller service. This will include processors, other controller services, and reporting tasks.
    final Optional<ControllerServiceReferencingComponentsEntity> referencesEntity = getReferences(controllerService.getId());
    final Set<ControllerServiceReferencingComponentEntity> references = referencesEntity.isPresent() ? flattenReferencingComponents(referencesEntity.get()).collect(Collectors.toSet()) : Collections.emptySet();
    // build the reference state and revision maps prior to making this update
    final Map<String, RevisionDTO> referencingSchedulableComponentRevisions = new HashMap<>();
    final Map<String, RevisionDTO> referencingServiceRevisions = new HashMap<>();
    references.forEach(reference -> {
        if (REFERENCE_TYPE_CONTROLLER_SERVICE.equals(reference.getComponent().getReferenceType())) {
            referencingServiceRevisions.put(reference.getId(), reference.getRevision());
        } else {
            referencingSchedulableComponentRevisions.put(reference.getId(), reference.getRevision());
        }
    });
    // Update service and referencing components
    ControllerServiceDTO updatedService = null;
    Exception updateException = null;
    try {
        // Stop the referencing processors and ensure they are in the stopped state
        log.info("Stopping all component references to controller service {} ", controllerService.getName());
        if (!referencingSchedulableComponentRevisions.isEmpty() && !updateReferencingSchedulableComponents(controllerService, referencingSchedulableComponentRevisions, false)) {
            // error unable to change the state of the references. ... error
            throw new NifiClientRuntimeException("Unable to stop processor references to this controller service " + controllerService.getName() + " before making the update");
        }
        // Disable any controller service references
        if (!referencingServiceRevisions.isEmpty() && !updateReferencingServices(controllerService, referencingServiceRevisions, false)) {
            // error unable to change the state of the references. ... error
            throw new NifiClientRuntimeException("Unable to disable other controller service references to this controller service " + controllerService.getName() + " before making the update");
        }
        // Update the service and mark it disabled. This will throw a RuntimeException if it is not successful.
        log.info("Disabling the controller service  {} ", controllerService.getName());
        updateStateByIdWithRetries(controllerService.getId(), State.DISABLED.name(), 5, 500, TimeUnit.MILLISECONDS);
        // Perform the update to this controller service
        log.info("Updating the controller service  {} ", controllerService.getName());
        updatedService = update(controllerService);
        // Enable the service
        updateStateById(controllerService.getId(), State.ENABLED);
    } catch (final Exception e) {
        log.error("Reverting changes after controller service {} [{}] failed to update: ", controllerService.getId(), controllerService.getName(), e);
        updateException = e;
    }
    // Enable any controller service references
    boolean servicesUpdate;
    try {
        log.info("Enabling other controller services referencing this controller service  {} ", controllerService.getName());
        servicesUpdate = updateReferencingServices(controllerService, referencingServiceRevisions, true);
    } catch (final Exception e) {
        log.debug("Failed to restore referencing service states for controller service {} [{}]: {}", controllerService.getId(), controllerService.getName(), e, e);
        servicesUpdate = false;
    }
    // Update references back to previous states
    boolean componentsUpdate;
    try {
        final Set<ControllerServiceReferencingComponentEntity> runningComponents = references.stream().filter(reference -> !REFERENCE_TYPE_CONTROLLER_SERVICE.equals(reference.getComponent().getReferenceType())).filter(reference -> NifiProcessUtil.PROCESS_STATE.RUNNING.name().equals(reference.getComponent().getState())).collect(Collectors.toSet());
        if (runningComponents.size() == referencingSchedulableComponentRevisions.size()) {
            log.info("Updating all component references to be RUNNING for controller service  {} ", controllerService.getName());
            componentsUpdate = updateReferencingSchedulableComponents(controllerService, referencingSchedulableComponentRevisions, true);
        } else {
            log.info("The controller service component references ({} total) had mixed states prior to updating. Going through each processor/component and setting its state back to" + " what it was prior to the update.", referencingSchedulableComponentRevisions.size());
            componentsUpdate = startSchedulableComponents(runningComponents);
            log.info("Successfully updated controller service {} and updated {} components back to their prior state ", controllerService.getName(), runningComponents.size());
        }
    } catch (final Exception e) {
        log.debug("Failed to restore referencing component states for controller service {} [{}]: {}", controllerService.getId(), controllerService.getName(), e, e);
        componentsUpdate = false;
    }
    // Verify final state
    if (updateException != null) {
        throw Throwables.propagate(updateException);
    }
    if (!servicesUpdate) {
        // error unable to change the state of the references. ... error
        throw new NifiClientRuntimeException("Kylo was unable to enable other controller service references to the " + controllerService.getName() + " controller service. Please visit NiFi" + " to reconcile and fix any controller service issues.");
    }
    if (!componentsUpdate) {
        // error unable to change the state of the references. ... error
        throw new NifiClientRuntimeException("Kylo was unable to update the state of the processors as RUNNING. Please visit NiFi to reconcile and fix any controller service issues. " + controllerService.getName());
    }
    log.info("Successfully updated controller service {}", controllerService.getName());
    return updatedService;
}

Example 5 with ControllerServiceReferencingComponentEntity

use of org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity in project kylo by Teradata.

the class AbstractNiFiControllerServicesRestClient method startSchedulableComponents.

private boolean startSchedulableComponents(@Nonnull final Set<ControllerServiceReferencingComponentEntity> entities) {
    boolean updated = true;
    // Update references back to previous states
    for (final ControllerServiceReferencingComponentEntity entity : entities) {
        final ControllerServiceReferencingComponentDTO component = entity.getComponent();
        if (StringUtils.equals(component.getReferenceType(), REFERENCE_TYPE_PROCESSOR)) {
            final ProcessorDTO processorDetails = new ProcessorDTO();
            processorDetails.setId(component.getId());
            processorDetails.setState(NifiProcessUtil.PROCESS_STATE.RUNNING.name());
            final ProcessorEntity processorEntity = new ProcessorEntity();
            processorEntity.setId(entity.getId());
            processorEntity.setRevision(entity.getRevision());
            processorEntity.setComponent(processorDetails);
            try {
                getClient().processors().updateWithRetry(processorEntity, 3, 500, TimeUnit.MILLISECONDS);
            } catch (final Exception e) {
                updated = false;
                log.warn("Failed to start processor {} [{}]: {}", component.getId(), component.getName(), e);
            }
        } else if (StringUtils.equals(component.getReferenceType(), REFERENCE_TYPE_REPORTING_TASK)) {
            final ReportingTaskDTO reportingTaskDetails = new ReportingTaskDTO();
            reportingTaskDetails.setId(component.getId());
            reportingTaskDetails.setState(NifiProcessUtil.PROCESS_STATE.RUNNING.name());
            final ReportingTaskEntity reportingTaskEntity = new ReportingTaskEntity();
            reportingTaskEntity.setId(entity.getId());
            reportingTaskEntity.setRevision(entity.getRevision());
            reportingTaskEntity.setComponent(reportingTaskDetails);
            try {
                getClient().reportingTasks().update(reportingTaskDetails);
            } catch (final Exception e) {
                updated = false;
                log.warn("Failed to start reporting task {} [{}]: {}", component.getId(), component.getName(), e);
            }
        } else {
            log.warn("Cannot start entity {} of unknown type: {}", entity.getId(), component.getReferenceType());
        }
    }
    return updated;
}