Examples with AffectedComponentDTO org.apache.nifi.web.api.dto.AffectedComponentDTO used on opensource projects

Example 1 with AffectedComponentDTO

use of org.apache.nifi.web.api.dto.AffectedComponentDTO in project nifi by apache.

the class VersionsResource method updateFlowVersion.

private VersionControlInformationEntity updateFlowVersion(final String groupId, final ComponentLifecycle componentLifecycle, final URI exampleUri, final Set<AffectedComponentEntity> affectedComponents, final NiFiUser user, final boolean replicateRequest, final Revision revision, final VersionControlInformationEntity requestEntity, final VersionedFlowSnapshot flowSnapshot, final AsynchronousWebRequest<VersionControlInformationEntity> asyncRequest, final String idGenerationSeed, final boolean verifyNotModified, final boolean updateDescendantVersionedFlows) throws LifecycleManagementException, ResumeFlowException {
    // Steps 6-7: Determine which components must be stopped and stop them.
    final Set<String> stoppableReferenceTypes = new HashSet<>();
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_PROCESSOR);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_INPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_OUTPUT_PORT);
    final Set<AffectedComponentEntity> runningComponents = affectedComponents.stream().filter(dto -> stoppableReferenceTypes.contains(dto.getComponent().getReferenceType())).filter(dto -> "Running".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
    logger.info("Stopping {} Processors", runningComponents.size());
    final CancellableTimedPause stopComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    asyncRequest.setCancelCallback(stopComponentsPause::cancel);
    componentLifecycle.scheduleComponents(exampleUri, user, groupId, runningComponents, ScheduledState.STOPPED, stopComponentsPause);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Disabling Affected Controller Services", 20);
    // Steps 8-9. Disable enabled controller services that are affected
    final Set<AffectedComponentEntity> enabledServices = affectedComponents.stream().filter(dto -> AffectedComponentDTO.COMPONENT_TYPE_CONTROLLER_SERVICE.equals(dto.getComponent().getReferenceType())).filter(dto -> "Enabled".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
    logger.info("Disabling {} Controller Services", enabledServices.size());
    final CancellableTimedPause disableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    asyncRequest.setCancelCallback(disableServicesPause::cancel);
    componentLifecycle.activateControllerServices(exampleUri, user, groupId, enabledServices, ControllerServiceState.DISABLED, disableServicesPause);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Updating Flow", 40);
    logger.info("Updating Process Group with ID {} to version {} of the Versioned Flow", groupId, flowSnapshot.getSnapshotMetadata().getVersion());
    // by replicating a PUT to /nifi-api/versions/process-groups/{groupId}
    try {
        if (replicateRequest) {
            final URI updateUri;
            try {
                updateUri = new URI(exampleUri.getScheme(), exampleUri.getUserInfo(), exampleUri.getHost(), exampleUri.getPort(), "/nifi-api/versions/process-groups/" + groupId, null, exampleUri.getFragment());
            } catch (URISyntaxException e) {
                throw new RuntimeException(e);
            }
            final Map<String, String> headers = new HashMap<>();
            headers.put("content-type", MediaType.APPLICATION_JSON);
            final VersionedFlowSnapshotEntity snapshotEntity = new VersionedFlowSnapshotEntity();
            snapshotEntity.setProcessGroupRevision(dtoFactory.createRevisionDTO(revision));
            snapshotEntity.setRegistryId(requestEntity.getVersionControlInformation().getRegistryId());
            snapshotEntity.setVersionedFlow(flowSnapshot);
            snapshotEntity.setUpdateDescendantVersionedFlows(updateDescendantVersionedFlows);
            final NodeResponse clusterResponse;
            try {
                logger.debug("Replicating PUT request to {} for user {}", updateUri, user);
                if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
                    clusterResponse = getRequestReplicator().replicate(user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
                } else {
                    clusterResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
                }
            } catch (final InterruptedException ie) {
                logger.warn("Interrupted while replicating PUT request to {} for user {}", updateUri, user);
                Thread.currentThread().interrupt();
                throw new LifecycleManagementException("Interrupted while updating flows across cluster", ie);
            }
            final int updateFlowStatus = clusterResponse.getStatus();
            if (updateFlowStatus != Status.OK.getStatusCode()) {
                final String explanation = getResponseEntity(clusterResponse, String.class);
                logger.error("Failed to update flow across cluster when replicating PUT request to {} for user {}. Received {} response with explanation: {}", updateUri, user, updateFlowStatus, explanation);
                throw new LifecycleManagementException("Failed to update Flow on all nodes in cluster due to " + explanation);
            }
        } else {
            // Step 10: Ensure that if any connection exists in the flow and does not exist in the proposed snapshot,
            // that it has no data in it. Ensure that no Input Port was removed, unless it currently has no incoming connections.
            // Ensure that no Output Port was removed, unless it currently has no outgoing connections.
            serviceFacade.verifyCanUpdate(groupId, flowSnapshot, true, verifyNotModified);
            // Step 11-12. Update Process Group to the new flow and update variable registry with any Variables that were added or removed
            final VersionControlInformationDTO requestVci = requestEntity.getVersionControlInformation();
            final Bucket bucket = flowSnapshot.getBucket();
            final VersionedFlow flow = flowSnapshot.getFlow();
            final VersionedFlowSnapshotMetadata metadata = flowSnapshot.getSnapshotMetadata();
            final VersionControlInformationDTO vci = new VersionControlInformationDTO();
            vci.setBucketId(metadata.getBucketIdentifier());
            vci.setBucketName(bucket.getName());
            vci.setFlowDescription(flow.getDescription());
            vci.setFlowId(flow.getIdentifier());
            vci.setFlowName(flow.getName());
            vci.setGroupId(groupId);
            vci.setRegistryId(requestVci.getRegistryId());
            vci.setRegistryName(serviceFacade.getFlowRegistryName(requestVci.getRegistryId()));
            vci.setVersion(metadata.getVersion());
            vci.setState(flowSnapshot.isLatest() ? VersionedFlowState.UP_TO_DATE.name() : VersionedFlowState.STALE.name());
            serviceFacade.updateProcessGroupContents(user, revision, groupId, vci, flowSnapshot, idGenerationSeed, verifyNotModified, false, updateDescendantVersionedFlows);
        }
    } finally {
        if (!asyncRequest.isCancelled()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Re-Enabling {} Controller Services: {}", enabledServices.size(), enabledServices);
            }
            asyncRequest.update(new Date(), "Re-Enabling Controller Services", 60);
            // Step 13. Re-enable all disabled controller services
            final CancellableTimedPause enableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
            asyncRequest.setCancelCallback(enableServicesPause::cancel);
            final Set<AffectedComponentEntity> servicesToEnable = getUpdatedEntities(enabledServices, user);
            logger.info("Successfully updated flow; re-enabling {} Controller Services", servicesToEnable.size());
            try {
                componentLifecycle.activateControllerServices(exampleUri, user, groupId, servicesToEnable, ControllerServiceState.ENABLED, enableServicesPause);
            } catch (final IllegalStateException ise) {
                // a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
                throw new ResumeFlowException("Failed to re-enable Controller Services because " + ise.getMessage(), ise);
            }
        }
        if (!asyncRequest.isCancelled()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Restart {} Processors: {}", runningComponents.size(), runningComponents);
            }
            asyncRequest.update(new Date(), "Restarting Processors", 80);
            // Step 14. Restart all components
            final Set<AffectedComponentEntity> componentsToStart = getUpdatedEntities(runningComponents, user);
            // If there are any Remote Group Ports that are supposed to be started and have no connections, we want to remove those from our Set.
            // This will happen if the Remote Group Port is transmitting when the version change happens but the new flow version does not have
            // a connection to the port. In such a case, the Port still is included in the Updated Entities because we do not remove them
            // when updating the flow (they are removed in the background).
            final Set<AffectedComponentEntity> avoidStarting = new HashSet<>();
            for (final AffectedComponentEntity componentEntity : componentsToStart) {
                final AffectedComponentDTO componentDto = componentEntity.getComponent();
                final String referenceType = componentDto.getReferenceType();
                if (!AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT.equals(referenceType) && !AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT.equals(referenceType)) {
                    continue;
                }
                boolean startComponent;
                try {
                    startComponent = serviceFacade.isRemoteGroupPortConnected(componentDto.getProcessGroupId(), componentDto.getId());
                } catch (final ResourceNotFoundException rnfe) {
                    // Could occur if RPG is refreshed at just the right time.
                    startComponent = false;
                }
                // rather than removing the component here, because doing so would result in a ConcurrentModificationException.
                if (!startComponent) {
                    avoidStarting.add(componentEntity);
                }
            }
            componentsToStart.removeAll(avoidStarting);
            final CancellableTimedPause startComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
            asyncRequest.setCancelCallback(startComponentsPause::cancel);
            logger.info("Restarting {} Processors", componentsToStart.size());
            try {
                componentLifecycle.scheduleComponents(exampleUri, user, groupId, componentsToStart, ScheduledState.RUNNING, startComponentsPause);
            } catch (final IllegalStateException ise) {
                // a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
                throw new ResumeFlowException("Failed to restart components because " + ise.getMessage(), ise);
            }
        }
    }
    asyncRequest.setCancelCallback(null);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Complete", 100);
    return serviceFacade.getVersionControlInformation(groupId);
}

Example 2 with AffectedComponentDTO

use of org.apache.nifi.web.api.dto.AffectedComponentDTO in project nifi by apache.

the class StandardNiFiServiceFacade method getActiveComponentsAffectedByVariableRegistryUpdate.

@Override
public Set<AffectedComponentDTO> getActiveComponentsAffectedByVariableRegistryUpdate(final VariableRegistryDTO variableRegistryDto) {
    final ProcessGroup group = processGroupDAO.getProcessGroup(variableRegistryDto.getProcessGroupId());
    if (group == null) {
        throw new ResourceNotFoundException("Could not find Process Group with ID " + variableRegistryDto.getProcessGroupId());
    }
    final Map<String, String> variableMap = new HashMap<>();
    // have to use forEach here instead of using Collectors.toMap because value may be null
    variableRegistryDto.getVariables().stream().map(VariableEntity::getVariable).forEach(var -> variableMap.put(var.getName(), var.getValue()));
    final Set<AffectedComponentDTO> affectedComponentDtos = new HashSet<>();
    final Set<String> updatedVariableNames = getUpdatedVariables(group, variableMap);
    for (final String variableName : updatedVariableNames) {
        final Set<ConfiguredComponent> affectedComponents = group.getComponentsAffectedByVariable(variableName);
        for (final ConfiguredComponent component : affectedComponents) {
            if (component instanceof ProcessorNode) {
                final ProcessorNode procNode = (ProcessorNode) component;
                if (procNode.isRunning()) {
                    affectedComponentDtos.add(dtoFactory.createAffectedComponentDto(procNode));
                }
            } else if (component instanceof ControllerServiceNode) {
                final ControllerServiceNode serviceNode = (ControllerServiceNode) component;
                if (serviceNode.isActive()) {
                    affectedComponentDtos.add(dtoFactory.createAffectedComponentDto(serviceNode));
                }
            } else {
                throw new RuntimeException("Found unexpected type of Component [" + component.getCanonicalClassName() + "] dependending on variable");
            }
        }
    }
    return affectedComponentDtos;
}

Example 3 with AffectedComponentDTO

use of org.apache.nifi.web.api.dto.AffectedComponentDTO in project nifi by apache.

the class ProcessGroupResource method scheduleProcessors.

private void scheduleProcessors(final String groupId, final URI originalUri, final VariableRegistryUpdateRequest updateRequest, final Pause pause, final Collection<AffectedComponentDTO> affectedProcessors, final ScheduledState desiredState, final VariableRegistryUpdateStep updateStep) throws InterruptedException {
    final Set<String> affectedProcessorIds = affectedProcessors.stream().map(component -> component.getId()).collect(Collectors.toSet());
    final Map<String, Revision> processorRevisionMap = getRevisions(groupId, affectedProcessorIds);
    final Map<String, RevisionDTO> processorRevisionDtoMap = processorRevisionMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> dtoFactory.createRevisionDTO(entry.getValue())));
    final ScheduleComponentsEntity scheduleProcessorsEntity = new ScheduleComponentsEntity();
    scheduleProcessorsEntity.setComponents(processorRevisionDtoMap);
    scheduleProcessorsEntity.setId(groupId);
    scheduleProcessorsEntity.setState(desiredState.name());
    URI scheduleGroupUri;
    try {
        scheduleGroupUri = new URI(originalUri.getScheme(), originalUri.getUserInfo(), originalUri.getHost(), originalUri.getPort(), "/nifi-api/flow/process-groups/" + groupId, null, originalUri.getFragment());
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    final Map<String, String> headers = new HashMap<>();
    headers.put("content-type", MediaType.APPLICATION_JSON);
    // Determine whether we should replicate only to the cluster coordinator, or if we should replicate directly to the cluster nodes themselves.
    final NodeResponse clusterResponse;
    if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
        clusterResponse = getRequestReplicator().replicate(HttpMethod.PUT, scheduleGroupUri, scheduleProcessorsEntity, headers).awaitMergedResponse();
    } else {
        clusterResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), HttpMethod.PUT, scheduleGroupUri, scheduleProcessorsEntity, headers).awaitMergedResponse();
    }
    final int stopProcessorStatus = clusterResponse.getStatus();
    if (stopProcessorStatus != Status.OK.getStatusCode()) {
        updateRequest.getStopProcessorsStep().setFailureReason("Failed while " + updateStep.getDescription());
        updateStep.setComplete(true);
        updateRequest.setFailureReason("Failed while " + updateStep.getDescription());
        return;
    }
    updateRequest.setLastUpdated(new Date());
    final boolean processorsTransitioned = waitForProcessorStatus(originalUri, groupId, affectedProcessorIds, desiredState, updateRequest, pause);
    updateStep.setComplete(true);
    if (!processorsTransitioned) {
        updateStep.setFailureReason("Failed while " + updateStep.getDescription());
        updateRequest.setComplete(true);
        updateRequest.setFailureReason("Failed while " + updateStep.getDescription());
    }
}

Example 4 with AffectedComponentDTO

use of org.apache.nifi.web.api.dto.AffectedComponentDTO in project nifi by apache.

the class ProcessGroupResource method isProcessorActionComplete.

private boolean isProcessorActionComplete(final Set<ProcessorEntity> processorEntities, final VariableRegistryUpdateRequest updateRequest, final Set<String> processorIds, final ScheduledState desiredState) {
    final String desiredStateName = desiredState.name();
    // update the affected processors
    processorEntities.stream().filter(entity -> updateRequest.getAffectedComponents().containsKey(entity.getId())).forEach(entity -> {
        final AffectedComponentEntity affectedComponentEntity = updateRequest.getAffectedComponents().get(entity.getId());
        affectedComponentEntity.setRevision(entity.getRevision());
        // only consider update this component if the user had permissions to it
        if (Boolean.TRUE.equals(affectedComponentEntity.getPermissions().getCanRead())) {
            final AffectedComponentDTO affectedComponent = affectedComponentEntity.getComponent();
            affectedComponent.setState(entity.getStatus().getAggregateSnapshot().getRunStatus());
            affectedComponent.setActiveThreadCount(entity.getStatus().getAggregateSnapshot().getActiveThreadCount());
            if (Boolean.TRUE.equals(entity.getPermissions().getCanRead())) {
                affectedComponent.setValidationErrors(entity.getComponent().getValidationErrors());
            }
        }
    });
    final boolean allProcessorsMatch = processorEntities.stream().filter(entity -> processorIds.contains(entity.getId())).allMatch(entity -> {
        final ProcessorStatusDTO status = entity.getStatus();
        final String runStatus = status.getAggregateSnapshot().getRunStatus();
        final boolean stateMatches = desiredStateName.equalsIgnoreCase(runStatus);
        if (!stateMatches) {
            return false;
        }
        if (desiredState == ScheduledState.STOPPED && status.getAggregateSnapshot().getActiveThreadCount() != 0) {
            return false;
        }
        return true;
    });
    if (!allProcessorsMatch) {
        return false;
    }
    return true;
}

Example 5 with AffectedComponentDTO

use of org.apache.nifi.web.api.dto.AffectedComponentDTO in project nifi by apache.

the class LocalComponentLifecycle method updateAffectedControllerServices.

/**
 * Updates the affected controller services in the specified updateRequest with the serviceEntities.
 *
 * @param serviceEntities service entities
 * @param affectedServices all Controller Services whose state should be equal to the given desired state
 */
private void updateAffectedControllerServices(final Set<ControllerServiceEntity> serviceEntities, final Map<String, AffectedComponentEntity> affectedServices) {
    // update the affected components
    serviceEntities.stream().filter(entity -> affectedServices.containsKey(entity.getId())).forEach(entity -> {
        final AffectedComponentEntity affectedComponentEntity = affectedServices.get(entity.getId());
        affectedComponentEntity.setRevision(entity.getRevision());
        // only consider update this component if the user had permissions to it
        if (Boolean.TRUE.equals(affectedComponentEntity.getPermissions().getCanRead())) {
            final AffectedComponentDTO affectedComponent = affectedComponentEntity.getComponent();
            affectedComponent.setState(entity.getComponent().getState());
            if (Boolean.TRUE.equals(entity.getPermissions().getCanRead())) {
                affectedComponent.setValidationErrors(entity.getComponent().getValidationErrors());
            }
        }
    });
}