Search in sources :

Example 1 with CancellableTimedPause

use of org.apache.nifi.web.util.CancellableTimedPause in project nifi by apache.

the class VersionsResource method updateFlowVersion.

private VersionControlInformationEntity updateFlowVersion(final String groupId, final ComponentLifecycle componentLifecycle, final URI exampleUri, final Set<AffectedComponentEntity> affectedComponents, final NiFiUser user, final boolean replicateRequest, final Revision revision, final VersionControlInformationEntity requestEntity, final VersionedFlowSnapshot flowSnapshot, final AsynchronousWebRequest<VersionControlInformationEntity> asyncRequest, final String idGenerationSeed, final boolean verifyNotModified, final boolean updateDescendantVersionedFlows) throws LifecycleManagementException, ResumeFlowException {
    // Steps 6-7: Determine which components must be stopped and stop them.
    final Set<String> stoppableReferenceTypes = new HashSet<>();
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_PROCESSOR);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_INPUT_PORT);
    stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_OUTPUT_PORT);
    final Set<AffectedComponentEntity> runningComponents = affectedComponents.stream().filter(dto -> stoppableReferenceTypes.contains(dto.getComponent().getReferenceType())).filter(dto -> "Running".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
    logger.info("Stopping {} Processors", runningComponents.size());
    final CancellableTimedPause stopComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    asyncRequest.setCancelCallback(stopComponentsPause::cancel);
    componentLifecycle.scheduleComponents(exampleUri, user, groupId, runningComponents, ScheduledState.STOPPED, stopComponentsPause);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Disabling Affected Controller Services", 20);
    // Steps 8-9. Disable enabled controller services that are affected
    final Set<AffectedComponentEntity> enabledServices = affectedComponents.stream().filter(dto -> AffectedComponentDTO.COMPONENT_TYPE_CONTROLLER_SERVICE.equals(dto.getComponent().getReferenceType())).filter(dto -> "Enabled".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
    logger.info("Disabling {} Controller Services", enabledServices.size());
    final CancellableTimedPause disableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    asyncRequest.setCancelCallback(disableServicesPause::cancel);
    componentLifecycle.activateControllerServices(exampleUri, user, groupId, enabledServices, ControllerServiceState.DISABLED, disableServicesPause);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Updating Flow", 40);
    logger.info("Updating Process Group with ID {} to version {} of the Versioned Flow", groupId, flowSnapshot.getSnapshotMetadata().getVersion());
    // by replicating a PUT to /nifi-api/versions/process-groups/{groupId}
    try {
        if (replicateRequest) {
            final URI updateUri;
            try {
                updateUri = new URI(exampleUri.getScheme(), exampleUri.getUserInfo(), exampleUri.getHost(), exampleUri.getPort(), "/nifi-api/versions/process-groups/" + groupId, null, exampleUri.getFragment());
            } catch (URISyntaxException e) {
                throw new RuntimeException(e);
            }
            final Map<String, String> headers = new HashMap<>();
            headers.put("content-type", MediaType.APPLICATION_JSON);
            final VersionedFlowSnapshotEntity snapshotEntity = new VersionedFlowSnapshotEntity();
            snapshotEntity.setProcessGroupRevision(dtoFactory.createRevisionDTO(revision));
            snapshotEntity.setRegistryId(requestEntity.getVersionControlInformation().getRegistryId());
            snapshotEntity.setVersionedFlow(flowSnapshot);
            snapshotEntity.setUpdateDescendantVersionedFlows(updateDescendantVersionedFlows);
            final NodeResponse clusterResponse;
            try {
                logger.debug("Replicating PUT request to {} for user {}", updateUri, user);
                if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
                    clusterResponse = getRequestReplicator().replicate(user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
                } else {
                    clusterResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
                }
            } catch (final InterruptedException ie) {
                logger.warn("Interrupted while replicating PUT request to {} for user {}", updateUri, user);
                Thread.currentThread().interrupt();
                throw new LifecycleManagementException("Interrupted while updating flows across cluster", ie);
            }
            final int updateFlowStatus = clusterResponse.getStatus();
            if (updateFlowStatus != Status.OK.getStatusCode()) {
                final String explanation = getResponseEntity(clusterResponse, String.class);
                logger.error("Failed to update flow across cluster when replicating PUT request to {} for user {}. Received {} response with explanation: {}", updateUri, user, updateFlowStatus, explanation);
                throw new LifecycleManagementException("Failed to update Flow on all nodes in cluster due to " + explanation);
            }
        } else {
            // Step 10: Ensure that if any connection exists in the flow and does not exist in the proposed snapshot,
            // that it has no data in it. Ensure that no Input Port was removed, unless it currently has no incoming connections.
            // Ensure that no Output Port was removed, unless it currently has no outgoing connections.
            serviceFacade.verifyCanUpdate(groupId, flowSnapshot, true, verifyNotModified);
            // Step 11-12. Update Process Group to the new flow and update variable registry with any Variables that were added or removed
            final VersionControlInformationDTO requestVci = requestEntity.getVersionControlInformation();
            final Bucket bucket = flowSnapshot.getBucket();
            final VersionedFlow flow = flowSnapshot.getFlow();
            final VersionedFlowSnapshotMetadata metadata = flowSnapshot.getSnapshotMetadata();
            final VersionControlInformationDTO vci = new VersionControlInformationDTO();
            vci.setBucketId(metadata.getBucketIdentifier());
            vci.setBucketName(bucket.getName());
            vci.setFlowDescription(flow.getDescription());
            vci.setFlowId(flow.getIdentifier());
            vci.setFlowName(flow.getName());
            vci.setGroupId(groupId);
            vci.setRegistryId(requestVci.getRegistryId());
            vci.setRegistryName(serviceFacade.getFlowRegistryName(requestVci.getRegistryId()));
            vci.setVersion(metadata.getVersion());
            vci.setState(flowSnapshot.isLatest() ? VersionedFlowState.UP_TO_DATE.name() : VersionedFlowState.STALE.name());
            serviceFacade.updateProcessGroupContents(user, revision, groupId, vci, flowSnapshot, idGenerationSeed, verifyNotModified, false, updateDescendantVersionedFlows);
        }
    } finally {
        if (!asyncRequest.isCancelled()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Re-Enabling {} Controller Services: {}", enabledServices.size(), enabledServices);
            }
            asyncRequest.update(new Date(), "Re-Enabling Controller Services", 60);
            // Step 13. Re-enable all disabled controller services
            final CancellableTimedPause enableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
            asyncRequest.setCancelCallback(enableServicesPause::cancel);
            final Set<AffectedComponentEntity> servicesToEnable = getUpdatedEntities(enabledServices, user);
            logger.info("Successfully updated flow; re-enabling {} Controller Services", servicesToEnable.size());
            try {
                componentLifecycle.activateControllerServices(exampleUri, user, groupId, servicesToEnable, ControllerServiceState.ENABLED, enableServicesPause);
            } catch (final IllegalStateException ise) {
                // a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
                throw new ResumeFlowException("Failed to re-enable Controller Services because " + ise.getMessage(), ise);
            }
        }
        if (!asyncRequest.isCancelled()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Restart {} Processors: {}", runningComponents.size(), runningComponents);
            }
            asyncRequest.update(new Date(), "Restarting Processors", 80);
            // Step 14. Restart all components
            final Set<AffectedComponentEntity> componentsToStart = getUpdatedEntities(runningComponents, user);
            // If there are any Remote Group Ports that are supposed to be started and have no connections, we want to remove those from our Set.
            // This will happen if the Remote Group Port is transmitting when the version change happens but the new flow version does not have
            // a connection to the port. In such a case, the Port still is included in the Updated Entities because we do not remove them
            // when updating the flow (they are removed in the background).
            final Set<AffectedComponentEntity> avoidStarting = new HashSet<>();
            for (final AffectedComponentEntity componentEntity : componentsToStart) {
                final AffectedComponentDTO componentDto = componentEntity.getComponent();
                final String referenceType = componentDto.getReferenceType();
                if (!AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT.equals(referenceType) && !AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT.equals(referenceType)) {
                    continue;
                }
                boolean startComponent;
                try {
                    startComponent = serviceFacade.isRemoteGroupPortConnected(componentDto.getProcessGroupId(), componentDto.getId());
                } catch (final ResourceNotFoundException rnfe) {
                    // Could occur if RPG is refreshed at just the right time.
                    startComponent = false;
                }
                // rather than removing the component here, because doing so would result in a ConcurrentModificationException.
                if (!startComponent) {
                    avoidStarting.add(componentEntity);
                }
            }
            componentsToStart.removeAll(avoidStarting);
            final CancellableTimedPause startComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
            asyncRequest.setCancelCallback(startComponentsPause::cancel);
            logger.info("Restarting {} Processors", componentsToStart.size());
            try {
                componentLifecycle.scheduleComponents(exampleUri, user, groupId, componentsToStart, ScheduledState.RUNNING, startComponentsPause);
            } catch (final IllegalStateException ise) {
                // a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
                throw new ResumeFlowException("Failed to restart components because " + ise.getMessage(), ise);
            }
        }
    }
    asyncRequest.setCancelCallback(null);
    if (asyncRequest.isCancelled()) {
        return null;
    }
    asyncRequest.update(new Date(), "Complete", 100);
    return serviceFacade.getVersionControlInformation(groupId);
}
Also used : Produces(javax.ws.rs.Produces) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) ApiParam(io.swagger.annotations.ApiParam) ComponentAuthorizable(org.apache.nifi.authorization.ComponentAuthorizable) StringUtils(org.apache.commons.lang3.StringUtils) ClientIdParameter(org.apache.nifi.web.api.request.ClientIdParameter) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) Map(java.util.Map) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) DefaultValue(javax.ws.rs.DefaultValue) AsyncRequestManager(org.apache.nifi.web.api.concurrent.AsyncRequestManager) URI(java.net.URI) NodeResponse(org.apache.nifi.cluster.manager.NodeResponse) VersionedFlowUpdateRequestDTO(org.apache.nifi.web.api.dto.VersionedFlowUpdateRequestDTO) DELETE(javax.ws.rs.DELETE) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) Authorizable(org.apache.nifi.authorization.resource.Authorizable) VersionedFlowDTO(org.apache.nifi.web.api.dto.VersionedFlowDTO) Set(java.util.Set) UUID(java.util.UUID) BundleUtils(org.apache.nifi.util.BundleUtils) LongParameter(org.apache.nifi.web.api.request.LongParameter) Collectors(java.util.stream.Collectors) FlowController(org.apache.nifi.controller.FlowController) CreateActiveRequestEntity(org.apache.nifi.web.api.entity.CreateActiveRequestEntity) Response(javax.ws.rs.core.Response) ScheduledState(org.apache.nifi.controller.ScheduledState) ProcessGroupEntity(org.apache.nifi.web.api.entity.ProcessGroupEntity) VersionedFlowSnapshotEntity(org.apache.nifi.web.api.entity.VersionedFlowSnapshotEntity) ControllerServiceState(org.apache.nifi.controller.service.ControllerServiceState) DtoFactory(org.apache.nifi.web.api.dto.DtoFactory) Entity(org.apache.nifi.web.api.entity.Entity) PathParam(javax.ws.rs.PathParam) Bucket(org.apache.nifi.registry.bucket.Bucket) Revision(org.apache.nifi.web.Revision) GET(javax.ws.rs.GET) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) ConfigurableComponent(org.apache.nifi.components.ConfigurableComponent) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) RevisionDTO(org.apache.nifi.web.api.dto.RevisionDTO) AffectedComponentDTO(org.apache.nifi.web.api.dto.AffectedComponentDTO) ComponentLifecycle(org.apache.nifi.web.util.ComponentLifecycle) HttpMethod(javax.ws.rs.HttpMethod) HashSet(java.util.HashSet) FlowRegistryUtils(org.apache.nifi.registry.flow.FlowRegistryUtils) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) Api(io.swagger.annotations.Api) VersionControlComponentMappingEntity(org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity) CancellableTimedPause(org.apache.nifi.web.util.CancellableTimedPause) Status(javax.ws.rs.core.Response.Status) LinkedHashSet(java.util.LinkedHashSet) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) AffectedComponentUtils(org.apache.nifi.web.util.AffectedComponentUtils) NiFiServiceFacade(org.apache.nifi.web.NiFiServiceFacade) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) RequestAction(org.apache.nifi.authorization.RequestAction) IOException(java.io.IOException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) StartVersionControlRequestEntity(org.apache.nifi.web.api.entity.StartVersionControlRequestEntity) Authorizer(org.apache.nifi.authorization.Authorizer) VersionControlInformationEntity(org.apache.nifi.web.api.entity.VersionControlInformationEntity) ProcessGroupAuthorizable(org.apache.nifi.authorization.ProcessGroupAuthorizable) ApiResponse(io.swagger.annotations.ApiResponse) NiFiUserUtils(org.apache.nifi.authorization.user.NiFiUserUtils) PUT(javax.ws.rs.PUT) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) VersionedFlowUpdateRequestEntity(org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity) Authorization(io.swagger.annotations.Authorization) RequestManager(org.apache.nifi.web.api.concurrent.RequestManager) Collections(java.util.Collections) AsynchronousWebRequest(org.apache.nifi.web.api.concurrent.AsynchronousWebRequest) HashMap(java.util.HashMap) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) NodeResponse(org.apache.nifi.cluster.manager.NodeResponse) VersionedFlowSnapshotEntity(org.apache.nifi.web.api.entity.VersionedFlowSnapshotEntity) URISyntaxException(java.net.URISyntaxException) URI(java.net.URI) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) CancellableTimedPause(org.apache.nifi.web.util.CancellableTimedPause) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) Date(java.util.Date) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) Bucket(org.apache.nifi.registry.bucket.Bucket) AffectedComponentDTO(org.apache.nifi.web.api.dto.AffectedComponentDTO)

Aggregations

Api (io.swagger.annotations.Api)1 ApiOperation (io.swagger.annotations.ApiOperation)1 ApiParam (io.swagger.annotations.ApiParam)1 ApiResponse (io.swagger.annotations.ApiResponse)1 ApiResponses (io.swagger.annotations.ApiResponses)1 Authorization (io.swagger.annotations.Authorization)1 IOException (java.io.IOException)1 URI (java.net.URI)1 URISyntaxException (java.net.URISyntaxException)1 Collections (java.util.Collections)1 Date (java.util.Date)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 LinkedHashSet (java.util.LinkedHashSet)1 Map (java.util.Map)1 Set (java.util.Set)1 UUID (java.util.UUID)1 TimeUnit (java.util.concurrent.TimeUnit)1 Consumer (java.util.function.Consumer)1 Collectors (java.util.stream.Collectors)1