Examples with VariableRegistryUpdateRequest org.apache.nifi.registry.variable.VariableRegistryUpdateRequest used on opensource projects

Example 6 with VariableRegistryUpdateRequest

use of org.apache.nifi.registry.variable.VariableRegistryUpdateRequest in project nifi by apache.

the class ProcessGroupResource method updateAffectedControllerServices.

/**
 * Updates the affected controller services in the specified updateRequest with the serviceEntities.
 *
 * @param serviceEntities service entities
 * @param updateRequest update request
 */
private void updateAffectedControllerServices(final Set<ControllerServiceEntity> serviceEntities, final VariableRegistryUpdateRequest updateRequest) {
    // update the affected components
    serviceEntities.stream().filter(entity -> updateRequest.getAffectedComponents().containsKey(entity.getId())).forEach(entity -> {
        final AffectedComponentEntity affectedComponentEntity = updateRequest.getAffectedComponents().get(entity.getId());
        affectedComponentEntity.setRevision(entity.getRevision());
        // only consider update this component if the user had permissions to it
        if (Boolean.TRUE.equals(affectedComponentEntity.getPermissions().getCanRead())) {
            final AffectedComponentDTO affectedComponent = affectedComponentEntity.getComponent();
            affectedComponent.setState(entity.getComponent().getState());
            if (Boolean.TRUE.equals(entity.getPermissions().getCanRead())) {
                affectedComponent.setValidationErrors(entity.getComponent().getValidationErrors());
            }
        }
    });
}

Example 7 with VariableRegistryUpdateRequest

use of org.apache.nifi.registry.variable.VariableRegistryUpdateRequest in project nifi by apache.

the class ProcessGroupResource method createVariableRegistryUpdateRequest.

private VariableRegistryUpdateRequest createVariableRegistryUpdateRequest(final String groupId, final Set<AffectedComponentEntity> affectedComponents, final NiFiUser user) {
    final VariableRegistryUpdateRequest updateRequest = new VariableRegistryUpdateRequest(UUID.randomUUID().toString(), groupId, affectedComponents, user);
    // before adding to the request map, purge any old requests. Must do this by creating a List of ID's
    // and then removing those ID's one-at-a-time in order to avoid ConcurrentModificationException.
    final Date oneMinuteAgo = new Date(System.currentTimeMillis() - VARIABLE_REGISTRY_UPDATE_REQUEST_EXPIRATION);
    final List<String> completedRequestIds = varRegistryUpdateRequests.entrySet().stream().filter(entry -> entry.getValue().isComplete()).filter(entry -> entry.getValue().getLastUpdated().before(oneMinuteAgo)).map(Map.Entry::getKey).collect(Collectors.toList());
    completedRequestIds.stream().forEach(id -> varRegistryUpdateRequests.remove(id));
    final int requestCount = varRegistryUpdateRequests.size();
    if (requestCount > MAX_VARIABLE_REGISTRY_UPDATE_REQUESTS) {
        throw new IllegalStateException("There are already " + requestCount + " update requests for variable registries. " + "Cannot issue any more requests until the older ones are deleted or expire");
    }
    this.varRegistryUpdateRequests.put(updateRequest.getRequestId(), updateRequest);
    return updateRequest;
}

Example 8 with VariableRegistryUpdateRequest

use of org.apache.nifi.registry.variable.VariableRegistryUpdateRequest in project nifi by apache.

the class ProcessGroupResource method updateVariableRegistryLocal.

private Response updateVariableRegistryLocal(final String groupId, final Set<AffectedComponentEntity> affectedComponents, final List<AffectedComponentDTO> affectedProcessors, final List<AffectedComponentDTO> affectedServices, final NiFiUser user, final Revision requestRevision, final VariableRegistryEntity requestEntity) {
    final Set<String> affectedProcessorIds = affectedProcessors == null ? Collections.emptySet() : affectedProcessors.stream().map(component -> component.getId()).collect(Collectors.toSet());
    Map<String, Revision> processorRevisionMap = getRevisions(groupId, affectedProcessorIds);
    final Set<String> affectedServiceIds = affectedServices == null ? Collections.emptySet() : affectedServices.stream().map(component -> component.getId()).collect(Collectors.toSet());
    Map<String, Revision> serviceRevisionMap = getRevisions(groupId, affectedServiceIds);
    // update the variable registry
    final VariableRegistryUpdateRequest updateRequest = createVariableRegistryUpdateRequest(groupId, affectedComponents, user);
    updateRequest.getIdentifyRelevantComponentsStep().setComplete(true);
    final Pause pause = createPause(updateRequest);
    final Runnable updateTask = new Runnable() {

        @Override
        public void run() {
            try {
                // Stop processors
                performUpdateVariableRegistryStep(groupId, updateRequest, updateRequest.getStopProcessorsStep(), "Stopping Processors", () -> stopProcessors(user, updateRequest, groupId, processorRevisionMap, pause));
                // Update revision map because this will have modified the revisions of our components.
                final Map<String, Revision> updatedProcessorRevisionMap = getRevisions(groupId, affectedProcessorIds);
                // Disable controller services
                performUpdateVariableRegistryStep(groupId, updateRequest, updateRequest.getDisableServicesStep(), "Disabling Controller Services", () -> disableControllerServices(user, updateRequest, groupId, serviceRevisionMap, pause));
                // Update revision map because this will have modified the revisions of our components.
                final Map<String, Revision> updatedServiceRevisionMap = getRevisions(groupId, affectedServiceIds);
                // Apply the updates
                performUpdateVariableRegistryStep(groupId, updateRequest, updateRequest.getApplyUpdatesStep(), "Applying updates to Variable Registry", () -> {
                    final VariableRegistryEntity entity = serviceFacade.updateVariableRegistry(user, requestRevision, requestEntity.getVariableRegistry());
                    updateRequest.setProcessGroupRevision(entity.getProcessGroupRevision());
                });
                // Re-enable the controller services
                performUpdateVariableRegistryStep(groupId, updateRequest, updateRequest.getEnableServicesStep(), "Re-enabling Controller Services", () -> enableControllerServices(user, updateRequest, groupId, updatedServiceRevisionMap, pause));
                // Restart processors
                performUpdateVariableRegistryStep(groupId, updateRequest, updateRequest.getStartProcessorsStep(), "Restarting Processors", () -> startProcessors(user, updateRequest, groupId, updatedProcessorRevisionMap, pause));
                // Set complete
                updateRequest.setComplete(true);
                updateRequest.setLastUpdated(new Date());
            } catch (final Exception e) {
                logger.error("Failed to update Variable Registry for Proces Group with ID " + groupId, e);
                updateRequest.setComplete(true);
                updateRequest.setFailureReason("An unexpected error has occurred: " + e);
            }
        }
    };
    // Submit the task to be run in the background
    variableRegistryThreadPool.submit(updateTask);
    final VariableRegistryUpdateRequestEntity responseEntity = new VariableRegistryUpdateRequestEntity();
    responseEntity.setRequest(dtoFactory.createVariableRegistryUpdateRequestDto(updateRequest));
    responseEntity.setProcessGroupRevision(updateRequest.getProcessGroupRevision());
    responseEntity.getRequest().setUri(generateResourceUri("process-groups", groupId, "variable-registry", "update-requests", updateRequest.getRequestId()));
    final URI location = URI.create(responseEntity.getRequest().getUri());
    return Response.status(Status.ACCEPTED).location(location).entity(responseEntity).build();
}

Example 9 with VariableRegistryUpdateRequest

use of org.apache.nifi.registry.variable.VariableRegistryUpdateRequest in project nifi by apache.

the class ProcessGroupResource method activateControllerServices.

private void activateControllerServices(final String groupId, final URI originalUri, final VariableRegistryUpdateRequest updateRequest, final Pause pause, final Collection<AffectedComponentDTO> affectedServices, final ControllerServiceState desiredState, final VariableRegistryUpdateStep updateStep) throws InterruptedException {
    final Set<String> affectedServiceIds = affectedServices.stream().map(component -> component.getId()).collect(Collectors.toSet());
    final Map<String, Revision> serviceRevisionMap = getRevisions(groupId, affectedServiceIds);
    final Map<String, RevisionDTO> serviceRevisionDtoMap = serviceRevisionMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> dtoFactory.createRevisionDTO(entry.getValue())));
    final ActivateControllerServicesEntity activateServicesEntity = new ActivateControllerServicesEntity();
    activateServicesEntity.setComponents(serviceRevisionDtoMap);
    activateServicesEntity.setId(groupId);
    activateServicesEntity.setState(desiredState.name());
    URI controllerServicesUri;
    try {
        controllerServicesUri = new URI(originalUri.getScheme(), originalUri.getUserInfo(), originalUri.getHost(), originalUri.getPort(), "/nifi-api/flow/process-groups/" + groupId + "/controller-services", null, originalUri.getFragment());
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    final Map<String, String> headers = new HashMap<>();
    headers.put("content-type", MediaType.APPLICATION_JSON);
    // Determine whether we should replicate only to the cluster coordinator, or if we should replicate directly to the cluster nodes themselves.
    final NodeResponse clusterResponse;
    if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
        clusterResponse = getRequestReplicator().replicate(HttpMethod.PUT, controllerServicesUri, activateServicesEntity, headers).awaitMergedResponse();
    } else {
        clusterResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), HttpMethod.PUT, controllerServicesUri, activateServicesEntity, headers).awaitMergedResponse();
    }
    final int disableServicesStatus = clusterResponse.getStatus();
    if (disableServicesStatus != Status.OK.getStatusCode()) {
        updateStep.setFailureReason("Failed while " + updateStep.getDescription());
        updateStep.setComplete(true);
        updateRequest.setFailureReason("Failed while " + updateStep.getDescription());
        return;
    }
    updateRequest.setLastUpdated(new Date());
    final boolean serviceTransitioned = waitForControllerServiceStatus(originalUri, groupId, affectedServiceIds, desiredState, updateRequest, pause);
    updateStep.setComplete(true);
    if (!serviceTransitioned) {
        updateStep.setFailureReason("Failed while " + updateStep.getDescription());
        updateRequest.setComplete(true);
        updateRequest.setFailureReason("Failed while " + updateStep.getDescription());
    }
}

Example 10 with VariableRegistryUpdateRequest

use of org.apache.nifi.registry.variable.VariableRegistryUpdateRequest in project nifi by apache.

the class ProcessGroupResource method submitUpdateVariableRegistryRequest.

/**
 * Updates the variable registry for the specified process group.
 *
 * @param httpServletRequest request
 * @param groupId The id of the process group.
 * @param requestVariableRegistryEntity the Variable Registry Entity
 * @return A Variable Registry Entry.
 */
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("{id}/variable-registry/update-requests")
@ApiOperation(value = "Submits a request to update a process group's variable registry", response = VariableRegistryUpdateRequestEntity.class, notes = NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Write - /process-groups/{uuid}") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response submitUpdateVariableRegistryRequest(@Context final HttpServletRequest httpServletRequest, @ApiParam(value = "The process group id.", required = true) @PathParam("id") final String groupId, @ApiParam(value = "The variable registry configuration details.", required = true) final VariableRegistryEntity requestVariableRegistryEntity) {
    if (requestVariableRegistryEntity == null || requestVariableRegistryEntity.getVariableRegistry() == null) {
        throw new IllegalArgumentException("Variable Registry details must be specified.");
    }
    if (requestVariableRegistryEntity.getProcessGroupRevision() == null) {
        throw new IllegalArgumentException("Process Group Revision must be specified.");
    }
    // In order to update variables in a variable registry, we have to perform the following steps:
    // 1. Determine Affected Components (this includes any Processors and Controller Services and any components that reference an affected Controller Service).
    // 1a. Determine ID's of components
    // 1b. Determine Revision's of associated components
    // 2. Stop All Active Affected Processors
    // 3. Disable All Active Affected Controller Services
    // 4. Update the Variables
    // 5. Re-Enable all previously Active Affected Controller Services (services only, not dependent components)
    // 6. Re-Enable all previously Active Processors that Depended on the Controller Services
    // Determine the affected components (and their associated revisions)
    final VariableRegistryEntity computedEntity = serviceFacade.populateAffectedComponents(requestVariableRegistryEntity.getVariableRegistry());
    final VariableRegistryDTO computedRegistryDto = computedEntity.getVariableRegistry();
    if (computedRegistryDto == null) {
        throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId));
    }
    final Set<AffectedComponentEntity> allAffectedComponents = serviceFacade.getComponentsAffectedByVariableRegistryUpdate(requestVariableRegistryEntity.getVariableRegistry());
    final Set<AffectedComponentDTO> activeAffectedComponents = serviceFacade.getActiveComponentsAffectedByVariableRegistryUpdate(requestVariableRegistryEntity.getVariableRegistry());
    final Map<String, List<AffectedComponentDTO>> activeAffectedComponentsByType = activeAffectedComponents.stream().collect(Collectors.groupingBy(comp -> comp.getReferenceType()));
    final List<AffectedComponentDTO> activeAffectedProcessors = activeAffectedComponentsByType.get(AffectedComponentDTO.COMPONENT_TYPE_PROCESSOR);
    final List<AffectedComponentDTO> activeAffectedServices = activeAffectedComponentsByType.get(AffectedComponentDTO.COMPONENT_TYPE_CONTROLLER_SERVICE);
    final NiFiUser user = NiFiUserUtils.getNiFiUser();
    // define access authorize for execution below
    final AuthorizeAccess authorizeAccess = lookup -> {
        final Authorizable groupAuthorizable = lookup.getProcessGroup(groupId).getAuthorizable();
        groupAuthorizable.authorize(authorizer, RequestAction.WRITE, user);
        // (because this action requires stopping the component).
        if (activeAffectedProcessors != null) {
            for (final AffectedComponentDTO activeAffectedComponent : activeAffectedProcessors) {
                final Authorizable authorizable = lookup.getProcessor(activeAffectedComponent.getId()).getAuthorizable();
                authorizable.authorize(authorizer, RequestAction.READ, user);
                authorizable.authorize(authorizer, RequestAction.WRITE, user);
            }
        }
        if (activeAffectedServices != null) {
            for (final AffectedComponentDTO activeAffectedComponent : activeAffectedServices) {
                final Authorizable authorizable = lookup.getControllerService(activeAffectedComponent.getId()).getAuthorizable();
                authorizable.authorize(authorizer, RequestAction.READ, user);
                authorizable.authorize(authorizer, RequestAction.WRITE, user);
            }
        }
    };
    if (isReplicateRequest()) {
        // authorize access
        serviceFacade.authorizeAccess(authorizeAccess);
        // update the variable registry
        final VariableRegistryUpdateRequest updateRequest = createVariableRegistryUpdateRequest(groupId, allAffectedComponents, user);
        updateRequest.getIdentifyRelevantComponentsStep().setComplete(true);
        final URI originalUri = getAbsolutePath();
        // Submit the task to be run in the background
        final Runnable taskWrapper = () -> {
            try {
                // set the user authentication token
                final Authentication authentication = new NiFiAuthenticationToken(new NiFiUserDetails(user));
                SecurityContextHolder.getContext().setAuthentication(authentication);
                updateVariableRegistryReplicated(groupId, originalUri, activeAffectedProcessors, activeAffectedServices, updateRequest, requestVariableRegistryEntity);
                // ensure the request is marked complete
                updateRequest.setComplete(true);
            } catch (final Exception e) {
                logger.error("Failed to update variable registry", e);
                updateRequest.setComplete(true);
                updateRequest.setFailureReason("An unexpected error has occurred: " + e);
            } finally {
                // clear the authentication token
                SecurityContextHolder.getContext().setAuthentication(null);
            }
        };
        variableRegistryThreadPool.submit(taskWrapper);
        final VariableRegistryUpdateRequestEntity responseEntity = new VariableRegistryUpdateRequestEntity();
        responseEntity.setRequest(dtoFactory.createVariableRegistryUpdateRequestDto(updateRequest));
        responseEntity.setProcessGroupRevision(updateRequest.getProcessGroupRevision());
        responseEntity.getRequest().setUri(generateResourceUri("process-groups", groupId, "variable-registry", "update-requests", updateRequest.getRequestId()));
        final URI location = URI.create(responseEntity.getRequest().getUri());
        return Response.status(Status.ACCEPTED).location(location).entity(responseEntity).build();
    }
    final UpdateVariableRegistryRequestWrapper requestWrapper = new UpdateVariableRegistryRequestWrapper(allAffectedComponents, activeAffectedProcessors, activeAffectedServices, requestVariableRegistryEntity);
    final Revision requestRevision = getRevision(requestVariableRegistryEntity.getProcessGroupRevision(), groupId);
    return withWriteLock(serviceFacade, requestWrapper, requestRevision, authorizeAccess, null, (revision, wrapper) -> updateVariableRegistryLocal(groupId, wrapper.getAllAffectedComponents(), wrapper.getActiveAffectedProcessors(), wrapper.getActiveAffectedServices(), user, revision, wrapper.getVariableRegistryEntity()));
}