Search in sources :

Example 26 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class StandardNiFiServiceFacade method getVersionedFlowSnapshot.

@Override
public VersionedFlowSnapshot getVersionedFlowSnapshot(final VersionControlInformationDTO versionControlInfo, final boolean fetchRemoteFlows) {
    final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(versionControlInfo.getRegistryId());
    if (flowRegistry == null) {
        throw new ResourceNotFoundException("Could not find any Flow Registry registered with identifier " + versionControlInfo.getRegistryId());
    }
    final VersionedFlowSnapshot snapshot;
    try {
        snapshot = flowRegistry.getFlowContents(versionControlInfo.getBucketId(), versionControlInfo.getFlowId(), versionControlInfo.getVersion(), fetchRemoteFlows, NiFiUserUtils.getNiFiUser());
    } catch (final NiFiRegistryException | IOException e) {
        throw new IllegalArgumentException("The Flow Registry with ID " + versionControlInfo.getRegistryId() + " reports that no Flow exists with Bucket " + versionControlInfo.getBucketId() + ", Flow " + versionControlInfo.getFlowId() + ", Version " + versionControlInfo.getVersion());
    }
    return snapshot;
}
Also used : FlowRegistry(org.apache.nifi.registry.flow.FlowRegistry) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) IOException(java.io.IOException) NiFiRegistryException(org.apache.nifi.registry.client.NiFiRegistryException)

Example 27 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class ImportFlowVersion method doExecute.

@Override
public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException {
    final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID);
    final String inputFile = getRequiredArg(properties, CommandOption.INPUT_SOURCE);
    String contents;
    try {
        // try a public resource URL
        URL url = new URL(inputFile);
        contents = IOUtils.toString(url, StandardCharsets.UTF_8);
    } catch (MalformedURLException e) {
        // assume a local file then
        URI uri = Paths.get(inputFile).toAbsolutePath().toUri();
        contents = IOUtils.toString(uri, StandardCharsets.UTF_8);
    }
    final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient();
    final ObjectMapper objectMapper = JacksonUtils.getObjectMapper();
    final VersionedFlowSnapshot deserializedSnapshot = objectMapper.readValue(contents, VersionedFlowSnapshot.class);
    if (deserializedSnapshot == null) {
        throw new IOException("Unable to deserialize flow version from " + inputFile);
    }
    // determine the bucket for the provided flow id
    final String bucketId = getBucketId(client, flowId);
    // determine the latest existing version in the destination system
    Integer version;
    try {
        final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(bucketId, flowId);
        version = latestMetadata.getVersion() + 1;
    } catch (NiFiRegistryException e) {
        // when there are no versions it produces a 404 not found
        version = 1;
    }
    // create new metadata using the passed in bucket and flow in the target registry, keep comments
    final VersionedFlowSnapshotMetadata metadata = new VersionedFlowSnapshotMetadata();
    metadata.setBucketIdentifier(bucketId);
    metadata.setFlowIdentifier(flowId);
    metadata.setVersion(version);
    metadata.setComments(deserializedSnapshot.getSnapshotMetadata().getComments());
    // create a new snapshot using the new metadata and the contents from the deserialized snapshot
    final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot();
    snapshot.setSnapshotMetadata(metadata);
    snapshot.setFlowContents(deserializedSnapshot.getFlowContents());
    final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot);
    final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata();
    return new StringResult(String.valueOf(createdMetadata.getVersion()), getContext().isInteractive());
}
Also used : MalformedURLException(java.net.MalformedURLException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) FlowSnapshotClient(org.apache.nifi.registry.client.FlowSnapshotClient) IOException(java.io.IOException) StringResult(org.apache.nifi.toolkit.cli.impl.result.StringResult) NiFiRegistryException(org.apache.nifi.registry.client.NiFiRegistryException) URI(java.net.URI) URL(java.net.URL) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata)

Example 28 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class TransferFlowVersion method doExecute.

@Override
public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException {
    final String srcPropsValue = getArg(properties, CommandOption.SRC_PROPS);
    final String srcFlowId = getRequiredArg(properties, CommandOption.SRC_FLOW_ID);
    final Integer srcFlowVersion = getIntArg(properties, CommandOption.SRC_FLOW_VERSION);
    final String destFlowId = getRequiredArg(properties, CommandOption.FLOW_ID);
    final NiFiRegistryClient srcClient = getSourceClient(client, srcPropsValue);
    // determine the bucket ids of the source and dest flows
    final String srcBucketId = getBucketId(srcClient, srcFlowId);
    final String destBucketId = getBucketId(client, destFlowId);
    // get the snapshot of the source flow, either the version specified or the latest
    final VersionedFlowSnapshot srcSnapshot;
    if (srcFlowVersion == null) {
        srcSnapshot = srcClient.getFlowSnapshotClient().getLatest(srcBucketId, srcFlowId);
    } else {
        srcSnapshot = srcClient.getFlowSnapshotClient().get(srcBucketId, srcFlowId, srcFlowVersion);
    }
    // determine the next version number for the destination flow
    final List<Integer> destVersions = getVersions(client, destBucketId, destFlowId);
    final Integer destFlowVersion = destVersions.isEmpty() ? 1 : destVersions.get(0) + 1;
    // create the new metadata for the destination snapshot
    final VersionedFlowSnapshotMetadata destMetadata = new VersionedFlowSnapshotMetadata();
    destMetadata.setBucketIdentifier(destBucketId);
    destMetadata.setFlowIdentifier(destFlowId);
    destMetadata.setVersion(destFlowVersion);
    destMetadata.setComments(srcSnapshot.getSnapshotMetadata().getComments());
    // update the source snapshot with the destination metadata
    srcSnapshot.setFlow(null);
    srcSnapshot.setBucket(null);
    srcSnapshot.setSnapshotMetadata(destMetadata);
    // create the destination snapshot
    client.getFlowSnapshotClient().create(srcSnapshot);
    if (getContext().isInteractive()) {
        println();
        println("Transferred version " + srcSnapshot.getSnapshotMetadata().getVersion() + " of source flow to version " + destFlowVersion + " of destination flow");
    }
    return new OkResult(getContext().isInteractive());
}
Also used : OkResult(org.apache.nifi.toolkit.cli.impl.result.OkResult) NiFiRegistryClient(org.apache.nifi.registry.client.NiFiRegistryClient) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata)

Example 29 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class VersionsResource method initiateVersionControlUpdate.

@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("update-requests/process-groups/{id}")
@ApiOperation(value = "Initiate the Update Request of a Process Group with the given ID", response = VersionedFlowUpdateRequestEntity.class, notes = "For a Process Group that is already under Version Control, this will initiate the action of changing " + "from a specific version of the flow in the Flow Registry to a different version of the flow. This can be a lengthy " + "process, as it will stop any Processors and disable any Controller Services necessary to perform the action and then restart them. As a result, " + "the endpoint will immediately return a VersionedFlowUpdateRequestEntity, and the process of updating the flow will occur " + "asynchronously in the background. The client may then periodically poll the status of the request by issuing a GET request to " + "/versions/update-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE request to " + "/versions/update-requests/{requestId}. " + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), @Authorization(value = "Read - /{component-type}/{uuid} - For all encapsulated components"), @Authorization(value = "Write - /{component-type}/{uuid} - For all encapsulated components"), @Authorization(value = "Write - if the template contains any restricted components - /restricted-components") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response initiateVersionControlUpdate(@ApiParam("The process group id.") @PathParam("id") final String groupId, @ApiParam(value = "The controller service configuration details.", required = true) final VersionControlInformationEntity requestEntity) {
    // Verify the request
    final RevisionDTO revisionDto = requestEntity.getProcessGroupRevision();
    if (revisionDto == null) {
        throw new IllegalArgumentException("Process Group Revision must be specified");
    }
    final VersionControlInformationDTO requestVersionControlInfoDto = requestEntity.getVersionControlInformation();
    if (requestVersionControlInfoDto == null) {
        throw new IllegalArgumentException("Version Control Information must be supplied.");
    }
    if (requestVersionControlInfoDto.getGroupId() == null) {
        throw new IllegalArgumentException("The Process Group ID must be supplied.");
    }
    if (!requestVersionControlInfoDto.getGroupId().equals(groupId)) {
        throw new IllegalArgumentException("The Process Group ID in the request body does not match the Process Group ID of the requested resource.");
    }
    if (requestVersionControlInfoDto.getBucketId() == null) {
        throw new IllegalArgumentException("The Bucket ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getFlowId() == null) {
        throw new IllegalArgumentException("The Flow ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getRegistryId() == null) {
        throw new IllegalArgumentException("The Registry ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getVersion() == null) {
        throw new IllegalArgumentException("The Version of the flow must be supplied.");
    }
    // We will perform the updating of the Versioned Flow in a background thread because it can be a long-running process.
    // In order to do this, we will need some parameters that are only available as Thread-Local variables to the current
    // thread, so we will gather the values for these parameters up front.
    final boolean replicateRequest = isReplicateRequest();
    final ComponentLifecycle componentLifecycle = replicateRequest ? clusterComponentLifecycle : localComponentLifecycle;
    final NiFiUser user = NiFiUserUtils.getNiFiUser();
    // Workflow for this process:
    // 0. Obtain the versioned flow snapshot to use for the update
    // a. Contact registry to download the desired version.
    // b. Get Variable Registry of this Process Group and all ancestor groups
    // c. Perform diff to find any new variables
    // d. Get Variable Registry of any child Process Group in the versioned flow
    // e. Perform diff to find any new variables
    // f. Prompt user to fill in values for all new variables
    // 1. Determine which components would be affected (and are enabled/running)
    // a. Component itself is modified in some way, other than position changing.
    // b. Source and Destination of any Connection that is modified.
    // c. Any Processor or Controller Service that references a Controller Service that is modified.
    // 2. Verify READ and WRITE permissions for user, for every component.
    // 3. Verify that all components in the snapshot exist on all nodes (i.e., the NAR exists)?
    // 4. Verify that Process Group is already under version control. If not, must start Version Control instead of updateFlow
    // 5. Verify that Process Group is not 'dirty'.
    // 6. Stop all Processors, Funnels, Ports that are affected.
    // 7. Wait for all of the components to finish stopping.
    // 8. Disable all Controller Services that are affected.
    // 9. Wait for all Controller Services to finish disabling.
    // 10. Ensure that if any connection was deleted, that it has no data in it. Ensure that no Input Port
    // was removed, unless it currently has no incoming connections. Ensure that no Output Port was removed,
    // unless it currently has no outgoing connections. Checking ports & connections could be done before
    // stopping everything, but removal of Connections cannot.
    // 11. Update variable registry to include new variables
    // (only new variables so don't have to worry about affected components? Or do we need to in case a processor
    // is already referencing the variable? In which case we need to include the affected components above in the
    // Set of affected components before stopping/disabling.).
    // 12. Update components in the Process Group; update Version Control Information.
    // 13. Re-Enable all affected Controller Services that were not removed.
    // 14. Re-Start all Processors, Funnels, Ports that are affected and not removed.
    // Step 0: Get the Versioned Flow Snapshot from the Flow Registry
    final VersionedFlowSnapshot flowSnapshot = serviceFacade.getVersionedFlowSnapshot(requestEntity.getVersionControlInformation(), true);
    // The flow in the registry may not contain the same versions of components that we have in our flow. As a result, we need to update
    // the flow snapshot to contain compatible bundles.
    BundleUtils.discoverCompatibleBundles(flowSnapshot.getFlowContents());
    // Step 1: Determine which components will be affected by updating the version
    final Set<AffectedComponentEntity> affectedComponents = serviceFacade.getComponentsAffectedByVersionChange(groupId, flowSnapshot, user);
    // build a request wrapper
    final InitiateChangeFlowVersionRequestWrapper requestWrapper = new InitiateChangeFlowVersionRequestWrapper(requestEntity, componentLifecycle, getAbsolutePath(), affectedComponents, replicateRequest, flowSnapshot);
    final Revision requestRevision = getRevision(requestEntity.getProcessGroupRevision(), groupId);
    return withWriteLock(serviceFacade, requestWrapper, requestRevision, lookup -> {
        // Step 2: Verify READ and WRITE permissions for user, for every component.
        final ProcessGroupAuthorizable groupAuthorizable = lookup.getProcessGroup(groupId);
        authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.READ, true, false, true, true);
        authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.WRITE, true, false, true, true);
        final VersionedProcessGroup groupContents = flowSnapshot.getFlowContents();
        final Set<ConfigurableComponent> restrictedComponents = FlowRegistryUtils.getRestrictedComponents(groupContents);
        restrictedComponents.forEach(restrictedComponent -> {
            final ComponentAuthorizable restrictedComponentAuthorizable = lookup.getConfigurableComponent(restrictedComponent);
            authorizeRestrictions(authorizer, restrictedComponentAuthorizable);
        });
    }, () -> {
        // Step 3: Verify that all components in the snapshot exist on all nodes
        // Step 4: Verify that Process Group is already under version control. If not, must start Version Control instead of updating flow
        // Step 5: Verify that Process Group is not 'dirty'
        serviceFacade.verifyCanUpdate(groupId, flowSnapshot, false, true);
    }, (revision, wrapper) -> {
        final String idGenerationSeed = getIdGenerationSeed().orElse(null);
        // Create an asynchronous request that will occur in the background, because this request may
        // result in stopping components, which can take an indeterminate amount of time.
        final String requestId = UUID.randomUUID().toString();
        final AsynchronousWebRequest<VersionControlInformationEntity> request = new StandardAsynchronousWebRequest<>(requestId, groupId, user, "Stopping Affected Processors");
        // Submit the request to be performed in the background
        final Consumer<AsynchronousWebRequest<VersionControlInformationEntity>> updateTask = vcur -> {
            try {
                final VersionControlInformationEntity updatedVersionControlEntity = updateFlowVersion(groupId, wrapper.getComponentLifecycle(), wrapper.getExampleUri(), wrapper.getAffectedComponents(), user, wrapper.isReplicateRequest(), revision, wrapper.getVersionControlInformationEntity(), wrapper.getFlowSnapshot(), request, idGenerationSeed, true, true);
                vcur.markComplete(updatedVersionControlEntity);
            } catch (final ResumeFlowException rfe) {
                // Treat ResumeFlowException differently because we don't want to include a message that we couldn't update the flow
                // since in this case the flow was successfully updated - we just couldn't re-enable the components.
                logger.error(rfe.getMessage(), rfe);
                vcur.setFailureReason(rfe.getMessage());
            } catch (final Exception e) {
                logger.error("Failed to update flow to new version", e);
                vcur.setFailureReason("Failed to update flow to new version due to " + e);
            }
        };
        requestManager.submitRequest("update-requests", requestId, request, updateTask);
        // Generate the response.
        final VersionedFlowUpdateRequestDTO updateRequestDto = new VersionedFlowUpdateRequestDTO();
        updateRequestDto.setComplete(request.isComplete());
        updateRequestDto.setFailureReason(request.getFailureReason());
        updateRequestDto.setLastUpdated(request.getLastUpdated());
        updateRequestDto.setProcessGroupId(groupId);
        updateRequestDto.setRequestId(requestId);
        updateRequestDto.setUri(generateResourceUri("versions", "update-requests", requestId));
        updateRequestDto.setPercentCompleted(request.getPercentComplete());
        updateRequestDto.setState(request.getState());
        final VersionedFlowUpdateRequestEntity updateRequestEntity = new VersionedFlowUpdateRequestEntity();
        final RevisionDTO groupRevision = serviceFacade.getProcessGroup(groupId).getRevision();
        updateRequestEntity.setProcessGroupRevision(groupRevision);
        updateRequestEntity.setRequest(updateRequestDto);
        return generateOkResponse(updateRequestEntity).build();
    });
}
Also used : ComponentAuthorizable(org.apache.nifi.authorization.ComponentAuthorizable) Produces(javax.ws.rs.Produces) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) ApiParam(io.swagger.annotations.ApiParam) ComponentAuthorizable(org.apache.nifi.authorization.ComponentAuthorizable) StringUtils(org.apache.commons.lang3.StringUtils) ClientIdParameter(org.apache.nifi.web.api.request.ClientIdParameter) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) Map(java.util.Map) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) DefaultValue(javax.ws.rs.DefaultValue) AsyncRequestManager(org.apache.nifi.web.api.concurrent.AsyncRequestManager) URI(java.net.URI) NodeResponse(org.apache.nifi.cluster.manager.NodeResponse) VersionedFlowUpdateRequestDTO(org.apache.nifi.web.api.dto.VersionedFlowUpdateRequestDTO) DELETE(javax.ws.rs.DELETE) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) Authorizable(org.apache.nifi.authorization.resource.Authorizable) VersionedFlowDTO(org.apache.nifi.web.api.dto.VersionedFlowDTO) Set(java.util.Set) UUID(java.util.UUID) BundleUtils(org.apache.nifi.util.BundleUtils) LongParameter(org.apache.nifi.web.api.request.LongParameter) Collectors(java.util.stream.Collectors) FlowController(org.apache.nifi.controller.FlowController) CreateActiveRequestEntity(org.apache.nifi.web.api.entity.CreateActiveRequestEntity) Response(javax.ws.rs.core.Response) ScheduledState(org.apache.nifi.controller.ScheduledState) ProcessGroupEntity(org.apache.nifi.web.api.entity.ProcessGroupEntity) VersionedFlowSnapshotEntity(org.apache.nifi.web.api.entity.VersionedFlowSnapshotEntity) ControllerServiceState(org.apache.nifi.controller.service.ControllerServiceState) DtoFactory(org.apache.nifi.web.api.dto.DtoFactory) Entity(org.apache.nifi.web.api.entity.Entity) PathParam(javax.ws.rs.PathParam) Bucket(org.apache.nifi.registry.bucket.Bucket) Revision(org.apache.nifi.web.Revision) GET(javax.ws.rs.GET) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) ConfigurableComponent(org.apache.nifi.components.ConfigurableComponent) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) RevisionDTO(org.apache.nifi.web.api.dto.RevisionDTO) AffectedComponentDTO(org.apache.nifi.web.api.dto.AffectedComponentDTO) ComponentLifecycle(org.apache.nifi.web.util.ComponentLifecycle) HttpMethod(javax.ws.rs.HttpMethod) HashSet(java.util.HashSet) FlowRegistryUtils(org.apache.nifi.registry.flow.FlowRegistryUtils) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) Api(io.swagger.annotations.Api) VersionControlComponentMappingEntity(org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity) CancellableTimedPause(org.apache.nifi.web.util.CancellableTimedPause) Status(javax.ws.rs.core.Response.Status) LinkedHashSet(java.util.LinkedHashSet) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) AffectedComponentUtils(org.apache.nifi.web.util.AffectedComponentUtils) NiFiServiceFacade(org.apache.nifi.web.NiFiServiceFacade) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) RequestAction(org.apache.nifi.authorization.RequestAction) IOException(java.io.IOException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) StartVersionControlRequestEntity(org.apache.nifi.web.api.entity.StartVersionControlRequestEntity) Authorizer(org.apache.nifi.authorization.Authorizer) VersionControlInformationEntity(org.apache.nifi.web.api.entity.VersionControlInformationEntity) ProcessGroupAuthorizable(org.apache.nifi.authorization.ProcessGroupAuthorizable) ApiResponse(io.swagger.annotations.ApiResponse) NiFiUserUtils(org.apache.nifi.authorization.user.NiFiUserUtils) PUT(javax.ws.rs.PUT) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) VersionedFlowUpdateRequestEntity(org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity) Authorization(io.swagger.annotations.Authorization) RequestManager(org.apache.nifi.web.api.concurrent.RequestManager) Collections(java.util.Collections) AsynchronousWebRequest(org.apache.nifi.web.api.concurrent.AsynchronousWebRequest) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) ProcessGroupAuthorizable(org.apache.nifi.authorization.ProcessGroupAuthorizable) VersionControlInformationEntity(org.apache.nifi.web.api.entity.VersionControlInformationEntity) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) VersionedFlowUpdateRequestDTO(org.apache.nifi.web.api.dto.VersionedFlowUpdateRequestDTO) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) ConfigurableComponent(org.apache.nifi.components.ConfigurableComponent) RevisionDTO(org.apache.nifi.web.api.dto.RevisionDTO) URISyntaxException(java.net.URISyntaxException) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) IOException(java.io.IOException) VersionedFlowUpdateRequestEntity(org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity) Revision(org.apache.nifi.web.Revision) ComponentLifecycle(org.apache.nifi.web.util.ComponentLifecycle) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) AsynchronousWebRequest(org.apache.nifi.web.api.concurrent.AsynchronousWebRequest) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) Consumes(javax.ws.rs.Consumes) Produces(javax.ws.rs.Produces) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 30 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class VersionsResource method initiateRevertFlowVersion.

@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("revert-requests/process-groups/{id}")
@ApiOperation(value = "Initiate the Revert Request of a Process Group with the given ID", response = VersionedFlowUpdateRequestEntity.class, notes = "For a Process Group that is already under Version Control, this will initiate the action of reverting " + "any local changes that have been made to the Process Group since it was last synchronized with the Flow Registry. This will result in the " + "flow matching the Versioned Flow that exists in the Flow Registry. This can be a lengthy " + "process, as it will stop any Processors and disable any Controller Services necessary to perform the action and then restart them. As a result, " + "the endpoint will immediately return a VersionedFlowUpdateRequestEntity, and the process of updating the flow will occur " + "asynchronously in the background. The client may then periodically poll the status of the request by issuing a GET request to " + "/versions/revert-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE request to " + "/versions/revert-requests/{requestId}. " + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), @Authorization(value = "Read - /{component-type}/{uuid} - For all encapsulated components"), @Authorization(value = "Write - /{component-type}/{uuid} - For all encapsulated components"), @Authorization(value = "Write - if the template contains any restricted components - /restricted-components") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response initiateRevertFlowVersion(@ApiParam("The process group id.") @PathParam("id") final String groupId, @ApiParam(value = "The controller service configuration details.", required = true) final VersionControlInformationEntity requestEntity) throws IOException {
    // Verify the request
    final RevisionDTO revisionDto = requestEntity.getProcessGroupRevision();
    if (revisionDto == null) {
        throw new IllegalArgumentException("Process Group Revision must be specified");
    }
    final VersionControlInformationDTO requestVersionControlInfoDto = requestEntity.getVersionControlInformation();
    if (requestVersionControlInfoDto == null) {
        throw new IllegalArgumentException("Version Control Information must be supplied.");
    }
    if (requestVersionControlInfoDto.getGroupId() == null) {
        throw new IllegalArgumentException("The Process Group ID must be supplied.");
    }
    if (!requestVersionControlInfoDto.getGroupId().equals(groupId)) {
        throw new IllegalArgumentException("The Process Group ID in the request body does not match the Process Group ID of the requested resource.");
    }
    if (requestVersionControlInfoDto.getBucketId() == null) {
        throw new IllegalArgumentException("The Bucket ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getFlowId() == null) {
        throw new IllegalArgumentException("The Flow ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getRegistryId() == null) {
        throw new IllegalArgumentException("The Registry ID must be supplied.");
    }
    if (requestVersionControlInfoDto.getVersion() == null) {
        throw new IllegalArgumentException("The Version of the flow must be supplied.");
    }
    // We will perform the updating of the Versioned Flow in a background thread because it can be a long-running process.
    // In order to do this, we will need some parameters that are only available as Thread-Local variables to the current
    // thread, so we will gather the values for these parameters up front.
    final boolean replicateRequest = isReplicateRequest();
    final ComponentLifecycle componentLifecycle = replicateRequest ? clusterComponentLifecycle : localComponentLifecycle;
    final NiFiUser user = NiFiUserUtils.getNiFiUser();
    // Step 0: Get the Versioned Flow Snapshot from the Flow Registry
    final VersionedFlowSnapshot flowSnapshot = serviceFacade.getVersionedFlowSnapshot(requestEntity.getVersionControlInformation(), true);
    // The flow in the registry may not contain the same versions of components that we have in our flow. As a result, we need to update
    // the flow snapshot to contain compatible bundles.
    BundleUtils.discoverCompatibleBundles(flowSnapshot.getFlowContents());
    // Step 1: Determine which components will be affected by updating the version
    final Set<AffectedComponentEntity> affectedComponents = serviceFacade.getComponentsAffectedByVersionChange(groupId, flowSnapshot, user);
    // build a request wrapper
    final InitiateChangeFlowVersionRequestWrapper requestWrapper = new InitiateChangeFlowVersionRequestWrapper(requestEntity, componentLifecycle, getAbsolutePath(), affectedComponents, replicateRequest, flowSnapshot);
    final Revision requestRevision = getRevision(requestEntity.getProcessGroupRevision(), groupId);
    return withWriteLock(serviceFacade, requestWrapper, requestRevision, lookup -> {
        // Step 2: Verify READ and WRITE permissions for user, for every component.
        final ProcessGroupAuthorizable groupAuthorizable = lookup.getProcessGroup(groupId);
        authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.READ, true, false, true, true);
        authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.WRITE, true, false, true, true);
        final VersionedProcessGroup groupContents = flowSnapshot.getFlowContents();
        final Set<ConfigurableComponent> restrictedComponents = FlowRegistryUtils.getRestrictedComponents(groupContents);
        restrictedComponents.forEach(restrictedComponent -> {
            final ComponentAuthorizable restrictedComponentAuthorizable = lookup.getConfigurableComponent(restrictedComponent);
            authorizeRestrictions(authorizer, restrictedComponentAuthorizable);
        });
    }, () -> {
        // Step 3: Verify that all components in the snapshot exist on all nodes
        // Step 4: Verify that Process Group is already under version control. If not, must start Version Control instead of updating flow
        serviceFacade.verifyCanRevertLocalModifications(groupId, flowSnapshot);
    }, (revision, wrapper) -> {
        final VersionControlInformationEntity versionControlInformationEntity = wrapper.getVersionControlInformationEntity();
        final VersionControlInformationDTO versionControlInformationDTO = versionControlInformationEntity.getVersionControlInformation();
        // Ensure that the information passed in is correct
        final VersionControlInformationEntity currentVersionEntity = serviceFacade.getVersionControlInformation(groupId);
        if (currentVersionEntity == null) {
            throw new IllegalStateException("Process Group cannot be reverted to the previous version of the flow because Process Group is not under Version Control.");
        }
        final VersionControlInformationDTO currentVersion = currentVersionEntity.getVersionControlInformation();
        if (!currentVersion.getBucketId().equals(versionControlInformationDTO.getBucketId())) {
            throw new IllegalArgumentException("The Version Control Information provided does not match the flow that the Process Group is currently synchronized with.");
        }
        if (!currentVersion.getFlowId().equals(versionControlInformationDTO.getFlowId())) {
            throw new IllegalArgumentException("The Version Control Information provided does not match the flow that the Process Group is currently synchronized with.");
        }
        if (!currentVersion.getRegistryId().equals(versionControlInformationDTO.getRegistryId())) {
            throw new IllegalArgumentException("The Version Control Information provided does not match the flow that the Process Group is currently synchronized with.");
        }
        if (!currentVersion.getVersion().equals(versionControlInformationDTO.getVersion())) {
            throw new IllegalArgumentException("The Version Control Information provided does not match the flow that the Process Group is currently synchronized with.");
        }
        final String idGenerationSeed = getIdGenerationSeed().orElse(null);
        // Create an asynchronous request that will occur in the background, because this request may
        // result in stopping components, which can take an indeterminate amount of time.
        final String requestId = UUID.randomUUID().toString();
        final AsynchronousWebRequest<VersionControlInformationEntity> request = new StandardAsynchronousWebRequest<>(requestId, groupId, user, "Stopping Affected Processors");
        // Submit the request to be performed in the background
        final Consumer<AsynchronousWebRequest<VersionControlInformationEntity>> updateTask = vcur -> {
            try {
                final VersionControlInformationEntity updatedVersionControlEntity = updateFlowVersion(groupId, wrapper.getComponentLifecycle(), wrapper.getExampleUri(), wrapper.getAffectedComponents(), user, wrapper.isReplicateRequest(), revision, versionControlInformationEntity, wrapper.getFlowSnapshot(), request, idGenerationSeed, false, true);
                vcur.markComplete(updatedVersionControlEntity);
            } catch (final ResumeFlowException rfe) {
                // Treat ResumeFlowException differently because we don't want to include a message that we couldn't update the flow
                // since in this case the flow was successfully updated - we just couldn't re-enable the components.
                logger.error(rfe.getMessage(), rfe);
                vcur.setFailureReason(rfe.getMessage());
            } catch (final Exception e) {
                logger.error("Failed to update flow to new version", e);
                vcur.setFailureReason("Failed to update flow to new version due to " + e.getMessage());
            }
        };
        requestManager.submitRequest("revert-requests", requestId, request, updateTask);
        // Generate the response.
        final VersionedFlowUpdateRequestDTO updateRequestDto = new VersionedFlowUpdateRequestDTO();
        updateRequestDto.setComplete(request.isComplete());
        updateRequestDto.setFailureReason(request.getFailureReason());
        updateRequestDto.setLastUpdated(request.getLastUpdated());
        updateRequestDto.setProcessGroupId(groupId);
        updateRequestDto.setRequestId(requestId);
        updateRequestDto.setState(request.getState());
        updateRequestDto.setPercentCompleted(request.getPercentComplete());
        updateRequestDto.setUri(generateResourceUri("versions", "revert-requests", requestId));
        final VersionedFlowUpdateRequestEntity updateRequestEntity = new VersionedFlowUpdateRequestEntity();
        final RevisionDTO groupRevision = serviceFacade.getProcessGroup(groupId).getRevision();
        updateRequestEntity.setProcessGroupRevision(groupRevision);
        updateRequestEntity.setRequest(updateRequestDto);
        return generateOkResponse(updateRequestEntity).build();
    });
}
Also used : ComponentAuthorizable(org.apache.nifi.authorization.ComponentAuthorizable) Produces(javax.ws.rs.Produces) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) ApiParam(io.swagger.annotations.ApiParam) ComponentAuthorizable(org.apache.nifi.authorization.ComponentAuthorizable) StringUtils(org.apache.commons.lang3.StringUtils) ClientIdParameter(org.apache.nifi.web.api.request.ClientIdParameter) VersionedFlowSnapshotMetadata(org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) Map(java.util.Map) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) DefaultValue(javax.ws.rs.DefaultValue) AsyncRequestManager(org.apache.nifi.web.api.concurrent.AsyncRequestManager) URI(java.net.URI) NodeResponse(org.apache.nifi.cluster.manager.NodeResponse) VersionedFlowUpdateRequestDTO(org.apache.nifi.web.api.dto.VersionedFlowUpdateRequestDTO) DELETE(javax.ws.rs.DELETE) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) Authorizable(org.apache.nifi.authorization.resource.Authorizable) VersionedFlowDTO(org.apache.nifi.web.api.dto.VersionedFlowDTO) Set(java.util.Set) UUID(java.util.UUID) BundleUtils(org.apache.nifi.util.BundleUtils) LongParameter(org.apache.nifi.web.api.request.LongParameter) Collectors(java.util.stream.Collectors) FlowController(org.apache.nifi.controller.FlowController) CreateActiveRequestEntity(org.apache.nifi.web.api.entity.CreateActiveRequestEntity) Response(javax.ws.rs.core.Response) ScheduledState(org.apache.nifi.controller.ScheduledState) ProcessGroupEntity(org.apache.nifi.web.api.entity.ProcessGroupEntity) VersionedFlowSnapshotEntity(org.apache.nifi.web.api.entity.VersionedFlowSnapshotEntity) ControllerServiceState(org.apache.nifi.controller.service.ControllerServiceState) DtoFactory(org.apache.nifi.web.api.dto.DtoFactory) Entity(org.apache.nifi.web.api.entity.Entity) PathParam(javax.ws.rs.PathParam) Bucket(org.apache.nifi.registry.bucket.Bucket) Revision(org.apache.nifi.web.Revision) GET(javax.ws.rs.GET) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) ConfigurableComponent(org.apache.nifi.components.ConfigurableComponent) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) RevisionDTO(org.apache.nifi.web.api.dto.RevisionDTO) AffectedComponentDTO(org.apache.nifi.web.api.dto.AffectedComponentDTO) ComponentLifecycle(org.apache.nifi.web.util.ComponentLifecycle) HttpMethod(javax.ws.rs.HttpMethod) HashSet(java.util.HashSet) FlowRegistryUtils(org.apache.nifi.registry.flow.FlowRegistryUtils) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) Api(io.swagger.annotations.Api) VersionControlComponentMappingEntity(org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity) CancellableTimedPause(org.apache.nifi.web.util.CancellableTimedPause) Status(javax.ws.rs.core.Response.Status) LinkedHashSet(java.util.LinkedHashSet) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) AffectedComponentUtils(org.apache.nifi.web.util.AffectedComponentUtils) NiFiServiceFacade(org.apache.nifi.web.NiFiServiceFacade) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) RequestAction(org.apache.nifi.authorization.RequestAction) IOException(java.io.IOException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) StartVersionControlRequestEntity(org.apache.nifi.web.api.entity.StartVersionControlRequestEntity) Authorizer(org.apache.nifi.authorization.Authorizer) VersionControlInformationEntity(org.apache.nifi.web.api.entity.VersionControlInformationEntity) ProcessGroupAuthorizable(org.apache.nifi.authorization.ProcessGroupAuthorizable) ApiResponse(io.swagger.annotations.ApiResponse) NiFiUserUtils(org.apache.nifi.authorization.user.NiFiUserUtils) PUT(javax.ws.rs.PUT) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) VersionedFlowUpdateRequestEntity(org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity) Authorization(io.swagger.annotations.Authorization) RequestManager(org.apache.nifi.web.api.concurrent.RequestManager) Collections(java.util.Collections) AsynchronousWebRequest(org.apache.nifi.web.api.concurrent.AsynchronousWebRequest) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) ProcessGroupAuthorizable(org.apache.nifi.authorization.ProcessGroupAuthorizable) VersionControlInformationEntity(org.apache.nifi.web.api.entity.VersionControlInformationEntity) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) VersionedFlowUpdateRequestDTO(org.apache.nifi.web.api.dto.VersionedFlowUpdateRequestDTO) AffectedComponentEntity(org.apache.nifi.web.api.entity.AffectedComponentEntity) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) ConfigurableComponent(org.apache.nifi.components.ConfigurableComponent) RevisionDTO(org.apache.nifi.web.api.dto.RevisionDTO) URISyntaxException(java.net.URISyntaxException) ResourceNotFoundException(org.apache.nifi.web.ResourceNotFoundException) LifecycleManagementException(org.apache.nifi.web.util.LifecycleManagementException) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) ResumeFlowException(org.apache.nifi.web.ResumeFlowException) IOException(java.io.IOException) VersionedFlowUpdateRequestEntity(org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity) Revision(org.apache.nifi.web.Revision) ComponentLifecycle(org.apache.nifi.web.util.ComponentLifecycle) StandardAsynchronousWebRequest(org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest) AsynchronousWebRequest(org.apache.nifi.web.api.concurrent.AsynchronousWebRequest) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) Consumes(javax.ws.rs.Consumes) Produces(javax.ws.rs.Produces) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Aggregations

VersionedFlowSnapshot (org.apache.nifi.registry.flow.VersionedFlowSnapshot)38 VersionedFlowSnapshotMetadata (org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata)18 VersionedProcessGroup (org.apache.nifi.registry.flow.VersionedProcessGroup)16 VersionedFlow (org.apache.nifi.registry.flow.VersionedFlow)15 Bucket (org.apache.nifi.registry.bucket.Bucket)12 IOException (java.io.IOException)11 Date (java.util.Date)11 Test (org.junit.Test)11 ApiOperation (io.swagger.annotations.ApiOperation)8 ApiResponses (io.swagger.annotations.ApiResponses)8 Consumes (javax.ws.rs.Consumes)8 Path (javax.ws.rs.Path)8 Produces (javax.ws.rs.Produces)8 Authorizable (org.apache.nifi.authorization.resource.Authorizable)8 VersionedFlowState (org.apache.nifi.registry.flow.VersionedFlowState)8 RevisionDTO (org.apache.nifi.web.api.dto.RevisionDTO)8 VersionControlInformationDTO (org.apache.nifi.web.api.dto.VersionControlInformationDTO)8 HashMap (java.util.HashMap)7 NiFiRegistryException (org.apache.nifi.registry.client.NiFiRegistryException)7 Collections (java.util.Collections)6