use of org.apache.nifi.registry.flow.VersionedFlow in project nifi by apache.
the class StandardNiFiServiceFacade method registerFlowWithFlowRegistry.
@Override
public VersionControlComponentMappingEntity registerFlowWithFlowRegistry(final String groupId, final StartVersionControlRequestEntity requestEntity) {
final ProcessGroup processGroup = processGroupDAO.getProcessGroup(groupId);
final VersionControlInformation currentVci = processGroup.getVersionControlInformation();
final int expectedVersion = currentVci == null ? 1 : currentVci.getVersion() + 1;
// Create a VersionedProcessGroup snapshot of the flow as it is currently.
final InstantiatedVersionedProcessGroup versionedProcessGroup = createFlowSnapshot(groupId);
final VersionedFlowDTO versionedFlowDto = requestEntity.getVersionedFlow();
final String flowId = versionedFlowDto.getFlowId() == null ? UUID.randomUUID().toString() : versionedFlowDto.getFlowId();
final VersionedFlow versionedFlow = new VersionedFlow();
versionedFlow.setBucketIdentifier(versionedFlowDto.getBucketId());
versionedFlow.setCreatedTimestamp(System.currentTimeMillis());
versionedFlow.setDescription(versionedFlowDto.getDescription());
versionedFlow.setModifiedTimestamp(versionedFlow.getCreatedTimestamp());
versionedFlow.setName(versionedFlowDto.getFlowName());
versionedFlow.setIdentifier(flowId);
// Add the Versioned Flow and first snapshot to the Flow Registry
final String registryId = requestEntity.getVersionedFlow().getRegistryId();
final VersionedFlowSnapshot registeredSnapshot;
final VersionedFlow registeredFlow;
String action = "create the flow";
try {
// first, create the flow in the registry, if necessary
if (versionedFlowDto.getFlowId() == null) {
registeredFlow = registerVersionedFlow(registryId, versionedFlow);
} else {
registeredFlow = getVersionedFlow(registryId, versionedFlowDto.getBucketId(), versionedFlowDto.getFlowId());
}
action = "add the local flow to the Flow Registry as the first Snapshot";
// add first snapshot to the flow in the registry
registeredSnapshot = registerVersionedFlowSnapshot(registryId, registeredFlow, versionedProcessGroup, versionedFlowDto.getComments(), expectedVersion);
} catch (final NiFiRegistryException e) {
throw new IllegalArgumentException(e.getLocalizedMessage());
} catch (final IOException ioe) {
throw new IllegalStateException("Failed to communicate with Flow Registry when attempting to " + action);
}
final Bucket bucket = registeredSnapshot.getBucket();
final VersionedFlow flow = registeredSnapshot.getFlow();
// Update the Process Group with the new VersionControlInformation. (Send this to all nodes).
final VersionControlInformationDTO vci = new VersionControlInformationDTO();
vci.setBucketId(bucket.getIdentifier());
vci.setBucketName(bucket.getName());
vci.setFlowId(flow.getIdentifier());
vci.setFlowName(flow.getName());
vci.setFlowDescription(flow.getDescription());
vci.setGroupId(groupId);
vci.setRegistryId(registryId);
vci.setRegistryName(getFlowRegistryName(registryId));
vci.setVersion(registeredSnapshot.getSnapshotMetadata().getVersion());
vci.setState(VersionedFlowState.UP_TO_DATE.name());
final Map<String, String> mapping = dtoFactory.createVersionControlComponentMappingDto(versionedProcessGroup);
final Revision groupRevision = revisionManager.getRevision(groupId);
final RevisionDTO groupRevisionDto = dtoFactory.createRevisionDTO(groupRevision);
final VersionControlComponentMappingEntity entity = new VersionControlComponentMappingEntity();
entity.setVersionControlInformation(vci);
entity.setProcessGroupRevision(groupRevisionDto);
entity.setVersionControlComponentMapping(mapping);
return entity;
}
use of org.apache.nifi.registry.flow.VersionedFlow in project nifi by apache.
the class VersionsResource method updateFlowVersion.
private VersionControlInformationEntity updateFlowVersion(final String groupId, final ComponentLifecycle componentLifecycle, final URI exampleUri, final Set<AffectedComponentEntity> affectedComponents, final NiFiUser user, final boolean replicateRequest, final Revision revision, final VersionControlInformationEntity requestEntity, final VersionedFlowSnapshot flowSnapshot, final AsynchronousWebRequest<VersionControlInformationEntity> asyncRequest, final String idGenerationSeed, final boolean verifyNotModified, final boolean updateDescendantVersionedFlows) throws LifecycleManagementException, ResumeFlowException {
// Steps 6-7: Determine which components must be stopped and stop them.
final Set<String> stoppableReferenceTypes = new HashSet<>();
stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_PROCESSOR);
stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT);
stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT);
stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_INPUT_PORT);
stoppableReferenceTypes.add(AffectedComponentDTO.COMPONENT_TYPE_OUTPUT_PORT);
final Set<AffectedComponentEntity> runningComponents = affectedComponents.stream().filter(dto -> stoppableReferenceTypes.contains(dto.getComponent().getReferenceType())).filter(dto -> "Running".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
logger.info("Stopping {} Processors", runningComponents.size());
final CancellableTimedPause stopComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
asyncRequest.setCancelCallback(stopComponentsPause::cancel);
componentLifecycle.scheduleComponents(exampleUri, user, groupId, runningComponents, ScheduledState.STOPPED, stopComponentsPause);
if (asyncRequest.isCancelled()) {
return null;
}
asyncRequest.update(new Date(), "Disabling Affected Controller Services", 20);
// Steps 8-9. Disable enabled controller services that are affected
final Set<AffectedComponentEntity> enabledServices = affectedComponents.stream().filter(dto -> AffectedComponentDTO.COMPONENT_TYPE_CONTROLLER_SERVICE.equals(dto.getComponent().getReferenceType())).filter(dto -> "Enabled".equalsIgnoreCase(dto.getComponent().getState())).collect(Collectors.toSet());
logger.info("Disabling {} Controller Services", enabledServices.size());
final CancellableTimedPause disableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
asyncRequest.setCancelCallback(disableServicesPause::cancel);
componentLifecycle.activateControllerServices(exampleUri, user, groupId, enabledServices, ControllerServiceState.DISABLED, disableServicesPause);
if (asyncRequest.isCancelled()) {
return null;
}
asyncRequest.update(new Date(), "Updating Flow", 40);
logger.info("Updating Process Group with ID {} to version {} of the Versioned Flow", groupId, flowSnapshot.getSnapshotMetadata().getVersion());
// by replicating a PUT to /nifi-api/versions/process-groups/{groupId}
try {
if (replicateRequest) {
final URI updateUri;
try {
updateUri = new URI(exampleUri.getScheme(), exampleUri.getUserInfo(), exampleUri.getHost(), exampleUri.getPort(), "/nifi-api/versions/process-groups/" + groupId, null, exampleUri.getFragment());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
final Map<String, String> headers = new HashMap<>();
headers.put("content-type", MediaType.APPLICATION_JSON);
final VersionedFlowSnapshotEntity snapshotEntity = new VersionedFlowSnapshotEntity();
snapshotEntity.setProcessGroupRevision(dtoFactory.createRevisionDTO(revision));
snapshotEntity.setRegistryId(requestEntity.getVersionControlInformation().getRegistryId());
snapshotEntity.setVersionedFlow(flowSnapshot);
snapshotEntity.setUpdateDescendantVersionedFlows(updateDescendantVersionedFlows);
final NodeResponse clusterResponse;
try {
logger.debug("Replicating PUT request to {} for user {}", updateUri, user);
if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
clusterResponse = getRequestReplicator().replicate(user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
} else {
clusterResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), user, HttpMethod.PUT, updateUri, snapshotEntity, headers).awaitMergedResponse();
}
} catch (final InterruptedException ie) {
logger.warn("Interrupted while replicating PUT request to {} for user {}", updateUri, user);
Thread.currentThread().interrupt();
throw new LifecycleManagementException("Interrupted while updating flows across cluster", ie);
}
final int updateFlowStatus = clusterResponse.getStatus();
if (updateFlowStatus != Status.OK.getStatusCode()) {
final String explanation = getResponseEntity(clusterResponse, String.class);
logger.error("Failed to update flow across cluster when replicating PUT request to {} for user {}. Received {} response with explanation: {}", updateUri, user, updateFlowStatus, explanation);
throw new LifecycleManagementException("Failed to update Flow on all nodes in cluster due to " + explanation);
}
} else {
// Step 10: Ensure that if any connection exists in the flow and does not exist in the proposed snapshot,
// that it has no data in it. Ensure that no Input Port was removed, unless it currently has no incoming connections.
// Ensure that no Output Port was removed, unless it currently has no outgoing connections.
serviceFacade.verifyCanUpdate(groupId, flowSnapshot, true, verifyNotModified);
// Step 11-12. Update Process Group to the new flow and update variable registry with any Variables that were added or removed
final VersionControlInformationDTO requestVci = requestEntity.getVersionControlInformation();
final Bucket bucket = flowSnapshot.getBucket();
final VersionedFlow flow = flowSnapshot.getFlow();
final VersionedFlowSnapshotMetadata metadata = flowSnapshot.getSnapshotMetadata();
final VersionControlInformationDTO vci = new VersionControlInformationDTO();
vci.setBucketId(metadata.getBucketIdentifier());
vci.setBucketName(bucket.getName());
vci.setFlowDescription(flow.getDescription());
vci.setFlowId(flow.getIdentifier());
vci.setFlowName(flow.getName());
vci.setGroupId(groupId);
vci.setRegistryId(requestVci.getRegistryId());
vci.setRegistryName(serviceFacade.getFlowRegistryName(requestVci.getRegistryId()));
vci.setVersion(metadata.getVersion());
vci.setState(flowSnapshot.isLatest() ? VersionedFlowState.UP_TO_DATE.name() : VersionedFlowState.STALE.name());
serviceFacade.updateProcessGroupContents(user, revision, groupId, vci, flowSnapshot, idGenerationSeed, verifyNotModified, false, updateDescendantVersionedFlows);
}
} finally {
if (!asyncRequest.isCancelled()) {
if (logger.isDebugEnabled()) {
logger.debug("Re-Enabling {} Controller Services: {}", enabledServices.size(), enabledServices);
}
asyncRequest.update(new Date(), "Re-Enabling Controller Services", 60);
// Step 13. Re-enable all disabled controller services
final CancellableTimedPause enableServicesPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
asyncRequest.setCancelCallback(enableServicesPause::cancel);
final Set<AffectedComponentEntity> servicesToEnable = getUpdatedEntities(enabledServices, user);
logger.info("Successfully updated flow; re-enabling {} Controller Services", servicesToEnable.size());
try {
componentLifecycle.activateControllerServices(exampleUri, user, groupId, servicesToEnable, ControllerServiceState.ENABLED, enableServicesPause);
} catch (final IllegalStateException ise) {
// a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
throw new ResumeFlowException("Failed to re-enable Controller Services because " + ise.getMessage(), ise);
}
}
if (!asyncRequest.isCancelled()) {
if (logger.isDebugEnabled()) {
logger.debug("Restart {} Processors: {}", runningComponents.size(), runningComponents);
}
asyncRequest.update(new Date(), "Restarting Processors", 80);
// Step 14. Restart all components
final Set<AffectedComponentEntity> componentsToStart = getUpdatedEntities(runningComponents, user);
// If there are any Remote Group Ports that are supposed to be started and have no connections, we want to remove those from our Set.
// This will happen if the Remote Group Port is transmitting when the version change happens but the new flow version does not have
// a connection to the port. In such a case, the Port still is included in the Updated Entities because we do not remove them
// when updating the flow (they are removed in the background).
final Set<AffectedComponentEntity> avoidStarting = new HashSet<>();
for (final AffectedComponentEntity componentEntity : componentsToStart) {
final AffectedComponentDTO componentDto = componentEntity.getComponent();
final String referenceType = componentDto.getReferenceType();
if (!AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT.equals(referenceType) && !AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT.equals(referenceType)) {
continue;
}
boolean startComponent;
try {
startComponent = serviceFacade.isRemoteGroupPortConnected(componentDto.getProcessGroupId(), componentDto.getId());
} catch (final ResourceNotFoundException rnfe) {
// Could occur if RPG is refreshed at just the right time.
startComponent = false;
}
// rather than removing the component here, because doing so would result in a ConcurrentModificationException.
if (!startComponent) {
avoidStarting.add(componentEntity);
}
}
componentsToStart.removeAll(avoidStarting);
final CancellableTimedPause startComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
asyncRequest.setCancelCallback(startComponentsPause::cancel);
logger.info("Restarting {} Processors", componentsToStart.size());
try {
componentLifecycle.scheduleComponents(exampleUri, user, groupId, componentsToStart, ScheduledState.RUNNING, startComponentsPause);
} catch (final IllegalStateException ise) {
// a more intelligent error message as to exactly what happened, rather than indicate that the flow could not be updated.
throw new ResumeFlowException("Failed to restart components because " + ise.getMessage(), ise);
}
}
}
asyncRequest.setCancelCallback(null);
if (asyncRequest.isCancelled()) {
return null;
}
asyncRequest.update(new Date(), "Complete", 100);
return serviceFacade.getVersionControlInformation(groupId);
}
use of org.apache.nifi.registry.flow.VersionedFlow in project nifi by apache.
the class ProcessGroupResource method createProcessGroup.
/**
* Adds the specified process group.
*
* @param httpServletRequest request
* @param groupId The group id
* @param requestProcessGroupEntity A processGroupEntity
* @return A processGroupEntity
* @throws IOException if the request indicates that the Process Group should be imported from a Flow Registry and NiFi is unable to communicate with the Flow Registry
*/
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("{id}/process-groups")
@ApiOperation(value = "Creates a process group", response = ProcessGroupEntity.class, authorizations = { @Authorization(value = "Write - /process-groups/{uuid}") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response createProcessGroup(@Context final HttpServletRequest httpServletRequest, @ApiParam(value = "The process group id.", required = true) @PathParam("id") final String groupId, @ApiParam(value = "The process group configuration details.", required = true) final ProcessGroupEntity requestProcessGroupEntity) throws IOException {
if (requestProcessGroupEntity == null || requestProcessGroupEntity.getComponent() == null) {
throw new IllegalArgumentException("Process group details must be specified.");
}
if (requestProcessGroupEntity.getRevision() == null || (requestProcessGroupEntity.getRevision().getVersion() == null || requestProcessGroupEntity.getRevision().getVersion() != 0)) {
throw new IllegalArgumentException("A revision of 0 must be specified when creating a new Process group.");
}
if (requestProcessGroupEntity.getComponent().getId() != null) {
throw new IllegalArgumentException("Process group ID cannot be specified.");
}
final PositionDTO proposedPosition = requestProcessGroupEntity.getComponent().getPosition();
if (proposedPosition != null) {
if (proposedPosition.getX() == null || proposedPosition.getY() == null) {
throw new IllegalArgumentException("The x and y coordinate of the proposed position must be specified.");
}
}
// if the group name isn't specified, ensure the group is being imported from version control
if (StringUtils.isBlank(requestProcessGroupEntity.getComponent().getName()) && requestProcessGroupEntity.getComponent().getVersionControlInformation() == null) {
throw new IllegalArgumentException("The group name is required when the group is not imported from version control.");
}
if (requestProcessGroupEntity.getComponent().getParentGroupId() != null && !groupId.equals(requestProcessGroupEntity.getComponent().getParentGroupId())) {
throw new IllegalArgumentException(String.format("If specified, the parent process group id %s must be the same as specified in the URI %s", requestProcessGroupEntity.getComponent().getParentGroupId(), groupId));
}
requestProcessGroupEntity.getComponent().setParentGroupId(groupId);
// Step 1: Ensure that user has write permissions to the Process Group. If not, then immediately fail.
// Step 2: Retrieve flow from Flow Registry
// Step 3: Resolve Bundle info
// Step 4: Update contents of the ProcessGroupDTO passed in to include the components that need to be added.
// Step 5: If any of the components is a Restricted Component, then we must authorize the user
// for write access to the RestrictedComponents resource
// Step 6: Replicate the request or call serviceFacade.updateProcessGroup
final VersionControlInformationDTO versionControlInfo = requestProcessGroupEntity.getComponent().getVersionControlInformation();
if (versionControlInfo != null && requestProcessGroupEntity.getVersionedFlowSnapshot() == null) {
// Step 1: Ensure that user has write permissions to the Process Group. If not, then immediately fail.
// Step 2: Retrieve flow from Flow Registry
final VersionedFlowSnapshot flowSnapshot = serviceFacade.getVersionedFlowSnapshot(versionControlInfo, true);
final Bucket bucket = flowSnapshot.getBucket();
final VersionedFlow flow = flowSnapshot.getFlow();
versionControlInfo.setBucketName(bucket.getName());
versionControlInfo.setFlowName(flow.getName());
versionControlInfo.setFlowDescription(flow.getDescription());
versionControlInfo.setRegistryName(serviceFacade.getFlowRegistryName(versionControlInfo.getRegistryId()));
final VersionedFlowState flowState = flowSnapshot.isLatest() ? VersionedFlowState.UP_TO_DATE : VersionedFlowState.STALE;
versionControlInfo.setState(flowState.name());
// Step 3: Resolve Bundle info
BundleUtils.discoverCompatibleBundles(flowSnapshot.getFlowContents());
// Step 4: Update contents of the ProcessGroupDTO passed in to include the components that need to be added.
requestProcessGroupEntity.setVersionedFlowSnapshot(flowSnapshot);
}
if (versionControlInfo != null) {
final VersionedFlowSnapshot flowSnapshot = requestProcessGroupEntity.getVersionedFlowSnapshot();
serviceFacade.verifyImportProcessGroup(versionControlInfo, flowSnapshot.getFlowContents(), groupId);
}
// Step 6: Replicate the request or call serviceFacade.updateProcessGroup
if (isReplicateRequest()) {
return replicate(HttpMethod.POST, requestProcessGroupEntity);
}
return withWriteLock(serviceFacade, requestProcessGroupEntity, lookup -> {
final Authorizable processGroup = lookup.getProcessGroup(groupId).getAuthorizable();
processGroup.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser());
// Step 5: If any of the components is a Restricted Component, then we must authorize the user
// for write access to the RestrictedComponents resource
final VersionedFlowSnapshot versionedFlowSnapshot = requestProcessGroupEntity.getVersionedFlowSnapshot();
if (versionedFlowSnapshot != null) {
final Set<ConfigurableComponent> restrictedComponents = FlowRegistryUtils.getRestrictedComponents(versionedFlowSnapshot.getFlowContents());
restrictedComponents.forEach(restrictedComponent -> {
final ComponentAuthorizable restrictedComponentAuthorizable = lookup.getConfigurableComponent(restrictedComponent);
authorizeRestrictions(authorizer, restrictedComponentAuthorizable);
});
}
}, () -> {
final VersionedFlowSnapshot versionedFlowSnapshot = requestProcessGroupEntity.getVersionedFlowSnapshot();
if (versionedFlowSnapshot != null) {
serviceFacade.verifyComponentTypes(versionedFlowSnapshot.getFlowContents());
}
}, processGroupEntity -> {
final ProcessGroupDTO processGroup = processGroupEntity.getComponent();
// set the processor id as appropriate
processGroup.setId(generateUuid());
// ensure the group name comes from the versioned flow
final VersionedFlowSnapshot flowSnapshot = processGroupEntity.getVersionedFlowSnapshot();
if (flowSnapshot != null && StringUtils.isNotBlank(flowSnapshot.getFlowContents().getName()) && StringUtils.isBlank(processGroup.getName())) {
processGroup.setName(flowSnapshot.getFlowContents().getName());
}
// create the process group contents
final Revision revision = getRevision(processGroupEntity, processGroup.getId());
ProcessGroupEntity entity = serviceFacade.createProcessGroup(revision, groupId, processGroup);
if (flowSnapshot != null) {
final RevisionDTO revisionDto = entity.getRevision();
final String newGroupId = entity.getComponent().getId();
final Revision newGroupRevision = new Revision(revisionDto.getVersion(), revisionDto.getClientId(), newGroupId);
// We don't want the Process Group's position to be updated because we want to keep the position where the user
// placed the Process Group. However, we do want to use the name of the Process Group that is in the Flow Contents.
// To accomplish this, we call updateProcessGroupContents() passing 'true' for the updateSettings flag but null out the position.
flowSnapshot.getFlowContents().setPosition(null);
entity = serviceFacade.updateProcessGroupContents(NiFiUserUtils.getNiFiUser(), newGroupRevision, newGroupId, versionControlInfo, flowSnapshot, getIdGenerationSeed().orElse(null), false, true, true);
}
populateRemainingProcessGroupEntityContent(entity);
// generate a 201 created response
String uri = entity.getUri();
return generateCreatedResponse(URI.create(uri), entity).build();
});
}
use of org.apache.nifi.registry.flow.VersionedFlow in project nifi by apache.
the class CreateFlow method doExecute.
@Override
public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException {
final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID);
final String flowName = getRequiredArg(properties, CommandOption.FLOW_NAME);
final String flowDesc = getArg(properties, CommandOption.FLOW_DESC);
final VersionedFlow flow = new VersionedFlow();
flow.setName(flowName);
flow.setDescription(flowDesc);
flow.setBucketIdentifier(bucketId);
final FlowClient flowClient = client.getFlowClient();
final VersionedFlow createdFlow = flowClient.create(flow);
return new StringResult(createdFlow.getIdentifier(), getContext().isInteractive());
}
use of org.apache.nifi.registry.flow.VersionedFlow in project nifi by apache.
the class ListFlows method doExecute.
@Override
public VersionedFlowsResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException {
final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID);
final FlowClient flowClient = client.getFlowClient();
final List<VersionedFlow> flows = flowClient.getByBucket(bucketId);
return new VersionedFlowsResult(getResultType(properties), flows);
}
Aggregations