use of org.apache.nifi.web.api.dto.PortDTO in project nifi by apache.
the class SnippetUtils method populateFlowSnippet.
/**
* Populates the specified snippet and returns the details.
*
* @param snippet snippet
* @param recurse recurse
* @param includeControllerServices whether or not to include controller services in the flow snippet dto
* @return snippet
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public FlowSnippetDTO populateFlowSnippet(final Snippet snippet, final boolean recurse, final boolean includeControllerServices, boolean removeInstanceId) {
final FlowSnippetDTO snippetDto = new FlowSnippetDTO(removeInstanceId);
final String groupId = snippet.getParentGroupId();
final ProcessGroup processGroup = flowController.getGroup(groupId);
// ensure the group could be found
if (processGroup == null) {
throw new IllegalStateException("The parent process group for this snippet could not be found.");
}
// We need to ensure that the Controller Services that are added get added to the proper group.
// This can potentially get a little bit tricky. Consider this scenario:
// We have a Process Group G1. Within Process Group G1 is a Controller Service C1.
// Also within G1 is a child Process Group, G2. Within G2 is a child Process Group, G3.
// Within G3 are two child Process Groups: G4 and G5. Within each of these children,
// we have a Processor (P1, P2) that references the Controller Service C1, defined 3 levels above.
// Now, we create a template that encompasses only Process Groups G4 and G5. We need to ensure
// that the Controller Service C1 is included at the 'root' of the template so that those
// Processors within G4 and G5 both have access to the same Controller Service. This can be drawn
// out thus:
//
// G1 -- C1
// |
// |
// G2
// |
// |
// G3
// | \
// | \
// G4 G5
// | |
// | |
// P1 P2
//
// Both P1 and P2 reference C1.
//
// In order to accomplish this, we maintain two collections. First, we keep a Set of all Controller Services that have
// been added. If we add a new Controller Service to the set, then we know it hasn't been added anywhere in the Snippet.
// In that case, we determine the service's group ID. In the flow described above, if we template just groups G4 and G5,
// then we need to include the Controller Service defined at G1. So we also keep a Map of Group ID to controller services
// in that group. If the ParentGroupId of a Controller Service is not in our snippet, then we instead update the parent
// ParentGroupId to be that of our highest-level process group (in this case G3, as that's where the template is created)
// and then add the controller services to that group (NOTE: here, when we say we change the group ID and add to that group,
// we are talking only about the DTO objects that make up the snippet. We do not actually modify the Process Group or the
// Controller Services in our flow themselves!)
final Set<ControllerServiceDTO> allServicesReferenced = new HashSet<>();
final Map<String, FlowSnippetDTO> contentsByGroup = new HashMap<>();
contentsByGroup.put(processGroup.getIdentifier(), snippetDto);
// add any processors
final Set<ControllerServiceDTO> controllerServices = new HashSet<>();
final Set<ProcessorDTO> processors = new LinkedHashSet<>();
if (!snippet.getProcessors().isEmpty()) {
for (final String processorId : snippet.getProcessors().keySet()) {
final ProcessorNode processor = processGroup.getProcessor(processorId);
if (processor == null) {
throw new IllegalStateException("A processor in this snippet could not be found.");
}
processors.add(dtoFactory.createProcessorDto(processor));
if (includeControllerServices) {
// Include all referenced services that are not already included in this snippet.
getControllerServices(processor.getProperties()).stream().filter(svc -> allServicesReferenced.add(svc)).forEach(svc -> {
final String svcGroupId = svc.getParentGroupId();
final String destinationGroupId = contentsByGroup.containsKey(svcGroupId) ? svcGroupId : processGroup.getIdentifier();
svc.setParentGroupId(destinationGroupId);
controllerServices.add(svc);
});
}
}
}
// add any connections
final Set<ConnectionDTO> connections = new LinkedHashSet<>();
if (!snippet.getConnections().isEmpty()) {
for (final String connectionId : snippet.getConnections().keySet()) {
final Connection connection = processGroup.getConnection(connectionId);
if (connection == null) {
throw new IllegalStateException("A connection in this snippet could not be found.");
}
connections.add(dtoFactory.createConnectionDto(connection));
}
}
// add any funnels
final Set<FunnelDTO> funnels = new LinkedHashSet<>();
if (!snippet.getFunnels().isEmpty()) {
for (final String funnelId : snippet.getFunnels().keySet()) {
final Funnel funnel = processGroup.getFunnel(funnelId);
if (funnel == null) {
throw new IllegalStateException("A funnel in this snippet could not be found.");
}
funnels.add(dtoFactory.createFunnelDto(funnel));
}
}
// add any input ports
final Set<PortDTO> inputPorts = new LinkedHashSet<>();
if (!snippet.getInputPorts().isEmpty()) {
for (final String inputPortId : snippet.getInputPorts().keySet()) {
final Port inputPort = processGroup.getInputPort(inputPortId);
if (inputPort == null) {
throw new IllegalStateException("An input port in this snippet could not be found.");
}
inputPorts.add(dtoFactory.createPortDto(inputPort));
}
}
// add any labels
final Set<LabelDTO> labels = new LinkedHashSet<>();
if (!snippet.getLabels().isEmpty()) {
for (final String labelId : snippet.getLabels().keySet()) {
final Label label = processGroup.getLabel(labelId);
if (label == null) {
throw new IllegalStateException("A label in this snippet could not be found.");
}
labels.add(dtoFactory.createLabelDto(label));
}
}
// add any output ports
final Set<PortDTO> outputPorts = new LinkedHashSet<>();
if (!snippet.getOutputPorts().isEmpty()) {
for (final String outputPortId : snippet.getOutputPorts().keySet()) {
final Port outputPort = processGroup.getOutputPort(outputPortId);
if (outputPort == null) {
throw new IllegalStateException("An output port in this snippet could not be found.");
}
outputPorts.add(dtoFactory.createPortDto(outputPort));
}
}
// add any process groups
final Set<ProcessGroupDTO> processGroups = new LinkedHashSet<>();
if (!snippet.getProcessGroups().isEmpty()) {
for (final String childGroupId : snippet.getProcessGroups().keySet()) {
final ProcessGroup childGroup = processGroup.getProcessGroup(childGroupId);
if (childGroup == null) {
throw new IllegalStateException("A process group in this snippet could not be found.");
}
final ProcessGroupDTO childGroupDto = dtoFactory.createProcessGroupDto(childGroup, recurse);
processGroups.add(childGroupDto);
// maintain a listing of visited groups starting with each group in the snippet. this is used to determine
// whether a referenced controller service should be included in the resulting snippet. if the service is
// defined at groupId or one of it's ancestors, its considered outside of this snippet and will only be included
// when the includeControllerServices is set to true. this happens above when considering the processors in this snippet
final Set<String> visitedGroupIds = new HashSet<>();
addControllerServices(childGroup, childGroupDto, allServicesReferenced, includeControllerServices, visitedGroupIds, contentsByGroup, processGroup.getIdentifier());
}
}
// add any remote process groups
final Set<RemoteProcessGroupDTO> remoteProcessGroups = new LinkedHashSet<>();
if (!snippet.getRemoteProcessGroups().isEmpty()) {
for (final String remoteProcessGroupId : snippet.getRemoteProcessGroups().keySet()) {
final RemoteProcessGroup remoteProcessGroup = processGroup.getRemoteProcessGroup(remoteProcessGroupId);
if (remoteProcessGroup == null) {
throw new IllegalStateException("A remote process group in this snippet could not be found.");
}
remoteProcessGroups.add(dtoFactory.createRemoteProcessGroupDto(remoteProcessGroup));
}
}
// Normalize the coordinates based on the locations of the other components
final List<? extends ComponentDTO> components = new ArrayList<>();
components.addAll((Set) processors);
components.addAll((Set) connections);
components.addAll((Set) funnels);
components.addAll((Set) inputPorts);
components.addAll((Set) labels);
components.addAll((Set) outputPorts);
components.addAll((Set) processGroups);
components.addAll((Set) remoteProcessGroups);
normalizeCoordinates(components);
Set<ControllerServiceDTO> updatedControllerServices = snippetDto.getControllerServices();
if (updatedControllerServices == null) {
updatedControllerServices = new HashSet<>();
}
updatedControllerServices.addAll(controllerServices);
snippetDto.setControllerServices(updatedControllerServices);
snippetDto.setProcessors(processors);
snippetDto.setConnections(connections);
snippetDto.setFunnels(funnels);
snippetDto.setInputPorts(inputPorts);
snippetDto.setLabels(labels);
snippetDto.setOutputPorts(outputPorts);
snippetDto.setProcessGroups(processGroups);
snippetDto.setRemoteProcessGroups(remoteProcessGroups);
return snippetDto;
}
use of org.apache.nifi.web.api.dto.PortDTO in project nifi by apache.
the class SnippetUtils method copyContentsForGroup.
private FlowSnippetDTO copyContentsForGroup(final FlowSnippetDTO snippetContents, final String groupId, final Map<String, ConnectableDTO> parentConnectableMap, Map<String, String> serviceIdMap, final String idGenerationSeed, boolean isCopy) {
final FlowSnippetDTO snippetContentsCopy = new FlowSnippetDTO();
try {
//
if (serviceIdMap == null) {
serviceIdMap = new HashMap<>();
}
final Set<ControllerServiceDTO> services = new HashSet<>();
if (snippetContents.getControllerServices() != null) {
for (final ControllerServiceDTO serviceDTO : snippetContents.getControllerServices()) {
final ControllerServiceDTO service = dtoFactory.copy(serviceDTO);
service.setId(generateId(serviceDTO.getId(), idGenerationSeed, isCopy));
service.setState(ControllerServiceState.DISABLED.name());
services.add(service);
// Map old service ID to new service ID so that we can make sure that we reference the new ones.
serviceIdMap.put(serviceDTO.getId(), service.getId());
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.ControllerService, serviceDTO.getId(), serviceDTO.getName()), ResourceFactory.getComponentResource(ResourceType.ControllerService, service.getId(), service.getName()), idGenerationSeed);
}
}
}
// if there is any controller service that maps to another controller service, update the id's
for (final ControllerServiceDTO serviceDTO : services) {
final Map<String, String> properties = serviceDTO.getProperties();
final Map<String, PropertyDescriptorDTO> descriptors = serviceDTO.getDescriptors();
if (properties != null && descriptors != null) {
for (final PropertyDescriptorDTO descriptor : descriptors.values()) {
if (descriptor.getIdentifiesControllerService() != null) {
final String currentServiceId = properties.get(descriptor.getName());
if (currentServiceId == null) {
continue;
}
final String newServiceId = serviceIdMap.get(currentServiceId);
properties.put(descriptor.getName(), newServiceId);
}
}
}
}
snippetContentsCopy.setControllerServices(services);
//
// Copy the labels
//
final Set<LabelDTO> labels = new HashSet<>();
if (snippetContents.getLabels() != null) {
for (final LabelDTO labelDTO : snippetContents.getLabels()) {
final LabelDTO label = dtoFactory.copy(labelDTO);
label.setId(generateId(labelDTO.getId(), idGenerationSeed, isCopy));
label.setParentGroupId(groupId);
labels.add(label);
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.Label, labelDTO.getId(), labelDTO.getLabel()), ResourceFactory.getComponentResource(ResourceType.Label, label.getId(), label.getLabel()), idGenerationSeed);
}
}
}
snippetContentsCopy.setLabels(labels);
//
// Copy connectable components
//
// maps a group ID-ID of a Connectable in the template to the new instance
final Map<String, ConnectableDTO> connectableMap = new HashMap<>();
//
// Copy the funnels
//
final Set<FunnelDTO> funnels = new HashSet<>();
if (snippetContents.getFunnels() != null) {
for (final FunnelDTO funnelDTO : snippetContents.getFunnels()) {
final FunnelDTO cp = dtoFactory.copy(funnelDTO);
cp.setId(generateId(funnelDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
funnels.add(cp);
connectableMap.put(funnelDTO.getParentGroupId() + "-" + funnelDTO.getId(), dtoFactory.createConnectableDto(cp));
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.Funnel, funnelDTO.getId(), funnelDTO.getId()), ResourceFactory.getComponentResource(ResourceType.Funnel, cp.getId(), cp.getId()), idGenerationSeed);
}
}
}
snippetContentsCopy.setFunnels(funnels);
final Set<PortDTO> inputPorts = new HashSet<>();
if (snippetContents.getInputPorts() != null) {
for (final PortDTO portDTO : snippetContents.getInputPorts()) {
final PortDTO cp = dtoFactory.copy(portDTO);
cp.setId(generateId(portDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
cp.setState(ScheduledState.STOPPED.toString());
inputPorts.add(cp);
final ConnectableDTO portConnectable = dtoFactory.createConnectableDto(cp, ConnectableType.INPUT_PORT);
connectableMap.put(portDTO.getParentGroupId() + "-" + portDTO.getId(), portConnectable);
if (parentConnectableMap != null) {
parentConnectableMap.put(portDTO.getParentGroupId() + "-" + portDTO.getId(), portConnectable);
}
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.InputPort, portDTO.getId(), portDTO.getName()), ResourceFactory.getComponentResource(ResourceType.InputPort, cp.getId(), cp.getName()), idGenerationSeed);
}
}
}
snippetContentsCopy.setInputPorts(inputPorts);
final Set<PortDTO> outputPorts = new HashSet<>();
if (snippetContents.getOutputPorts() != null) {
for (final PortDTO portDTO : snippetContents.getOutputPorts()) {
final PortDTO cp = dtoFactory.copy(portDTO);
cp.setId(generateId(portDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
cp.setState(ScheduledState.STOPPED.toString());
outputPorts.add(cp);
final ConnectableDTO portConnectable = dtoFactory.createConnectableDto(cp, ConnectableType.OUTPUT_PORT);
connectableMap.put(portDTO.getParentGroupId() + "-" + portDTO.getId(), portConnectable);
if (parentConnectableMap != null) {
parentConnectableMap.put(portDTO.getParentGroupId() + "-" + portDTO.getId(), portConnectable);
}
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.OutputPort, portDTO.getId(), portDTO.getName()), ResourceFactory.getComponentResource(ResourceType.OutputPort, cp.getId(), cp.getName()), idGenerationSeed);
}
}
}
snippetContentsCopy.setOutputPorts(outputPorts);
//
// Copy the processors
//
final Set<ProcessorDTO> processors = new HashSet<>();
if (snippetContents.getProcessors() != null) {
for (final ProcessorDTO processorDTO : snippetContents.getProcessors()) {
final ProcessorDTO cp = dtoFactory.copy(processorDTO);
cp.setId(generateId(processorDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
if (processorDTO.getState() != null && processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
cp.setState(ScheduledState.DISABLED.toString());
} else {
cp.setState(ScheduledState.STOPPED.toString());
}
processors.add(cp);
connectableMap.put(processorDTO.getParentGroupId() + "-" + processorDTO.getId(), dtoFactory.createConnectableDto(cp));
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.Processor, processorDTO.getId(), processorDTO.getName()), ResourceFactory.getComponentResource(ResourceType.Processor, cp.getId(), cp.getName()), idGenerationSeed);
}
}
}
snippetContentsCopy.setProcessors(processors);
// if there is any controller service that maps to another controller service, update the id's
updateControllerServiceIdentifiers(snippetContentsCopy, serviceIdMap);
//
// Copy ProcessGroups
//
// instantiate the process groups, renaming as necessary
final Set<ProcessGroupDTO> groups = new HashSet<>();
if (snippetContents.getProcessGroups() != null) {
for (final ProcessGroupDTO groupDTO : snippetContents.getProcessGroups()) {
final ProcessGroupDTO cp = dtoFactory.copy(groupDTO, false);
cp.setId(generateId(groupDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
// copy the contents of this group - we do not copy via the dto factory since we want to specify new ids
final FlowSnippetDTO contentsCopy = copyContentsForGroup(groupDTO.getContents(), cp.getId(), connectableMap, serviceIdMap, idGenerationSeed, isCopy);
cp.setContents(contentsCopy);
groups.add(cp);
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.ProcessGroup, groupDTO.getId(), groupDTO.getName()), ResourceFactory.getComponentResource(ResourceType.ProcessGroup, cp.getId(), cp.getName()), idGenerationSeed);
}
}
}
snippetContentsCopy.setProcessGroups(groups);
final Set<RemoteProcessGroupDTO> remoteGroups = new HashSet<>();
if (snippetContents.getRemoteProcessGroups() != null) {
for (final RemoteProcessGroupDTO remoteGroupDTO : snippetContents.getRemoteProcessGroups()) {
final RemoteProcessGroupDTO cp = dtoFactory.copy(remoteGroupDTO);
cp.setId(generateId(remoteGroupDTO.getId(), idGenerationSeed, isCopy));
cp.setParentGroupId(groupId);
final RemoteProcessGroupContentsDTO contents = cp.getContents();
if (contents != null && contents.getInputPorts() != null) {
for (final RemoteProcessGroupPortDTO remotePort : contents.getInputPorts()) {
remotePort.setGroupId(cp.getId());
final String originalId = remotePort.getId();
if (remotePort.getTargetId() == null) {
remotePort.setTargetId(originalId);
}
remotePort.setId(generateId(remotePort.getId(), idGenerationSeed, isCopy));
connectableMap.put(remoteGroupDTO.getId() + "-" + originalId, dtoFactory.createConnectableDto(remotePort, ConnectableType.REMOTE_INPUT_PORT));
}
}
if (contents != null && contents.getOutputPorts() != null) {
for (final RemoteProcessGroupPortDTO remotePort : contents.getOutputPorts()) {
remotePort.setGroupId(cp.getId());
final String originalId = remotePort.getId();
if (remotePort.getTargetId() == null) {
remotePort.setTargetId(originalId);
}
remotePort.setId(generateId(remotePort.getId(), idGenerationSeed, isCopy));
connectableMap.put(remoteGroupDTO.getId() + "-" + originalId, dtoFactory.createConnectableDto(remotePort, ConnectableType.REMOTE_OUTPUT_PORT));
}
}
remoteGroups.add(cp);
// clone policies as appropriate
if (isCopy) {
cloneComponentSpecificPolicies(ResourceFactory.getComponentResource(ResourceType.RemoteProcessGroup, remoteGroupDTO.getId(), remoteGroupDTO.getName()), ResourceFactory.getComponentResource(ResourceType.RemoteProcessGroup, cp.getId(), cp.getName()), idGenerationSeed);
}
}
}
snippetContentsCopy.setRemoteProcessGroups(remoteGroups);
final Set<ConnectionDTO> connections = new HashSet<>();
if (snippetContents.getConnections() != null) {
for (final ConnectionDTO connectionDTO : snippetContents.getConnections()) {
final ConnectionDTO cp = dtoFactory.copy(connectionDTO);
final ConnectableDTO source = connectableMap.get(cp.getSource().getGroupId() + "-" + cp.getSource().getId());
final ConnectableDTO destination = connectableMap.get(cp.getDestination().getGroupId() + "-" + cp.getDestination().getId());
// ensure all referenced components are present
if (source == null || destination == null) {
throw new IllegalArgumentException("The flow snippet contains a Connection that references a component that is not included.");
}
cp.setId(generateId(connectionDTO.getId(), idGenerationSeed, isCopy));
cp.setSource(source);
cp.setDestination(destination);
cp.setParentGroupId(groupId);
connections.add(cp);
// note - no need to copy policies of a connection as their permissions are inferred through the source and destination
}
}
snippetContentsCopy.setConnections(connections);
return snippetContentsCopy;
} catch (Exception e) {
// attempt to role back any policies of the copies that were created in preparation for the clone
rollbackClonedPolicies(snippetContentsCopy);
// rethrow the original exception
throw e;
}
}
use of org.apache.nifi.web.api.dto.PortDTO in project nifi by apache.
the class ITInputPortAccessControl method testNoneUserPutInputPort.
/**
* Ensures the NONE user cannot put an input port.
*
* @throws Exception ex
*/
@Test
public void testNoneUserPutInputPort() throws Exception {
final PortEntity entity = getRandomInputPort(helper.getNoneUser());
assertFalse(entity.getPermissions().getCanRead());
assertFalse(entity.getPermissions().getCanWrite());
assertNull(entity.getComponent());
final String updatedName = "Updated Name" + count++;
// attempt to update the name
final PortDTO requestDto = new PortDTO();
requestDto.setId(entity.getId());
requestDto.setName(updatedName);
final long version = entity.getRevision().getVersion();
final RevisionDTO requestRevision = new RevisionDTO();
requestRevision.setVersion(version);
requestRevision.setClientId(AccessControlHelper.NONE_CLIENT_ID);
final PortEntity requestEntity = new PortEntity();
requestEntity.setId(entity.getId());
requestEntity.setRevision(requestRevision);
requestEntity.setComponent(requestDto);
// perform the request
final Response response = updateInputPort(helper.getNoneUser(), requestEntity);
// ensure forbidden response
assertEquals(403, response.getStatus());
}
use of org.apache.nifi.web.api.dto.PortDTO in project nifi by apache.
the class ITOutputPortAccessControl method createOutputPort.
private PortEntity createOutputPort(final String name) throws Exception {
String url = helper.getBaseUrl() + "/process-groups/root/output-ports";
final String updatedName = name + count++;
// create the output port
PortDTO outputPort = new PortDTO();
outputPort.setName(updatedName);
// create the revision
final RevisionDTO revision = new RevisionDTO();
revision.setClientId(READ_WRITE_CLIENT_ID);
revision.setVersion(0L);
// create the entity body
PortEntity entity = new PortEntity();
entity.setRevision(revision);
entity.setComponent(outputPort);
// perform the request
Response response = helper.getReadWriteUser().testPost(url, entity);
// ensure the request is successful
assertEquals(201, response.getStatus());
// get the entity body
entity = response.readEntity(PortEntity.class);
// verify creation
outputPort = entity.getComponent();
assertEquals(updatedName, outputPort.getName());
// get the output port
return entity;
}
use of org.apache.nifi.web.api.dto.PortDTO in project nifi by apache.
the class ITOutputPortAccessControl method testNoneUserPutOutputPort.
/**
* Ensures the NONE user cannot put an output port.
*
* @throws Exception ex
*/
@Test
public void testNoneUserPutOutputPort() throws Exception {
final PortEntity entity = getRandomOutputPort(helper.getNoneUser());
assertFalse(entity.getPermissions().getCanRead());
assertFalse(entity.getPermissions().getCanWrite());
assertNull(entity.getComponent());
final String updatedName = "Updated Name" + count++;
// attempt to update the name
final PortDTO requestDto = new PortDTO();
requestDto.setId(entity.getId());
requestDto.setName(updatedName);
final long version = entity.getRevision().getVersion();
final RevisionDTO requestRevision = new RevisionDTO();
requestRevision.setVersion(version);
requestRevision.setClientId(AccessControlHelper.NONE_CLIENT_ID);
final PortEntity requestEntity = new PortEntity();
requestEntity.setId(entity.getId());
requestEntity.setRevision(requestRevision);
requestEntity.setComponent(requestDto);
// perform the request
final Response response = updateOutputPort(helper.getNoneUser(), requestEntity);
// ensure forbidden response
assertEquals(403, response.getStatus());
}
Aggregations