use of org.apache.nifi.web.api.dto.RelationshipDTO in project nifi by apache.
the class TemplateUtils method scrubProcessors.
/**
* Scrubs processors prior to saving. This includes removing sensitive properties, validation errors, property descriptors, etc.
*
* @param processors procs
*/
private static void scrubProcessors(final Set<ProcessorDTO> processors) {
// go through each processor
for (final ProcessorDTO processorDTO : processors) {
final ProcessorConfigDTO processorConfig = processorDTO.getConfig();
// ensure that some property configuration have been specified
if (processorConfig != null) {
// if properties have been specified, remove sensitive ones
if (processorConfig.getProperties() != null) {
Map<String, String> processorProperties = processorConfig.getProperties();
// look for sensitive properties and remove them
if (processorConfig.getDescriptors() != null) {
final Collection<PropertyDescriptorDTO> descriptors = processorConfig.getDescriptors().values();
for (PropertyDescriptorDTO descriptor : descriptors) {
if (Boolean.TRUE.equals(descriptor.isSensitive())) {
processorProperties.put(descriptor.getName(), null);
}
scrubPropertyDescriptor(descriptor);
}
}
}
processorConfig.setCustomUiUrl(null);
processorConfig.setDefaultConcurrentTasks(null);
processorConfig.setDefaultSchedulingPeriod(null);
processorConfig.setAutoTerminatedRelationships(null);
}
if (processorDTO.getRelationships() != null) {
for (final RelationshipDTO relationship : processorDTO.getRelationships()) {
relationship.setDescription(null);
}
}
processorDTO.setExtensionMissing(null);
processorDTO.setMultipleVersionsAvailable(null);
processorDTO.setValidationErrors(null);
processorDTO.setInputRequirement(null);
processorDTO.setDescription(null);
processorDTO.setInputRequirement(null);
processorDTO.setPersistsState(null);
processorDTO.setSupportsBatching(null);
processorDTO.setSupportsEventDriven(null);
processorDTO.setSupportsParallelProcessing(null);
}
}
use of org.apache.nifi.web.api.dto.RelationshipDTO in project nifi by apache.
the class FlowController method instantiateSnippet.
private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
writeLock.lock();
try {
validateSnippetContents(requireNonNull(group), dto);
//
for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
serviceNode.setComments(controllerServiceDTO.getComments());
serviceNode.setName(controllerServiceDTO.getName());
if (!topLevel) {
serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
}
group.addControllerService(serviceNode);
}
// references another service.
for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
final String serviceId = controllerServiceDTO.getId();
final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
serviceNode.setProperties(controllerServiceDTO.getProperties());
}
//
for (final LabelDTO labelDTO : dto.getLabels()) {
final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
label.setPosition(toPosition(labelDTO.getPosition()));
if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
}
label.setStyle(labelDTO.getStyle());
if (!topLevel) {
label.setVersionedComponentId(labelDTO.getVersionedComponentId());
}
group.addLabel(label);
}
// Instantiate the funnels
for (final FunnelDTO funnelDTO : dto.getFunnels()) {
final Funnel funnel = createFunnel(funnelDTO.getId());
funnel.setPosition(toPosition(funnelDTO.getPosition()));
if (!topLevel) {
funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
}
group.addFunnel(funnel);
}
//
for (final PortDTO portDTO : dto.getInputPorts()) {
final Port inputPort;
if (group.isRootGroup()) {
inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
if (portDTO.getGroupAccessControl() != null) {
((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
}
if (portDTO.getUserAccessControl() != null) {
((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
}
} else {
inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
}
if (!topLevel) {
inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
}
inputPort.setPosition(toPosition(portDTO.getPosition()));
inputPort.setProcessGroup(group);
inputPort.setComments(portDTO.getComments());
group.addInputPort(inputPort);
}
for (final PortDTO portDTO : dto.getOutputPorts()) {
final Port outputPort;
if (group.isRootGroup()) {
outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
if (portDTO.getGroupAccessControl() != null) {
((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
}
if (portDTO.getUserAccessControl() != null) {
((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
}
} else {
outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
}
if (!topLevel) {
outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
}
outputPort.setPosition(toPosition(portDTO.getPosition()));
outputPort.setProcessGroup(group);
outputPort.setComments(portDTO.getComments());
group.addOutputPort(outputPort);
}
//
for (final ProcessorDTO processorDTO : dto.getProcessors()) {
final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
procNode.setPosition(toPosition(processorDTO.getPosition()));
procNode.setProcessGroup(group);
if (!topLevel) {
procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
}
final ProcessorConfigDTO config = processorDTO.getConfig();
procNode.setComments(config.getComments());
if (config.isLossTolerant() != null) {
procNode.setLossTolerant(config.isLossTolerant());
}
procNode.setName(processorDTO.getName());
procNode.setYieldPeriod(config.getYieldDuration());
procNode.setPenalizationPeriod(config.getPenaltyDuration());
procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
procNode.setAnnotationData(config.getAnnotationData());
procNode.setStyle(processorDTO.getStyle());
if (config.getRunDurationMillis() != null) {
procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
}
if (config.getSchedulingStrategy() != null) {
procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
}
if (config.getExecutionNode() != null) {
procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
}
if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
procNode.disable();
}
// ensure that the scheduling strategy is set prior to these values
procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
procNode.setScheduldingPeriod(config.getSchedulingPeriod());
final Set<Relationship> relationships = new HashSet<>();
if (processorDTO.getRelationships() != null) {
for (final RelationshipDTO rel : processorDTO.getRelationships()) {
if (rel.isAutoTerminate()) {
relationships.add(procNode.getRelationship(rel.getName()));
}
}
procNode.setAutoTerminatedRelationships(relationships);
}
if (config.getProperties() != null) {
procNode.setProperties(config.getProperties());
}
group.addProcessor(procNode);
}
//
for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
remoteGroup.setComments(remoteGroupDTO.getComments());
remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
if (!topLevel) {
remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
}
if (remoteGroupDTO.getTransportProtocol() == null) {
remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
} else {
remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
}
remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
remoteGroup.setProcessGroup(group);
// set the input/output ports
if (remoteGroupDTO.getContents() != null) {
final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
// ensure there are input ports
if (contents.getInputPorts() != null) {
remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
}
// ensure there are output ports
if (contents.getOutputPorts() != null) {
remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
}
}
group.addRemoteProcessGroup(remoteGroup);
}
//
for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
childGroup.setParent(group);
childGroup.setPosition(toPosition(groupDTO.getPosition()));
childGroup.setComments(groupDTO.getComments());
childGroup.setName(groupDTO.getName());
if (groupDTO.getVariables() != null) {
childGroup.setVariables(groupDTO.getVariables());
}
// We do this only if this component is the child of a Versioned Component.
if (!topLevel) {
childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
}
group.addProcessGroup(childGroup);
final FlowSnippetDTO contents = groupDTO.getContents();
// we want this to be recursive, so we will create a new template that contains only
// the contents of this child group and recursively call ourselves.
final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
childTemplateDTO.setConnections(contents.getConnections());
childTemplateDTO.setInputPorts(contents.getInputPorts());
childTemplateDTO.setLabels(contents.getLabels());
childTemplateDTO.setOutputPorts(contents.getOutputPorts());
childTemplateDTO.setProcessGroups(contents.getProcessGroups());
childTemplateDTO.setProcessors(contents.getProcessors());
childTemplateDTO.setFunnels(contents.getFunnels());
childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
childTemplateDTO.setControllerServices(contents.getControllerServices());
instantiateSnippet(childGroup, childTemplateDTO, false);
if (groupDTO.getVersionControlInformation() != null) {
final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
childGroup.setVersionControlInformation(vci, Collections.emptyMap());
}
}
//
for (final ConnectionDTO connectionDTO : dto.getConnections()) {
final ConnectableDTO sourceDTO = connectionDTO.getSource();
final ConnectableDTO destinationDTO = connectionDTO.getDestination();
final Connectable source;
final Connectable destination;
// see if the source connectable is a remote port
if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
source = remoteGroup.getOutputPort(sourceDTO.getId());
} else {
final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
source = sourceGroup.getConnectable(sourceDTO.getId());
}
// see if the destination connectable is a remote port
if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
destination = remoteGroup.getInputPort(destinationDTO.getId());
} else {
final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
destination = destinationGroup.getConnectable(destinationDTO.getId());
}
// determine the selection relationships for this connection
final Set<String> relationships = new HashSet<>();
if (connectionDTO.getSelectedRelationships() != null) {
relationships.addAll(connectionDTO.getSelectedRelationships());
}
final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
if (!topLevel) {
connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
}
if (connectionDTO.getBends() != null) {
final List<Position> bendPoints = new ArrayList<>();
for (final PositionDTO bend : connectionDTO.getBends()) {
bendPoints.add(new Position(bend.getX(), bend.getY()));
}
connection.setBendPoints(bendPoints);
}
final FlowFileQueue queue = connection.getFlowFileQueue();
queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
final List<String> prioritizers = connectionDTO.getPrioritizers();
if (prioritizers != null) {
final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
for (final String className : newPrioritizersClasses) {
try {
newPrioritizers.add(createPrioritizer(className));
} catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
}
}
queue.setPriorities(newPrioritizers);
}
connection.setProcessGroup(group);
group.addConnection(connection);
}
} finally {
writeLock.unlock();
}
}
use of org.apache.nifi.web.api.dto.RelationshipDTO in project nifi-minifi by apache.
the class ProcessorSchemaTest method setup.
@Before
public void setup() {
config = new ProcessorConfigDTO();
RelationshipDTO relationshipDTO = new RelationshipDTO();
relationshipDTO.setName(testRelationship);
relationshipDTO.setAutoTerminate(true);
dto = new ProcessorDTO();
dto.setConfig(config);
dto.setName(testName);
dto.setId(testId);
dto.setType(testProcessorClass);
config.setSchedulingStrategy(testSchedulingStrategy);
config.setSchedulingPeriod(testSchedulingPeriod);
config.setConcurrentlySchedulableTaskCount(testMaxConcurrentTasks);
config.setPenaltyDuration(testPenalizationPeriod);
config.setYieldDuration(testYieldDuration);
config.setRunDurationMillis(testRunDurationNanos / 1000);
config.setAnnotationData(testAnnotationData);
dto.setRelationships(Arrays.asList(relationshipDTO));
Map<String, String> properties = new HashMap<>();
properties.put(testKey, testValue);
config.setProperties(properties);
map = new HashMap<>();
map.put(CommonPropertyKeys.NAME_KEY, testName);
map.put(CommonPropertyKeys.ID_KEY, testId);
map.put(CLASS_KEY, testProcessorClass);
map.put(CommonPropertyKeys.SCHEDULING_STRATEGY_KEY, testSchedulingStrategy);
map.put(CommonPropertyKeys.SCHEDULING_PERIOD_KEY, testSchedulingPeriod);
map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, testMaxConcurrentTasks);
map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, testPenalizationPeriod);
map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, testYieldDuration);
map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, testRunDurationNanos);
map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, Arrays.asList(testRelationship));
map.put(PROPERTIES_KEY, new HashMap<>(properties));
map.put(ANNOTATION_DATA_KEY, testAnnotationData);
}
use of org.apache.nifi.web.api.dto.RelationshipDTO in project nifi-minifi by apache.
the class ProcessorSchemaFunction method apply.
@Override
public ProcessorSchema apply(ProcessorDTO processorDTO) {
ProcessorConfigDTO processorDTOConfig = processorDTO.getConfig();
Map<String, Object> map = new HashMap<>();
map.put(NAME_KEY, processorDTO.getName());
map.put(ID_KEY, processorDTO.getId());
map.put(CLASS_KEY, processorDTO.getType());
map.put(SCHEDULING_STRATEGY_KEY, processorDTOConfig.getSchedulingStrategy());
map.put(SCHEDULING_PERIOD_KEY, processorDTOConfig.getSchedulingPeriod());
map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, processorDTOConfig.getConcurrentlySchedulableTaskCount());
map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, processorDTOConfig.getPenaltyDuration());
map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, processorDTOConfig.getYieldDuration());
Long runDurationMillis = processorDTOConfig.getRunDurationMillis();
if (runDurationMillis != null) {
map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, runDurationMillis * 1000);
}
map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, nullToEmpty(processorDTO.getRelationships()).stream().filter(RelationshipDTO::isAutoTerminate).map(RelationshipDTO::getName).collect(Collectors.toList()));
map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(processorDTOConfig.getProperties())));
String annotationData = processorDTOConfig.getAnnotationData();
if (annotationData != null && !annotationData.isEmpty()) {
map.put(ANNOTATION_DATA_KEY, annotationData);
}
return new ProcessorSchema(map);
}
Aggregations