Examples with BundleCoordinate org.apache.nifi.bundle.BundleCoordinate used on opensource projects

Example 21 with BundleCoordinate

use of org.apache.nifi.bundle.BundleCoordinate in project nifi by apache.

the class FingerprintFactory method addFlowFileProcessorFingerprint.

private StringBuilder addFlowFileProcessorFingerprint(final StringBuilder builder, final Element processorElem) throws FingerprintException {
    // id
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "id"));
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "versionedComponentId"));
    // class
    final NodeList childNodes = DomUtils.getChildNodesByTagName(processorElem, "class");
    final String className = childNodes.item(0).getTextContent();
    appendFirstValue(builder, childNodes);
    // annotation data
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "annotationData"));
    // get the bundle details if possible
    final BundleDTO bundle = FlowFromDOMFactory.getBundle(DomUtils.getChild(processorElem, "bundle"));
    addBundleFingerprint(builder, bundle);
    // max concurrent tasks
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "maxConcurrentTasks"));
    // scheduling period
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "schedulingPeriod"));
    // penalization period
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "penalizationPeriod"));
    // yield period
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "yieldPeriod"));
    // bulletin level
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "bulletinLevel"));
    // loss tolerant
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "lossTolerant"));
    // scheduling strategy
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "schedulingStrategy"));
    // execution node
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "executionNode"));
    // run duration nanos
    appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "runDurationNanos"));
    // get the temp instance of the Processor so that we know the default property values
    final BundleCoordinate coordinate = getCoordinate(className, bundle);
    final ConfigurableComponent configurableComponent = ExtensionManager.getTempComponent(className, coordinate);
    if (configurableComponent == null) {
        logger.warn("Unable to get Processor of type {}; its default properties will be fingerprinted instead of being ignored.", className);
    }
    // properties
    final NodeList propertyElems = DomUtils.getChildNodesByTagName(processorElem, "property");
    final List<Element> sortedPropertyElems = sortElements(propertyElems, getProcessorPropertiesComparator());
    for (final Element propertyElem : sortedPropertyElems) {
        final String propName = DomUtils.getChildElementsByTagName(propertyElem, "name").get(0).getTextContent();
        String propValue = getFirstValue(DomUtils.getChildNodesByTagName(propertyElem, "value"), null);
        addPropertyFingerprint(builder, configurableComponent, propName, propValue);
    }
    final NodeList autoTerminateElems = DomUtils.getChildNodesByTagName(processorElem, "autoTerminatedRelationship");
    final List<Element> sortedAutoTerminateElems = sortElements(autoTerminateElems, getElementTextComparator());
    for (final Element autoTerminateElem : sortedAutoTerminateElems) {
        builder.append(autoTerminateElem.getTextContent());
    }
    return builder;
}

Example 22 with BundleCoordinate

use of org.apache.nifi.bundle.BundleCoordinate in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 23 with BundleCoordinate

use of org.apache.nifi.bundle.BundleCoordinate in project nifi by apache.

the class DocGenerator method documentConfigurableComponent.

/**
 * Documents a type of configurable component.
 *
 * @param extensionClasses types of a configurable component
 * @param explodedNiFiDocsDir base directory of component documentation
 */
private static void documentConfigurableComponent(final Set<Class> extensionClasses, final File explodedNiFiDocsDir) {
    for (final Class<?> extensionClass : extensionClasses) {
        if (ConfigurableComponent.class.isAssignableFrom(extensionClass)) {
            final String extensionClassName = extensionClass.getCanonicalName();
            final Bundle bundle = ExtensionManager.getBundle(extensionClass.getClassLoader());
            if (bundle == null) {
                logger.warn("No coordinate found for {}, skipping...", new Object[] { extensionClassName });
                continue;
            }
            final BundleCoordinate coordinate = bundle.getBundleDetails().getCoordinate();
            final String path = coordinate.getGroup() + "/" + coordinate.getId() + "/" + coordinate.getVersion() + "/" + extensionClassName;
            final File componentDirectory = new File(explodedNiFiDocsDir, path);
            componentDirectory.mkdirs();
            final Class<? extends ConfigurableComponent> componentClass = extensionClass.asSubclass(ConfigurableComponent.class);
            try {
                logger.debug("Documenting: " + componentClass);
                document(componentDirectory, componentClass, coordinate);
            } catch (Exception e) {
                logger.warn("Unable to document: " + componentClass, e);
            }
        }
    }
}

Example 24 with BundleCoordinate

use of org.apache.nifi.bundle.BundleCoordinate in project nifi by apache.

the class DocGeneratorTest method testProcessorLoadsNarResources.

@Test
public void testProcessorLoadsNarResources() throws IOException, ClassNotFoundException {
    TemporaryFolder temporaryFolder = new TemporaryFolder();
    temporaryFolder.create();
    NiFiProperties properties = loadSpecifiedProperties("/conf/nifi.properties", NiFiProperties.COMPONENT_DOCS_DIRECTORY, temporaryFolder.getRoot().getAbsolutePath());
    final Bundle systemBundle = SystemBundle.create(properties);
    final ExtensionMapping mapping = NarUnpacker.unpackNars(properties, systemBundle);
    NarClassLoaders.getInstance().init(properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory());
    ExtensionManager.discoverExtensions(systemBundle, NarClassLoaders.getInstance().getBundles());
    DocGenerator.generate(properties, mapping);
    final String extensionClassName = "org.apache.nifi.processors.WriteResourceToStream";
    final BundleCoordinate coordinate = mapping.getProcessorNames().get(extensionClassName).stream().findFirst().get();
    final String path = coordinate.getGroup() + "/" + coordinate.getId() + "/" + coordinate.getVersion() + "/" + extensionClassName;
    File processorDirectory = new File(temporaryFolder.getRoot(), path);
    File indexHtml = new File(processorDirectory, "index.html");
    Assert.assertTrue(indexHtml + " should have been generated", indexHtml.exists());
    String generatedHtml = FileUtils.readFileToString(indexHtml);
    Assert.assertNotNull(generatedHtml);
    Assert.assertTrue(generatedHtml.contains("This example processor loads a resource from the nar and writes it to the FlowFile content"));
    Assert.assertTrue(generatedHtml.contains("files that were successfully processed"));
    Assert.assertTrue(generatedHtml.contains("files that were not successfully processed"));
    Assert.assertTrue(generatedHtml.contains("resources"));
}

Example 25 with BundleCoordinate

use of org.apache.nifi.bundle.BundleCoordinate in project nifi-minifi by apache.

the class ExtensionManager method registerServiceClass.

/**
 * Registers extension for the specified type from the specified Bundle.
 *
 * @param type               the extension type
 * @param classNameBundleMap mapping of classname to Bundle
 * @param bundle             the Bundle being mapped to
 * @param classes            to map to this classloader but which come from its ancestors
 */
private static void registerServiceClass(final Class<?> type, final Map<String, List<Bundle>> classNameBundleMap, final Bundle bundle, final Set<Class> classes) {
    final String className = type.getName();
    // get the bundles that have already been registered for the class name
    List<Bundle> registeredBundles = classNameBundleMap.get(className);
    if (registeredBundles == null) {
        registeredBundles = new ArrayList<>();
        classNameBundleMap.put(className, registeredBundles);
    }
    boolean alreadyRegistered = false;
    for (final Bundle registeredBundle : registeredBundles) {
        final BundleCoordinate registeredCoordinate = registeredBundle.getBundleDetails().getCoordinate();
        // if the incoming bundle has the same coordinate as one of the registered bundles then consider it already registered
        if (registeredCoordinate.equals(bundle.getBundleDetails().getCoordinate())) {
            alreadyRegistered = true;
            break;
        }
        // fail registration because we don't support multiple versions of any other types
        if (!multipleVersionsAllowed(type)) {
            throw new IllegalStateException("Attempt was made to load " + className + " from " + bundle.getBundleDetails().getCoordinate().getCoordinate() + " but that class name is already loaded/registered from " + registeredBundle.getBundleDetails().getCoordinate() + " and multiple versions are not supported for this type");
        }
    }
    // if none of the above was true then register the new bundle
    if (!alreadyRegistered) {
        registeredBundles.add(bundle);
        classes.add(type);
        if (type.isAnnotationPresent(RequiresInstanceClassLoading.class)) {
            final String cacheKey = getClassBundleKey(className, bundle.getBundleDetails().getCoordinate());
            requiresInstanceClassLoading.put(cacheKey, type);
        }
    }
}