Examples with Processor org.apache.nifi.processor.Processor used on opensource projects

Example 1 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class DtoFactory method createProcessorConfigDto.

/**
 * Creates a ProcessorConfigDTO from the specified ProcessorNode.
 *
 * @param procNode node
 * @return dto
 */
public ProcessorConfigDTO createProcessorConfigDto(final ProcessorNode procNode) {
    if (procNode == null) {
        return null;
    }
    final ProcessorConfigDTO dto = new ProcessorConfigDTO();
    // sort a copy of the properties
    final Map<PropertyDescriptor, String> sortedProperties = new TreeMap<>(new Comparator<PropertyDescriptor>() {

        @Override
        public int compare(final PropertyDescriptor o1, final PropertyDescriptor o2) {
            return Collator.getInstance(Locale.US).compare(o1.getName(), o2.getName());
        }
    });
    sortedProperties.putAll(procNode.getProperties());
    // get the property order from the processor
    final Processor processor = procNode.getProcessor();
    final Map<PropertyDescriptor, String> orderedProperties = new LinkedHashMap<>();
    final List<PropertyDescriptor> descriptors = processor.getPropertyDescriptors();
    if (descriptors != null && !descriptors.isEmpty()) {
        for (final PropertyDescriptor descriptor : descriptors) {
            orderedProperties.put(descriptor, null);
        }
    }
    orderedProperties.putAll(sortedProperties);
    // build the descriptor and property dtos
    dto.setDescriptors(new LinkedHashMap<String, PropertyDescriptorDTO>());
    dto.setProperties(new LinkedHashMap<String, String>());
    for (final Map.Entry<PropertyDescriptor, String> entry : orderedProperties.entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        // store the property descriptor
        dto.getDescriptors().put(descriptor.getName(), createPropertyDescriptorDto(descriptor, procNode.getProcessGroup().getIdentifier()));
        // determine the property value - don't include sensitive properties
        String propertyValue = entry.getValue();
        if (propertyValue != null && descriptor.isSensitive()) {
            propertyValue = SENSITIVE_VALUE_MASK;
        } else if (propertyValue == null && descriptor.getDefaultValue() != null) {
            propertyValue = descriptor.getDefaultValue();
        }
        // set the property value
        dto.getProperties().put(descriptor.getName(), propertyValue);
    }
    dto.setSchedulingPeriod(procNode.getSchedulingPeriod());
    dto.setPenaltyDuration(procNode.getPenalizationPeriod());
    dto.setYieldDuration(procNode.getYieldPeriod());
    dto.setRunDurationMillis(procNode.getRunDuration(TimeUnit.MILLISECONDS));
    dto.setConcurrentlySchedulableTaskCount(procNode.getMaxConcurrentTasks());
    dto.setLossTolerant(procNode.isLossTolerant());
    dto.setComments(procNode.getComments());
    dto.setBulletinLevel(procNode.getBulletinLevel().name());
    dto.setSchedulingStrategy(procNode.getSchedulingStrategy().name());
    dto.setExecutionNode(procNode.getExecutionNode().name());
    dto.setAnnotationData(procNode.getAnnotationData());
    // set up the default values for concurrent tasks and scheduling period
    final Map<String, String> defaultConcurrentTasks = new HashMap<>();
    defaultConcurrentTasks.put(SchedulingStrategy.TIMER_DRIVEN.name(), String.valueOf(SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks()));
    defaultConcurrentTasks.put(SchedulingStrategy.EVENT_DRIVEN.name(), String.valueOf(SchedulingStrategy.EVENT_DRIVEN.getDefaultConcurrentTasks()));
    defaultConcurrentTasks.put(SchedulingStrategy.CRON_DRIVEN.name(), String.valueOf(SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks()));
    dto.setDefaultConcurrentTasks(defaultConcurrentTasks);
    final Map<String, String> defaultSchedulingPeriod = new HashMap<>();
    defaultSchedulingPeriod.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod());
    defaultSchedulingPeriod.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod());
    dto.setDefaultSchedulingPeriod(defaultSchedulingPeriod);
    return dto;
}

Example 2 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class DtoFactory method createFlowDto.

public FlowDTO createFlowDto(final ProcessGroup group, final ProcessGroupStatus groupStatus, final FlowSnippetDTO snippet, final RevisionManager revisionManager, final Function<ProcessGroup, List<BulletinEntity>> getProcessGroupBulletins) {
    if (snippet == null) {
        return null;
    }
    final FlowDTO flow = new FlowDTO();
    for (final ConnectionDTO snippetConnection : snippet.getConnections()) {
        final Connection connection = group.getConnection(snippetConnection.getId());
        // marshal the actual connection as the snippet is pruned
        final ConnectionDTO dto = createConnectionDto(connection);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(connection.getIdentifier()));
        final PermissionsDTO accessPolicy = createPermissionsDto(connection);
        final ConnectionStatusDTO status = getComponentStatus(() -> groupStatus.getConnectionStatus().stream().filter(connectionStatus -> connection.getIdentifier().equals(connectionStatus.getId())).findFirst().orElse(null), connectionStatus -> createConnectionStatusDto(connectionStatus));
        flow.getConnections().add(entityFactory.createConnectionEntity(dto, revision, accessPolicy, status));
    }
    for (final FunnelDTO snippetFunnel : snippet.getFunnels()) {
        final Funnel funnel = group.getFunnel(snippetFunnel.getId());
        // marshal the actual funnel as the snippet is pruned
        final FunnelDTO dto = createFunnelDto(funnel);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(funnel.getIdentifier()));
        final PermissionsDTO accessPolicy = createPermissionsDto(funnel);
        flow.getFunnels().add(entityFactory.createFunnelEntity(dto, revision, accessPolicy));
    }
    for (final PortDTO snippetInputPort : snippet.getInputPorts()) {
        final Port inputPort = group.getInputPort(snippetInputPort.getId());
        // marshal the actual port as the snippet is pruned
        final PortDTO dto = createPortDto(inputPort);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(inputPort.getIdentifier()));
        final PermissionsDTO permissions = createPermissionsDto(inputPort);
        final PortStatusDTO status = getComponentStatus(() -> groupStatus.getInputPortStatus().stream().filter(inputPortStatus -> inputPort.getIdentifier().equals(inputPortStatus.getId())).findFirst().orElse(null), inputPortStatus -> createPortStatusDto(inputPortStatus));
        final List<BulletinDTO> bulletins = createBulletinDtos(bulletinRepository.findBulletinsForSource(inputPort.getIdentifier()));
        final List<BulletinEntity> bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList());
        flow.getInputPorts().add(entityFactory.createPortEntity(dto, revision, permissions, status, bulletinEntities));
    }
    for (final PortDTO snippetOutputPort : snippet.getOutputPorts()) {
        final Port outputPort = group.getOutputPort(snippetOutputPort.getId());
        // marshal the actual port as the snippet is pruned
        final PortDTO dto = createPortDto(outputPort);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(outputPort.getIdentifier()));
        final PermissionsDTO permissions = createPermissionsDto(outputPort);
        final PortStatusDTO status = getComponentStatus(() -> groupStatus.getOutputPortStatus().stream().filter(outputPortStatus -> outputPort.getIdentifier().equals(outputPortStatus.getId())).findFirst().orElse(null), outputPortStatus -> createPortStatusDto(outputPortStatus));
        final List<BulletinDTO> bulletins = createBulletinDtos(bulletinRepository.findBulletinsForSource(outputPort.getIdentifier()));
        final List<BulletinEntity> bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList());
        flow.getOutputPorts().add(entityFactory.createPortEntity(dto, revision, permissions, status, bulletinEntities));
    }
    for (final LabelDTO snippetLabel : snippet.getLabels()) {
        final Label label = group.getLabel(snippetLabel.getId());
        // marshal the actual label as the snippet is pruned
        final LabelDTO dto = createLabelDto(label);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(label.getIdentifier()));
        final PermissionsDTO accessPolicy = createPermissionsDto(label);
        flow.getLabels().add(entityFactory.createLabelEntity(dto, revision, accessPolicy));
    }
    for (final ProcessGroupDTO snippetProcessGroup : snippet.getProcessGroups()) {
        final ProcessGroup processGroup = group.getProcessGroup(snippetProcessGroup.getId());
        // marshal the actual group as the snippet is pruned
        final ProcessGroupDTO dto = createProcessGroupDto(processGroup);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(processGroup.getIdentifier()));
        final PermissionsDTO permissions = createPermissionsDto(processGroup);
        final ProcessGroupStatusDTO status = getComponentStatus(() -> groupStatus.getProcessGroupStatus().stream().filter(processGroupStatus -> processGroup.getIdentifier().equals(processGroupStatus.getId())).findFirst().orElse(null), processGroupStatus -> createConciseProcessGroupStatusDto(processGroupStatus));
        final List<BulletinEntity> bulletins = getProcessGroupBulletins.apply(processGroup);
        flow.getProcessGroups().add(entityFactory.createProcessGroupEntity(dto, revision, permissions, status, bulletins));
    }
    for (final ProcessorDTO snippetProcessor : snippet.getProcessors()) {
        final ProcessorNode processor = group.getProcessor(snippetProcessor.getId());
        // marshal the actual processor as the snippet is pruned
        final ProcessorDTO dto = createProcessorDto(processor);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(processor.getIdentifier()));
        final PermissionsDTO permissions = createPermissionsDto(processor);
        final ProcessorStatusDTO status = getComponentStatus(() -> groupStatus.getProcessorStatus().stream().filter(processorStatus -> processor.getIdentifier().equals(processorStatus.getId())).findFirst().orElse(null), processorStatus -> createProcessorStatusDto(processorStatus));
        final List<BulletinDTO> bulletins = createBulletinDtos(bulletinRepository.findBulletinsForSource(processor.getIdentifier()));
        final List<BulletinEntity> bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList());
        flow.getProcessors().add(entityFactory.createProcessorEntity(dto, revision, permissions, status, bulletinEntities));
    }
    for (final RemoteProcessGroupDTO snippetRemoteProcessGroup : snippet.getRemoteProcessGroups()) {
        final RemoteProcessGroup remoteProcessGroup = group.getRemoteProcessGroup(snippetRemoteProcessGroup.getId());
        // marshal the actual rpm as the snippet is pruned
        final RemoteProcessGroupDTO dto = createRemoteProcessGroupDto(remoteProcessGroup);
        final RevisionDTO revision = createRevisionDTO(revisionManager.getRevision(remoteProcessGroup.getIdentifier()));
        final PermissionsDTO permissions = createPermissionsDto(remoteProcessGroup);
        final RemoteProcessGroupStatusDTO status = getComponentStatus(() -> groupStatus.getRemoteProcessGroupStatus().stream().filter(rpgStatus -> remoteProcessGroup.getIdentifier().equals(rpgStatus.getId())).findFirst().orElse(null), remoteProcessGroupStatus -> createRemoteProcessGroupStatusDto(remoteProcessGroupStatus));
        final List<BulletinDTO> bulletins = createBulletinDtos(bulletinRepository.findBulletinsForSource(remoteProcessGroup.getIdentifier()));
        final List<BulletinEntity> bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList());
        flow.getRemoteProcessGroups().add(entityFactory.createRemoteProcessGroupEntity(dto, revision, permissions, status, bulletinEntities));
    }
    return flow;
}

Example 3 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class ControllerFacade method getResources.

public List<Resource> getResources() {
    final List<Resource> resources = new ArrayList<>();
    resources.add(ResourceFactory.getFlowResource());
    resources.add(ResourceFactory.getSystemResource());
    resources.add(ResourceFactory.getControllerResource());
    resources.add(ResourceFactory.getCountersResource());
    resources.add(ResourceFactory.getProvenanceResource());
    resources.add(ResourceFactory.getPoliciesResource());
    resources.add(ResourceFactory.getTenantResource());
    resources.add(ResourceFactory.getProxyResource());
    resources.add(ResourceFactory.getResourceResource());
    resources.add(ResourceFactory.getSiteToSiteResource());
    // restricted components
    resources.add(ResourceFactory.getRestrictedComponentsResource());
    Arrays.stream(RequiredPermission.values()).forEach(requiredPermission -> resources.add(ResourceFactory.getRestrictedComponentsResource(requiredPermission)));
    final ProcessGroup root = flowController.getGroup(flowController.getRootGroupId());
    // include the root group
    final Resource rootResource = root.getResource();
    resources.add(rootResource);
    resources.add(ResourceFactory.getDataResource(rootResource));
    resources.add(ResourceFactory.getPolicyResource(rootResource));
    // add each processor
    for (final ProcessorNode processor : root.findAllProcessors()) {
        final Resource processorResource = processor.getResource();
        resources.add(processorResource);
        resources.add(ResourceFactory.getDataResource(processorResource));
        resources.add(ResourceFactory.getPolicyResource(processorResource));
    }
    // add each label
    for (final Label label : root.findAllLabels()) {
        final Resource labelResource = label.getResource();
        resources.add(labelResource);
        resources.add(ResourceFactory.getPolicyResource(labelResource));
    }
    // add each process group
    for (final ProcessGroup processGroup : root.findAllProcessGroups()) {
        final Resource processGroupResource = processGroup.getResource();
        resources.add(processGroupResource);
        resources.add(ResourceFactory.getDataResource(processGroupResource));
        resources.add(ResourceFactory.getPolicyResource(processGroupResource));
    }
    // add each remote process group
    for (final RemoteProcessGroup remoteProcessGroup : root.findAllRemoteProcessGroups()) {
        final Resource remoteProcessGroupResource = remoteProcessGroup.getResource();
        resources.add(remoteProcessGroupResource);
        resources.add(ResourceFactory.getDataResource(remoteProcessGroupResource));
        resources.add(ResourceFactory.getPolicyResource(remoteProcessGroupResource));
    }
    // add each input port
    for (final Port inputPort : root.findAllInputPorts()) {
        final Resource inputPortResource = inputPort.getResource();
        resources.add(inputPortResource);
        resources.add(ResourceFactory.getDataResource(inputPortResource));
        resources.add(ResourceFactory.getPolicyResource(inputPortResource));
        if (inputPort instanceof RootGroupPort) {
            resources.add(ResourceFactory.getDataTransferResource(inputPortResource));
        }
    }
    // add each output port
    for (final Port outputPort : root.findAllOutputPorts()) {
        final Resource outputPortResource = outputPort.getResource();
        resources.add(outputPortResource);
        resources.add(ResourceFactory.getDataResource(outputPortResource));
        resources.add(ResourceFactory.getPolicyResource(outputPortResource));
        if (outputPort instanceof RootGroupPort) {
            resources.add(ResourceFactory.getDataTransferResource(outputPortResource));
        }
    }
    // add each controller service
    final Consumer<ControllerServiceNode> csConsumer = controllerService -> {
        final Resource controllerServiceResource = controllerService.getResource();
        resources.add(controllerServiceResource);
        resources.add(ResourceFactory.getPolicyResource(controllerServiceResource));
    };
    flowController.getAllControllerServices().forEach(csConsumer);
    root.findAllControllerServices().forEach(csConsumer);
    // add each reporting task
    for (final ReportingTaskNode reportingTask : flowController.getAllReportingTasks()) {
        final Resource reportingTaskResource = reportingTask.getResource();
        resources.add(reportingTaskResource);
        resources.add(ResourceFactory.getPolicyResource(reportingTaskResource));
    }
    // add each template
    for (final Template template : root.findAllTemplates()) {
        final Resource templateResource = template.getResource();
        resources.add(templateResource);
        resources.add(ResourceFactory.getPolicyResource(templateResource));
    }
    return resources;
}

Example 4 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class ControllerSearchService method search.

private ComponentSearchResultDTO search(final String searchStr, final ProcessorNode procNode) {
    final List<String> matches = new ArrayList<>();
    final Processor processor = procNode.getProcessor();
    addIfAppropriate(searchStr, procNode.getIdentifier(), "Id", matches);
    addIfAppropriate(searchStr, procNode.getVersionedComponentId().orElse(null), "Version Control ID", matches);
    addIfAppropriate(searchStr, procNode.getName(), "Name", matches);
    addIfAppropriate(searchStr, procNode.getComments(), "Comments", matches);
    // consider scheduling strategy
    if (SchedulingStrategy.EVENT_DRIVEN.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("event", searchStr)) {
        matches.add("Scheduling strategy: Event driven");
    } else if (SchedulingStrategy.TIMER_DRIVEN.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("timer", searchStr)) {
        matches.add("Scheduling strategy: Timer driven");
    } else if (SchedulingStrategy.PRIMARY_NODE_ONLY.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("primary", searchStr)) {
        // PRIMARY_NODE_ONLY has been deprecated as a SchedulingStrategy and replaced by PRIMARY as an ExecutionNode.
        matches.add("Scheduling strategy: On primary node");
    }
    // consider execution node
    if (ExecutionNode.PRIMARY.equals(procNode.getExecutionNode()) && StringUtils.containsIgnoreCase("primary", searchStr)) {
        matches.add("Execution node: primary");
    }
    // consider scheduled state
    if (ScheduledState.DISABLED.equals(procNode.getScheduledState())) {
        if (StringUtils.containsIgnoreCase("disabled", searchStr)) {
            matches.add("Run status: Disabled");
        }
    } else {
        if (StringUtils.containsIgnoreCase("invalid", searchStr) && !procNode.isValid()) {
            matches.add("Run status: Invalid");
        } else if (ScheduledState.RUNNING.equals(procNode.getScheduledState()) && StringUtils.containsIgnoreCase("running", searchStr)) {
            matches.add("Run status: Running");
        } else if (ScheduledState.STOPPED.equals(procNode.getScheduledState()) && StringUtils.containsIgnoreCase("stopped", searchStr)) {
            matches.add("Run status: Stopped");
        }
    }
    for (final Relationship relationship : procNode.getRelationships()) {
        addIfAppropriate(searchStr, relationship.getName(), "Relationship", matches);
    }
    // Add both the actual class name and the component type. This allows us to search for 'Ghost'
    // to search for components that could not be instantiated.
    addIfAppropriate(searchStr, processor.getClass().getSimpleName(), "Type", matches);
    addIfAppropriate(searchStr, procNode.getComponentType(), "Type", matches);
    for (final Map.Entry<PropertyDescriptor, String> entry : procNode.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        addIfAppropriate(searchStr, descriptor.getName(), "Property name", matches);
        addIfAppropriate(searchStr, descriptor.getDescription(), "Property description", matches);
        // never include sensitive properties values in search results
        if (descriptor.isSensitive()) {
            continue;
        }
        String value = entry.getValue();
        // if unset consider default value
        if (value == null) {
            value = descriptor.getDefaultValue();
        }
        // evaluate if the value matches the search criteria
        if (StringUtils.containsIgnoreCase(value, searchStr)) {
            matches.add("Property value: " + descriptor.getName() + " - " + value);
        }
    }
    // consider searching the processor directly
    if (processor instanceof Searchable) {
        final Searchable searchable = (Searchable) processor;
        final SearchContext context = new StandardSearchContext(searchStr, procNode, flowController, variableRegistry);
        // search the processor using the appropriate thread context classloader
        try (final NarCloseable x = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
            final Collection<SearchResult> searchResults = searchable.search(context);
            if (CollectionUtils.isNotEmpty(searchResults)) {
                for (final SearchResult searchResult : searchResults) {
                    matches.add(searchResult.getLabel() + ": " + searchResult.getMatch());
                }
            }
        } catch (final Throwable t) {
        // log this as error
        }
    }
    if (matches.isEmpty()) {
        return null;
    }
    final ComponentSearchResultDTO result = new ComponentSearchResultDTO();
    result.setId(procNode.getIdentifier());
    result.setMatches(matches);
    result.setName(procNode.getName());
    return result;
}

Example 5 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class TestStandardProcessScheduler method testProcessorThrowsExceptionOnScheduledRetry.

// Test that if processor throws Exception in @OnScheduled, it keeps getting scheduled
@Test(timeout = 10000)
public void testProcessorThrowsExceptionOnScheduledRetry() throws InterruptedException {
    final FailOnScheduledProcessor proc = new FailOnScheduledProcessor();
    proc.setDesiredFailureCount(3);
    proc.initialize(new StandardProcessorInitializationContext(UUID.randomUUID().toString(), null, null, null, nifiProperties));
    final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);
    final LoggableComponent<Processor> loggableComponent = new LoggableComponent<>(proc, systemBundle.getBundleDetails().getCoordinate(), null);
    final ProcessorNode procNode = new StandardProcessorNode(loggableComponent, UUID.randomUUID().toString(), new StandardValidationContextFactory(controller, variableRegistry), scheduler, controller, nifiProperties, new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent);
    rootGroup.addProcessor(procNode);
    scheduler.startProcessor(procNode, true);
    while (!proc.isSucceess()) {
        Thread.sleep(5L);
    }
    assertEquals(3, proc.getOnScheduledInvocationCount());
}