Search in sources :

Example 1 with ProcessorSchema

use of org.apache.nifi.minifi.commons.schema.ProcessorSchema in project nifi-minifi by apache.

the class ConfigTransformer method addProcessGroup.

protected static void addProcessGroup(Document doc, Element element, ProcessGroupSchema processGroupSchema, ParentGroupIdResolver parentGroupIdResolver) throws ConfigurationChangeException {
    try {
        String processGroupId = processGroupSchema.getId();
        addTextElement(element, "id", processGroupId);
        addTextElement(element, "name", processGroupSchema.getName());
        addPosition(element);
        addTextElement(element, "comment", processGroupSchema.getComment());
        for (ProcessorSchema processorConfig : processGroupSchema.getProcessors()) {
            addProcessor(element, processorConfig);
        }
        for (PortSchema portSchema : processGroupSchema.getInputPortSchemas()) {
            addPort(doc, element, portSchema, "inputPort");
        }
        for (PortSchema portSchema : processGroupSchema.getOutputPortSchemas()) {
            addPort(doc, element, portSchema, "outputPort");
        }
        for (FunnelSchema funnelSchema : processGroupSchema.getFunnels()) {
            addFunnel(element, funnelSchema);
        }
        for (ProcessGroupSchema child : processGroupSchema.getProcessGroupSchemas()) {
            Element processGroups = doc.createElement("processGroup");
            element.appendChild(processGroups);
            addProcessGroup(doc, processGroups, child, parentGroupIdResolver);
        }
        for (RemoteProcessGroupSchema remoteProcessGroupSchema : processGroupSchema.getRemoteProcessGroups()) {
            addRemoteProcessGroup(element, remoteProcessGroupSchema);
        }
        for (ConnectionSchema connectionConfig : processGroupSchema.getConnections()) {
            addConnection(element, connectionConfig, parentGroupIdResolver);
        }
        for (ControllerServiceSchema controllerServiceSchema : processGroupSchema.getControllerServices()) {
            addControllerService(element, controllerServiceSchema);
        }
    } catch (ConfigurationChangeException e) {
        throw e;
    } catch (Exception e) {
        throw new ConfigurationChangeException("Failed to parse the config YAML while trying to creating the root Process Group", e);
    }
}
Also used : ConnectionSchema(org.apache.nifi.minifi.commons.schema.ConnectionSchema) RemoteProcessGroupSchema(org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema) ProcessGroupSchema(org.apache.nifi.minifi.commons.schema.ProcessGroupSchema) Element(org.w3c.dom.Element) ControllerServiceSchema(org.apache.nifi.minifi.commons.schema.ControllerServiceSchema) ConfigurationChangeException(org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException) FunnelSchema(org.apache.nifi.minifi.commons.schema.FunnelSchema) RemotePortSchema(org.apache.nifi.minifi.commons.schema.RemotePortSchema) PortSchema(org.apache.nifi.minifi.commons.schema.PortSchema) InvalidConfigurationException(org.apache.nifi.minifi.bootstrap.exception.InvalidConfigurationException) ConfigurationChangeException(org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException) TransformerException(javax.xml.transform.TransformerException) DOMException(org.w3c.dom.DOMException) IOException(java.io.IOException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema) RemoteProcessGroupSchema(org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema)

Example 2 with ProcessorSchema

use of org.apache.nifi.minifi.commons.schema.ProcessorSchema in project nifi-minifi by apache.

the class ConfigMainTest method testV1YmlIfPresent.

private void testV1YmlIfPresent(String name, Map<String, Object> yamlMap) throws IOException, SchemaLoaderException {
    InputStream upgradedInputStream = upgradeAndReturn(name + "-v1.yml");
    if (upgradedInputStream != null) {
        ConvertableSchema<ConfigSchema> configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream);
        ConfigSchema configSchemaUpgradedFromV1 = configSchemaConvertableSchema.convert();
        assertTrue(configSchemaUpgradedFromV1.isValid());
        assertEquals(configSchemaConvertableSchema, configSchemaUpgradedFromV1);
        ConfigSchema configSchemaFromCurrent = new ConfigSchema(yamlMap);
        List<ProcessorSchema> currentProcessors = configSchemaFromCurrent.getProcessGroupSchema().getProcessors();
        List<ProcessorSchema> v1Processors = configSchemaUpgradedFromV1.getProcessGroupSchema().getProcessors();
        assertEquals(currentProcessors.size(), v1Processors.size());
        // V1 doesn't have ids so we need to map the autogenerated ones to the ones from the template
        Map<String, String> v1IdToCurrentIdMap = new HashMap<>();
        for (int i = 0; i < currentProcessors.size(); i++) {
            ProcessorSchema currentProcessor = currentProcessors.get(i);
            ProcessorSchema v1Processor = v1Processors.get(i);
            assertEquals(currentProcessor.getName(), v1Processor.getName());
            v1IdToCurrentIdMap.put(v1Processor.getId(), currentProcessor.getId());
            v1Processor.setId(currentProcessor.getId());
        }
        List<RemoteProcessGroupSchema> currentRPGs = configSchemaFromCurrent.getProcessGroupSchema().getRemoteProcessGroups();
        List<RemoteProcessGroupSchema> v1RPGs = configSchemaUpgradedFromV1.getProcessGroupSchema().getRemoteProcessGroups();
        // V1 doesn't have ids so we need to map the autogenerated ones to the ones from the template
        for (int i = 0; i < currentRPGs.size(); i++) {
            RemoteProcessGroupSchema currentRPG = currentRPGs.get(i);
            RemoteProcessGroupSchema v1RPG = v1RPGs.get(i);
            assertEquals(currentRPG.getName(), v1RPG.getName());
            v1IdToCurrentIdMap.put(v1RPG.getId(), currentRPG.getId());
            v1RPG.setId(currentRPG.getId());
        }
        configSchemaUpgradedFromV1.getProcessGroupSchema().getRemoteProcessGroups().stream().flatMap(g -> g.getInputPorts().stream()).map(RemotePortSchema::getId).sequential().forEach(id -> v1IdToCurrentIdMap.put(id, id));
        List<ConnectionSchema> currentConnections = configSchemaFromCurrent.getProcessGroupSchema().getConnections();
        List<ConnectionSchema> v1Connections = configSchemaUpgradedFromV1.getProcessGroupSchema().getConnections();
        // Update source and dest ids, can set connection id equal because it isn't referenced elsewhere
        assertEquals(currentConnections.size(), v1Connections.size());
        for (int i = 0; i < currentConnections.size(); i++) {
            ConnectionSchema currentConnection = currentConnections.get(i);
            ConnectionSchema v1Connection = v1Connections.get(i);
            assertEquals(currentConnection.getName(), v1Connection.getName());
            v1Connection.setId(currentConnection.getId());
            v1Connection.setSourceId(v1IdToCurrentIdMap.get(v1Connection.getSourceId()));
            v1Connection.setDestinationId(v1IdToCurrentIdMap.get(v1Connection.getDestinationId()));
        }
        ConfigSchema.getAllProcessGroups(configSchemaFromCurrent.getProcessGroupSchema()).stream().flatMap(p -> p.getRemoteProcessGroups().stream()).forEach(r -> {
            clearV3orLaterProperties(r);
            r.setTransportProtocol(RemoteProcessGroupSchema.TransportProtocolOptions.RAW.name());
        });
        Map<String, Object> v1YamlMap = configSchemaUpgradedFromV1.toMap();
        assertNoMapDifferences(v1YamlMap, configSchemaFromCurrent.toMap());
    }
}
Also used : RemotePortSchema(org.apache.nifi.minifi.commons.schema.RemotePortSchema) RemoteProcessGroupSchema(org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema) ByteArrayOutputStream(java.io.ByteArrayOutputStream) SchemaLoader(org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader) Mock(org.mockito.Mock) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) LimitedInputStream(org.apache.nifi.controller.repository.io.LimitedInputStream) Before(org.junit.Before) Charsets(org.apache.commons.io.Charsets) OutputStream(java.io.OutputStream) ConvertableSchema(org.apache.nifi.minifi.commons.schema.common.ConvertableSchema) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) ConfigSchema(org.apache.nifi.minifi.commons.schema.ConfigSchema) Test(org.junit.Test) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) JAXBException(javax.xml.bind.JAXBException) FileNotFoundException(java.io.FileNotFoundException) List(java.util.List) MockitoJUnitRunner(org.mockito.runners.MockitoJUnitRunner) SchemaLoaderException(org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException) ConnectionSchema(org.apache.nifi.minifi.commons.schema.ConnectionSchema) SUCCESS(org.apache.nifi.minifi.toolkit.configuration.ConfigMain.SUCCESS) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) ConnectionSchema(org.apache.nifi.minifi.commons.schema.ConnectionSchema) HashMap(java.util.HashMap) ByteArrayInputStream(java.io.ByteArrayInputStream) LimitedInputStream(org.apache.nifi.controller.repository.io.LimitedInputStream) InputStream(java.io.InputStream) RemotePortSchema(org.apache.nifi.minifi.commons.schema.RemotePortSchema) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema) RemoteProcessGroupSchema(org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema) ConfigSchema(org.apache.nifi.minifi.commons.schema.ConfigSchema)

Example 3 with ProcessorSchema

use of org.apache.nifi.minifi.commons.schema.ProcessorSchema in project nifi-minifi by apache.

the class ConfigSchemaV1 method getConnectionSchemas.

protected List<ConnectionSchema> getConnectionSchemas(List<ProcessorSchema> processors, List<String> validationIssues) {
    Set<UUID> ids = new HashSet<>();
    Map<String, String> processorNameToIdMap = new HashMap<>();
    // We can't look up id by name for names that appear more than once
    Set<String> duplicateProcessorNames = new HashSet<>();
    if (processors != null) {
        processors.stream().forEachOrdered(p -> processorNameToIdMap.put(p.getName(), p.getId()));
        duplicateProcessorNames = new CollectionOverlap<>(processors.stream().map(ProcessorSchema::getName)).getDuplicates();
    }
    Set<String> remoteInputPortIds = new HashSet<>();
    if (remoteProcessingGroups != null) {
        remoteInputPortIds.addAll(remoteProcessingGroups.stream().filter(r -> r.getInputPorts() != null).flatMap(r -> r.getInputPorts().stream()).map(RemotePortSchema::getId).collect(Collectors.toSet()));
    }
    Set<String> problematicDuplicateNames = new HashSet<>();
    List<ConnectionSchema> connectionSchemas = new ArrayList<>(connections.size());
    for (ConnectionSchemaV1 connection : connections) {
        ConnectionSchema convert = connection.convert();
        convert.setId(getUniqueId(ids, convert.getName()));
        String sourceName = connection.getSourceName();
        if (remoteInputPortIds.contains(sourceName)) {
            convert.setSourceId(sourceName);
        } else {
            if (duplicateProcessorNames.contains(sourceName)) {
                problematicDuplicateNames.add(sourceName);
            }
            String sourceId = processorNameToIdMap.get(sourceName);
            if (!StringUtil.isNullOrEmpty(sourceId)) {
                convert.setSourceId(sourceId);
            }
        }
        String destinationName = connection.getDestinationName();
        if (remoteInputPortIds.contains(destinationName)) {
            convert.setDestinationId(destinationName);
        } else {
            if (duplicateProcessorNames.contains(destinationName)) {
                problematicDuplicateNames.add(destinationName);
            }
            String destinationId = processorNameToIdMap.get(destinationName);
            if (!StringUtil.isNullOrEmpty(destinationId)) {
                convert.setDestinationId(destinationId);
            }
        }
        connectionSchemas.add(convert);
    }
    if (problematicDuplicateNames.size() > 0) {
        validationIssues.add(CANNOT_LOOK_UP_PROCESSOR_ID_FROM_PROCESSOR_NAME_DUE_TO_DUPLICATE_PROCESSOR_NAMES + problematicDuplicateNames.stream().collect(Collectors.joining(", ")));
    }
    return connectionSchemas;
}
Also used : RemotePortSchema(org.apache.nifi.minifi.commons.schema.RemotePortSchema) RemoteProcessGroupSchema(org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema) BaseSchema(org.apache.nifi.minifi.commons.schema.common.BaseSchema) FLOW_CONTROLLER_PROPS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY) HashMap(java.util.HashMap) PROCESSORS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROCESSORS_KEY) ContentRepositorySchema(org.apache.nifi.minifi.commons.schema.ContentRepositorySchema) ArrayList(java.util.ArrayList) PROVENANCE_REPORTING_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPORTING_KEY) FlowControllerSchema(org.apache.nifi.minifi.commons.schema.FlowControllerSchema) HashSet(java.util.HashSet) TOP_LEVEL_NAME(org.apache.nifi.minifi.commons.schema.ConfigSchema.TOP_LEVEL_NAME) CONNECTIONS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONNECTIONS_KEY) ProvenanceRepositorySchema(org.apache.nifi.minifi.commons.schema.ProvenanceRepositorySchema) SECURITY_PROPS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SECURITY_PROPS_KEY) StringUtil(org.apache.nifi.minifi.commons.schema.common.StringUtil) Map(java.util.Map) CollectionOverlap(org.apache.nifi.minifi.commons.schema.common.CollectionOverlap) FLOWFILE_REPO_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOWFILE_REPO_KEY) CorePropertiesSchema(org.apache.nifi.minifi.commons.schema.CorePropertiesSchema) FlowFileRepositorySchema(org.apache.nifi.minifi.commons.schema.FlowFileRepositorySchema) ConvertableSchema(org.apache.nifi.minifi.commons.schema.common.ConvertableSchema) Set(java.util.Set) ConfigSchema(org.apache.nifi.minifi.commons.schema.ConfigSchema) CORE_PROPS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CORE_PROPS_KEY) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) ProvenanceReportingSchema(org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema) COMPONENT_STATUS_REPO_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMPONENT_STATUS_REPO_KEY) SecurityPropertiesSchema(org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema) List(java.util.List) VERSION(org.apache.nifi.minifi.commons.schema.ConfigSchema.VERSION) PROVENANCE_REPO_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPO_KEY) CONTENT_REPO_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONTENT_REPO_KEY) REMOTE_PROCESS_GROUPS_KEY(org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.REMOTE_PROCESS_GROUPS_KEY) ConnectionSchema(org.apache.nifi.minifi.commons.schema.ConnectionSchema) ComponentStatusRepositorySchema(org.apache.nifi.minifi.commons.schema.ComponentStatusRepositorySchema) ConnectionSchema(org.apache.nifi.minifi.commons.schema.ConnectionSchema) HashMap(java.util.HashMap) RemotePortSchema(org.apache.nifi.minifi.commons.schema.RemotePortSchema) ArrayList(java.util.ArrayList) UUID(java.util.UUID) CollectionOverlap(org.apache.nifi.minifi.commons.schema.common.CollectionOverlap) HashSet(java.util.HashSet)

Example 4 with ProcessorSchema

use of org.apache.nifi.minifi.commons.schema.ProcessorSchema in project nifi-minifi by apache.

the class ConfigSchemaV1 method getProcessorSchemas.

protected List<ProcessorSchema> getProcessorSchemas() {
    Set<UUID> ids = new HashSet<>();
    List<ProcessorSchema> processorSchemas = new ArrayList<>(processors.size());
    for (ProcessorSchemaV1 processor : processors) {
        ProcessorSchema processorSchema = processor.convert();
        processorSchema.setId(getUniqueId(ids, processorSchema.getName()));
        processorSchemas.add(processorSchema);
    }
    return processorSchemas;
}
Also used : ArrayList(java.util.ArrayList) UUID(java.util.UUID) HashSet(java.util.HashSet) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema)

Example 5 with ProcessorSchema

use of org.apache.nifi.minifi.commons.schema.ProcessorSchema in project nifi-minifi by apache.

the class ProcessorSchemaFunction method apply.

@Override
public ProcessorSchema apply(ProcessorDTO processorDTO) {
    ProcessorConfigDTO processorDTOConfig = processorDTO.getConfig();
    Map<String, Object> map = new HashMap<>();
    map.put(NAME_KEY, processorDTO.getName());
    map.put(ID_KEY, processorDTO.getId());
    map.put(CLASS_KEY, processorDTO.getType());
    map.put(SCHEDULING_STRATEGY_KEY, processorDTOConfig.getSchedulingStrategy());
    map.put(SCHEDULING_PERIOD_KEY, processorDTOConfig.getSchedulingPeriod());
    map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, processorDTOConfig.getConcurrentlySchedulableTaskCount());
    map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, processorDTOConfig.getPenaltyDuration());
    map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, processorDTOConfig.getYieldDuration());
    Long runDurationMillis = processorDTOConfig.getRunDurationMillis();
    if (runDurationMillis != null) {
        map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, runDurationMillis * 1000);
    }
    map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, nullToEmpty(processorDTO.getRelationships()).stream().filter(RelationshipDTO::isAutoTerminate).map(RelationshipDTO::getName).collect(Collectors.toList()));
    map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(processorDTOConfig.getProperties())));
    String annotationData = processorDTOConfig.getAnnotationData();
    if (annotationData != null && !annotationData.isEmpty()) {
        map.put(ANNOTATION_DATA_KEY, annotationData);
    }
    return new ProcessorSchema(map);
}
Also used : ProcessorConfigDTO(org.apache.nifi.web.api.dto.ProcessorConfigDTO) HashMap(java.util.HashMap) RelationshipDTO(org.apache.nifi.web.api.dto.RelationshipDTO) ProcessorSchema(org.apache.nifi.minifi.commons.schema.ProcessorSchema)

Aggregations

ProcessorSchema (org.apache.nifi.minifi.commons.schema.ProcessorSchema)5 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 HashSet (java.util.HashSet)3 ConnectionSchema (org.apache.nifi.minifi.commons.schema.ConnectionSchema)3 RemotePortSchema (org.apache.nifi.minifi.commons.schema.RemotePortSchema)3 RemoteProcessGroupSchema (org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema)3 IOException (java.io.IOException)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 UUID (java.util.UUID)2 ConfigSchema (org.apache.nifi.minifi.commons.schema.ConfigSchema)2 ConvertableSchema (org.apache.nifi.minifi.commons.schema.common.ConvertableSchema)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 FileNotFoundException (java.io.FileNotFoundException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 StandardCharsets (java.nio.charset.StandardCharsets)1