Search in sources :

Example 1 with ScraperConfigurationTriggeredImplBuilder

use of org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImplBuilder in project plc4x by apache.

the class Plc4xSourceTask method start.

@Override
public void start(Map<String, String> props) {
    AbstractConfig config = new AbstractConfig(CONFIG_DEF, props);
    String connectionName = config.getString(Constants.CONNECTION_NAME_CONFIG);
    String plc4xConnectionString = config.getString(Constants.CONNECTION_STRING_CONFIG);
    pollReturnInterval = config.getInt(Constants.KAFKA_POLL_RETURN_CONFIG);
    Integer bufferSize = config.getInt(Constants.BUFFER_SIZE_CONFIG);
    Map<String, String> topics = new HashMap<>();
    // Create a buffer with a capacity of BUFFER_SIZE_CONFIG elements which schedules access in a fair way.
    buffer = new ArrayBlockingQueue<>(bufferSize, true);
    ScraperConfigurationTriggeredImplBuilder builder = new ScraperConfigurationTriggeredImplBuilder();
    builder.addSource(connectionName, plc4xConnectionString);
    List<String> jobConfigs = config.getList(Constants.QUERIES_CONFIG);
    for (String jobConfig : jobConfigs) {
        String[] jobConfigSegments = jobConfig.split("\\|");
        if (jobConfigSegments.length < 4) {
            log.warn("Error in job configuration '{}'. " + "The configuration expects at least 4 segments: " + "{job-name}|{topic}|{rate}(|{field-alias}#{field-address})+", jobConfig);
            continue;
        }
        String jobName = jobConfigSegments[0];
        String topic = jobConfigSegments[1];
        Integer rate = Integer.valueOf(jobConfigSegments[2]);
        JobConfigurationTriggeredImplBuilder jobBuilder = builder.job(jobName, String.format("(SCHEDULED,%s)", rate)).source(connectionName);
        for (int i = 3; i < jobConfigSegments.length; i++) {
            String[] fieldSegments = jobConfigSegments[i].split("#");
            if (fieldSegments.length != 2) {
                log.warn("Error in job configuration '{}'. " + "The field segment expects a format {field-alias}#{field-address}, but got '%s'", jobName, jobConfigSegments[i]);
                continue;
            }
            String fieldAlias = fieldSegments[0];
            String fieldAddress = fieldSegments[1];
            jobBuilder.field(fieldAlias, fieldAddress);
            topics.put(jobName, topic);
        }
        jobBuilder.build();
    }
    ScraperConfigurationTriggeredImpl scraperConfig = builder.build();
    try {
        PlcDriverManager plcDriverManager = new PooledPlcDriverManager();
        TriggerCollector triggerCollector = new TriggerCollectorImpl(plcDriverManager);
        scraper = new TriggeredScraperImpl(scraperConfig, (jobName, sourceName, results) -> {
            try {
                Long timestamp = System.currentTimeMillis();
                Map<String, String> sourcePartition = new HashMap<>();
                sourcePartition.put("sourceName", sourceName);
                sourcePartition.put("jobName", jobName);
                Map<String, Long> sourceOffset = Collections.singletonMap("offset", timestamp);
                String topic = topics.get(jobName);
                // Prepare the key structure.
                Struct key = new Struct(KEY_SCHEMA).put(Constants.SOURCE_NAME_FIELD, sourceName).put(Constants.JOB_NAME_FIELD, jobName);
                // Build the Schema for the result struct.
                SchemaBuilder fieldSchemaBuilder = SchemaBuilder.struct().name("org.apache.plc4x.kafka.schema.Field");
                for (Map.Entry<String, Object> result : results.entrySet()) {
                    // Get field-name and -value from the results.
                    String fieldName = result.getKey();
                    Object fieldValue = result.getValue();
                    // Get the schema for the given value type.
                    Schema valueSchema = getSchema(fieldValue);
                    // Add the schema description for the current field.
                    fieldSchemaBuilder.field(fieldName, valueSchema);
                }
                Schema fieldSchema = fieldSchemaBuilder.build();
                Schema recordSchema = SchemaBuilder.struct().name("org.apache.plc4x.kafka.schema.JobResult").doc("PLC Job result. This contains all of the received PLCValues as well as a recieved timestamp").field(Constants.FIELDS_CONFIG, fieldSchema).field(Constants.TIMESTAMP_CONFIG, Schema.INT64_SCHEMA).field(Constants.EXPIRES_CONFIG, Schema.OPTIONAL_INT64_SCHEMA).build();
                // Build the struct itself.
                Struct fieldStruct = new Struct(fieldSchema);
                for (Map.Entry<String, Object> result : results.entrySet()) {
                    // Get field-name and -value from the results.
                    String fieldName = result.getKey();
                    Object fieldValue = result.getValue();
                    if (fieldSchema.field(fieldName).schema().type() == Schema.Type.ARRAY) {
                        fieldStruct.put(fieldName, ((List) fieldValue).stream().map(p -> ((PlcValue) p).getObject()).collect(Collectors.toList()));
                    } else {
                        fieldStruct.put(fieldName, fieldValue);
                    }
                }
                Struct recordStruct = new Struct(recordSchema).put(Constants.FIELDS_CONFIG, fieldStruct).put(Constants.TIMESTAMP_CONFIG, timestamp);
                // Prepare the source-record element.
                SourceRecord sourceRecord = new SourceRecord(sourcePartition, sourceOffset, topic, KEY_SCHEMA, key, recordSchema, recordStruct);
                // Add the new source-record to the buffer.
                buffer.add(sourceRecord);
            } catch (Exception e) {
                log.error("Error while parsing returned values", e);
            }
        }, triggerCollector);
        scraper.start();
        triggerCollector.start();
    } catch (ScraperException e) {
        log.error("Error starting the scraper", e);
    }
}
Also used : PlcDriverManager(org.apache.plc4x.java.PlcDriverManager) Date(org.apache.kafka.connect.data.Date) java.util(java.util) ScraperConfigurationTriggeredImpl(org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl) LoggerFactory(org.slf4j.LoggerFactory) LocalDateTime(java.time.LocalDateTime) PlcValue(org.apache.plc4x.java.api.value.PlcValue) JobConfigurationTriggeredImplBuilder(org.apache.plc4x.java.scraper.config.triggeredscraper.JobConfigurationTriggeredImplBuilder) ScraperException(org.apache.plc4x.java.scraper.exception.ScraperException) SecureRandom(java.security.SecureRandom) BigDecimal(java.math.BigDecimal) ScraperConfigurationTriggeredImplBuilder(org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImplBuilder) LocalTime(java.time.LocalTime) BigInteger(java.math.BigInteger) org.apache.kafka.connect.data(org.apache.kafka.connect.data) PooledPlcDriverManager(org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager) ConfigDef(org.apache.kafka.common.config.ConfigDef) VersionUtil(org.apache.plc4x.kafka.util.VersionUtil) Logger(org.slf4j.Logger) java.util.concurrent(java.util.concurrent) SourceRecord(org.apache.kafka.connect.source.SourceRecord) Collectors(java.util.stream.Collectors) TriggerCollectorImpl(org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollectorImpl) TimeUnit(java.util.concurrent.TimeUnit) Constants(org.apache.plc4x.kafka.config.Constants) AbstractConfig(org.apache.kafka.common.config.AbstractConfig) ConnectException(org.apache.kafka.connect.errors.ConnectException) LocalDate(java.time.LocalDate) TriggeredScraperImpl(org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl) TriggerCollector(org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollector) SourceTask(org.apache.kafka.connect.source.SourceTask) TriggerCollector(org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollector) SourceRecord(org.apache.kafka.connect.source.SourceRecord) TriggeredScraperImpl(org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl) PlcDriverManager(org.apache.plc4x.java.PlcDriverManager) PooledPlcDriverManager(org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager) ScraperException(org.apache.plc4x.java.scraper.exception.ScraperException) ScraperException(org.apache.plc4x.java.scraper.exception.ScraperException) ConnectException(org.apache.kafka.connect.errors.ConnectException) PooledPlcDriverManager(org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager) BigInteger(java.math.BigInteger) AbstractConfig(org.apache.kafka.common.config.AbstractConfig) PlcValue(org.apache.plc4x.java.api.value.PlcValue) ScraperConfigurationTriggeredImplBuilder(org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImplBuilder) ScraperConfigurationTriggeredImpl(org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl) JobConfigurationTriggeredImplBuilder(org.apache.plc4x.java.scraper.config.triggeredscraper.JobConfigurationTriggeredImplBuilder) TriggerCollectorImpl(org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollectorImpl)

Aggregations

BigDecimal (java.math.BigDecimal)1 BigInteger (java.math.BigInteger)1 SecureRandom (java.security.SecureRandom)1 LocalDate (java.time.LocalDate)1 LocalDateTime (java.time.LocalDateTime)1 LocalTime (java.time.LocalTime)1 java.util (java.util)1 java.util.concurrent (java.util.concurrent)1 TimeUnit (java.util.concurrent.TimeUnit)1 Collectors (java.util.stream.Collectors)1 AbstractConfig (org.apache.kafka.common.config.AbstractConfig)1 ConfigDef (org.apache.kafka.common.config.ConfigDef)1 org.apache.kafka.connect.data (org.apache.kafka.connect.data)1 Date (org.apache.kafka.connect.data.Date)1 ConnectException (org.apache.kafka.connect.errors.ConnectException)1 SourceRecord (org.apache.kafka.connect.source.SourceRecord)1 SourceTask (org.apache.kafka.connect.source.SourceTask)1 PlcDriverManager (org.apache.plc4x.java.PlcDriverManager)1 PlcValue (org.apache.plc4x.java.api.value.PlcValue)1 JobConfigurationTriggeredImplBuilder (org.apache.plc4x.java.scraper.config.triggeredscraper.JobConfigurationTriggeredImplBuilder)1