Examples with SchemaAndValue org.apache.kafka.connect.data.SchemaAndValue used on opensource projects

Example 86 with SchemaAndValue

use of org.apache.kafka.connect.data.SchemaAndValue in project apache-kafka-on-k8s by banzaicloud.

the class WorkerSinkTask method convertMessages.

private void convertMessages(ConsumerRecords<byte[], byte[]> msgs) {
    origOffsets.clear();
    for (ConsumerRecord<byte[], byte[]> msg : msgs) {
        log.trace("{} Consuming and converting message in topic '{}' partition {} at offset {} and timestamp {}", this, msg.topic(), msg.partition(), msg.offset(), msg.timestamp());
        SchemaAndValue keyAndSchema = keyConverter.toConnectData(msg.topic(), msg.key());
        SchemaAndValue valueAndSchema = valueConverter.toConnectData(msg.topic(), msg.value());
        Headers headers = convertHeadersFor(msg);
        Long timestamp = ConnectUtils.checkAndConvertTimestamp(msg.timestamp());
        SinkRecord origRecord = new SinkRecord(msg.topic(), msg.partition(), keyAndSchema.schema(), keyAndSchema.value(), valueAndSchema.schema(), valueAndSchema.value(), msg.offset(), timestamp, msg.timestampType(), headers);
        log.trace("{} Applying transformations to record in topic '{}' partition {} at offset {} and timestamp {} with key {} and value {}", this, msg.topic(), msg.partition(), msg.offset(), timestamp, keyAndSchema.value(), valueAndSchema.value());
        SinkRecord transRecord = transformationChain.apply(origRecord);
        origOffsets.put(new TopicPartition(origRecord.topic(), origRecord.kafkaPartition()), new OffsetAndMetadata(origRecord.kafkaOffset() + 1));
        if (transRecord != null) {
            messageBatch.add(transRecord);
        } else {
            log.trace("{} Transformations returned null, so dropping record in topic '{}' partition {} at offset {} and timestamp {} with key {} and value {}", this, msg.topic(), msg.partition(), msg.offset(), timestamp, keyAndSchema.value(), valueAndSchema.value());
        }
    }
    sinkTaskMetricsGroup.recordConsumedOffsets(origOffsets);
}

Example 87 with SchemaAndValue

use of org.apache.kafka.connect.data.SchemaAndValue in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStatusBackingStore method parseConnectorStatus.

private ConnectorStatus parseConnectorStatus(String connector, byte[] data) {
    try {
        SchemaAndValue schemaAndValue = converter.toConnectData(topic, data);
        if (!(schemaAndValue.value() instanceof Map)) {
            log.error("Invalid connector status type {}", schemaAndValue.value().getClass());
            return null;
        }
        @SuppressWarnings("unchecked") Map<String, Object> statusMap = (Map<String, Object>) schemaAndValue.value();
        TaskStatus.State state = TaskStatus.State.valueOf((String) statusMap.get(STATE_KEY_NAME));
        String trace = (String) statusMap.get(TRACE_KEY_NAME);
        String workerUrl = (String) statusMap.get(WORKER_ID_KEY_NAME);
        int generation = ((Long) statusMap.get(GENERATION_KEY_NAME)).intValue();
        return new ConnectorStatus(connector, state, trace, workerUrl, generation);
    } catch (Exception e) {
        log.error("Failed to deserialize connector status", e);
        return null;
    }
}

Example 88 with SchemaAndValue

use of org.apache.kafka.connect.data.SchemaAndValue in project apache-kafka-on-k8s by banzaicloud.

the class JsonConverterTest method timeToConnectWithDefaultValue.

@Test
public void timeToConnectWithDefaultValue() {
    java.util.Date reference = new java.util.Date(0);
    Schema schema = Time.builder().defaultValue(reference).schema();
    String msg = "{ \"schema\": { \"type\": \"int32\", \"name\": \"org.apache.kafka.connect.data.Time\", \"version\": 1, \"default\": 0 }, \"payload\": null }";
    SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
    assertEquals(schema, schemaAndValue.schema());
    assertEquals(reference, schemaAndValue.value());
}

Example 89 with SchemaAndValue

use of org.apache.kafka.connect.data.SchemaAndValue in project apache-kafka-on-k8s by banzaicloud.

the class JsonConverterTest method timeToConnectOptionalWithDefaultValue.

@Test
public void timeToConnectOptionalWithDefaultValue() {
    java.util.Date reference = new java.util.Date(0);
    Schema schema = Time.builder().optional().defaultValue(reference).schema();
    String msg = "{ \"schema\": { \"type\": \"int32\", \"name\": \"org.apache.kafka.connect.data.Time\", \"version\": 1, \"optional\": true, \"default\": 0 }, \"payload\": null }";
    SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
    assertEquals(schema, schemaAndValue.schema());
    assertEquals(reference, schemaAndValue.value());
}

Example 90 with SchemaAndValue

use of org.apache.kafka.connect.data.SchemaAndValue in project apache-kafka-on-k8s by banzaicloud.

the class JsonConverterTest method timestampToConnectOptional.

@Test
public void timestampToConnectOptional() {
    Schema schema = Timestamp.builder().optional().schema();
    String msg = "{ \"schema\": { \"type\": \"int64\", \"name\": \"org.apache.kafka.connect.data.Timestamp\", \"version\": 1, \"optional\": true }, \"payload\": null }";
    SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
    assertEquals(schema, schemaAndValue.schema());
    assertNull(schemaAndValue.value());
}