Examples with JsonConverter org.apache.kafka.connect.json.JsonConverter used on opensource projects

Example 1 with JsonConverter

use of org.apache.kafka.connect.json.JsonConverter in project kafka-connect-storage-cloud by confluentinc.

the class DataWriterJsonTest method setUp.

// @Before should be omitted in order to be able to add properties per test.
public void setUp() throws Exception {
    super.setUp();
    converter = new JsonConverter();
    converter.configure(Collections.singletonMap("schemas.enable", "false"), false);
    s3 = newS3Client(connectorConfig);
    storage = new S3Storage(connectorConfig, url, S3_TEST_BUCKET_NAME, s3);
    partitioner = new DefaultPartitioner<>();
    partitioner.configure(parsedConfig);
    format = new JsonFormat(storage);
    s3.createBucket(S3_TEST_BUCKET_NAME);
    assertTrue(s3.doesBucketExist(S3_TEST_BUCKET_NAME));
}

Example 2 with JsonConverter

use of org.apache.kafka.connect.json.JsonConverter in project debezium by debezium.

the class AbstractConnectorTest method readLastCommittedOffsets.

/**
 * Utility to read the last committed offsets for the specified partitions.
 *
 * @param config the configuration of the engine used to persist the offsets
 * @param partitions the partitions
 * @return the map of partitions to offsets; never null but possibly empty
 */
protected <T> Map<Map<String, T>, Map<String, Object>> readLastCommittedOffsets(Configuration config, Collection<Map<String, T>> partitions) {
    config = config.edit().with(EmbeddedEngine.ENGINE_NAME, "testing-connector").with(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH).with(EmbeddedEngine.OFFSET_FLUSH_INTERVAL_MS, 0).build();
    final String engineName = config.getString(EmbeddedEngine.ENGINE_NAME);
    Converter keyConverter = config.getInstance(EmbeddedEngine.INTERNAL_KEY_CONVERTER_CLASS, Converter.class);
    keyConverter.configure(config.subset(EmbeddedEngine.INTERNAL_KEY_CONVERTER_CLASS.name() + ".", true).asMap(), true);
    Converter valueConverter = config.getInstance(EmbeddedEngine.INTERNAL_VALUE_CONVERTER_CLASS, Converter.class);
    Configuration valueConverterConfig = config;
    if (valueConverter instanceof JsonConverter) {
        // Make sure that the JSON converter is configured to NOT enable schemas ...
        valueConverterConfig = config.edit().with(EmbeddedEngine.INTERNAL_VALUE_CONVERTER_CLASS + ".schemas.enable", false).build();
    }
    valueConverter.configure(valueConverterConfig.subset(EmbeddedEngine.INTERNAL_VALUE_CONVERTER_CLASS.name() + ".", true).asMap(), false);
    // Create the worker config, adding extra fields that are required for validation of a worker config
    // but that are not used within the embedded engine (since the source records are never serialized) ...
    Map<String, String> embeddedConfig = config.asMap(EmbeddedEngine.ALL_FIELDS);
    embeddedConfig.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName());
    embeddedConfig.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName());
    WorkerConfig workerConfig = new EmbeddedConfig(embeddedConfig);
    FileOffsetBackingStore offsetStore = new FileOffsetBackingStore();
    offsetStore.configure(workerConfig);
    offsetStore.start();
    try {
        OffsetStorageReaderImpl offsetReader = new OffsetStorageReaderImpl(offsetStore, engineName, keyConverter, valueConverter);
        return offsetReader.offsets(partitions);
    } finally {
        offsetStore.stop();
    }
}

Example 3 with JsonConverter

use of org.apache.kafka.connect.json.JsonConverter in project ksql by confluentinc.

the class SchemaMapper method createNewConverter.

private JsonConverter createNewConverter() {
    JsonConverter result = new JsonConverter();
    result.configure(configs, false);
    return result;
}

Example 4 with JsonConverter

use of org.apache.kafka.connect.json.JsonConverter in project debezium by debezium.

the class AbstractConnectorTest method initializeConnectorTestFramework.

@Before
public final void initializeConnectorTestFramework() {
    LoggingContext.forConnector(getClass().getSimpleName(), "", "test");
    keyJsonConverter = new JsonConverter();
    valueJsonConverter = new JsonConverter();
    keyJsonDeserializer = new JsonDeserializer();
    valueJsonDeserializer = new JsonDeserializer();
    Configuration converterConfig = Configuration.create().build();
    Configuration deserializerConfig = Configuration.create().build();
    keyJsonConverter.configure(converterConfig.asMap(), true);
    valueJsonConverter.configure(converterConfig.asMap(), false);
    keyJsonDeserializer.configure(deserializerConfig.asMap(), true);
    valueJsonDeserializer.configure(deserializerConfig.asMap(), false);
    resetBeforeEachTest();
    consumedLines = new ArrayBlockingQueue<>(getMaximumEnqueuedRecordCount());
    Testing.Files.delete(OFFSET_STORE_PATH);
    OFFSET_STORE_PATH.getParent().toFile().mkdirs();
}