Examples with GenericRecordImpl com.datastax.oss.sink.pulsar.GenericRecordImpl used on opensource projects

Example 26 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_fail_insert_json_using_query_parameter_with_deletes_enabled.

@Test
void should_fail_insert_json_using_query_parameter_with_deletes_enabled() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol)", keyspaceName)).put(deletesEnabled()).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int", extras));
    Long recordTimestamp = 123456L;
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson, recordTimestamp);
    assertThatThrownBy(() -> runTaskWithRecords(record)).isInstanceOf(ConfigException.class).hasMessageContaining("If you want to provide own query, set the deletesEnabled to false.");
}

Example 27 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter_and_timestamp.

@Test
void should_insert_json_using_query_parameter_and_timestamp() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(// when user provide own query, the timestampTimeUnit is ignored
    String.format("topic.mytopic.%s.%s.timestampTimeUnit", keyspaceName, "types"), "HOURS").put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol) USING TIMESTAMP :timestamp", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, timestamp=value.timestamp", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000).put("timestamp", 100000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    assertThat(row.getLong(2)).isEqualTo(100000L);
}

Example 28 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_map_when_null_to_unset.

@Test
void should_use_query_to_partially_update_map_when_null_to_unset() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("UPDATE %s.types SET mapCol[:key]=:value where bigintcol = :pk", keyspaceName)).put(deletesDisabled("types")).put(String.format("topic.mytopic.%s.types.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("pk=value.pk, key=value.key, value=value.value", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", 10), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    Map<String, Integer> mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    assertThat(mapcol).containsEntry("key_1", 10);
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", null), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT * FROM types").all();
    row = extractAndAssertThatOneRowInResult(results);
    mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    // update will null value will be skipped because nullToUnset = true
    assertThat(mapcol).containsEntry("key_1", 10);
}

Example 29 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_map_and_remove_when_using_null_to_unset_false.

@Test
void should_use_query_to_partially_update_map_and_remove_when_using_null_to_unset_false() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("UPDATE %s.types SET mapCol[:key]=:value where bigintcol = :pk", keyspaceName)).put(deletesDisabled("types")).put(String.format("topic.mytopic.%s.types.nullToUnset", keyspaceName), "false").build();
    taskConfigs.add(makeConnectorProperties("pk=value.pk, key=value.key, value=value.value", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", 10), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    Map<String, Integer> mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    assertThat(mapcol).containsEntry("key_1", 10);
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", null), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT * FROM types").all();
    // setting value for map = null when nullToUnset = false will cause the record to be removed
    assertThat(results.size()).isEqualTo(0);
}

Example 30 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class DeleteCCMIT method delete_compound_key.

@Test
void delete_compound_key() {
    // First insert a row...
    session.execute("INSERT INTO small_compound (bigintcol, booleancol, intcol) VALUES (1234567, true, 42)");
    List<Row> results = session.execute("SELECT * FROM small_compound").all();
    assertThat(results.size()).isEqualTo(1);
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, booleancol=value.boolean, intcol=value.int", "small_compound", null));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("boolean", true).put("int", null);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, recordType);
    runTaskWithRecords(record);
    // Verify that the record was deleted from the database.
    results = session.execute("SELECT * FROM small_compound").all();
    assertThat(results.size()).isEqualTo(0);
}