Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 36 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_map_when_null_to_unset.

@Test
void should_use_query_to_partially_update_map_when_null_to_unset() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("UPDATE %s.types SET mapCol[:key]=:value where bigintcol = :pk", keyspaceName)).put(deletesDisabled("types")).put(String.format("topic.mytopic.%s.types.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("pk=value.pk, key=value.key, value=value.value", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", 10), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    Map<String, Integer> mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    assertThat(mapcol).containsEntry("key_1", 10);
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", null), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT * FROM types").all();
    row = extractAndAssertThatOneRowInResult(results);
    mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    // update will null value will be skipped because nullToUnset = true
    assertThat(mapcol).containsEntry("key_1", 10);
}

Example 37 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_map_and_remove_when_using_null_to_unset_false.

@Test
void should_use_query_to_partially_update_map_and_remove_when_using_null_to_unset_false() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("UPDATE %s.types SET mapCol[:key]=:value where bigintcol = :pk", keyspaceName)).put(deletesDisabled("types")).put(String.format("topic.mytopic.%s.types.nullToUnset", keyspaceName), "false").build();
    taskConfigs.add(makeConnectorProperties("pk=value.pk, key=value.key, value=value.value", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", 10), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    Map<String, Integer> mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(1);
    assertThat(mapcol).containsEntry("key_1", 10);
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pk", 98761234).put("key", "key_1").put("value", null), Schema.STRING, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT * FROM types").all();
    // setting value for map = null when nullToUnset = false will cause the record to be removed
    assertThat(results.size()).isEqualTo(0);
}

Example 38 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class RawDataEndToEndCCMIT method raw_bytearray_value_as_string.

@Test
void raw_bytearray_value_as_string() {
    // we are treating a byte[] ad a String, in order to support schemaless topics
    taskConfigs.add(makeConnectorProperties("textcol=value,bigintcol=key"));
    byte[] raw = "foo".getBytes(UTF_8);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "123", raw, recordType);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT textcol,bigintcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getString("textcol")).isEqualTo("foo");
    assertThat(row.getLong("bigintcol")).isEqualTo(123);
}

Example 39 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class DeleteCCMIT method delete_compound_key.

@Test
void delete_compound_key() {
    // First insert a row...
    session.execute("INSERT INTO small_compound (bigintcol, booleancol, intcol) VALUES (1234567, true, 42)");
    List<Row> results = session.execute("SELECT * FROM small_compound").all();
    assertThat(results.size()).isEqualTo(1);
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, booleancol=value.boolean, intcol=value.int", "small_compound", null));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("boolean", true).put("int", null);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, recordType);
    runTaskWithRecords(record);
    // Verify that the record was deleted from the database.
    results = session.execute("SELECT * FROM small_compound").all();
    assertThat(results.size()).isEqualTo(0);
}

Example 40 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class NowFunctionCCMIT method delete_simple_key_json_when_using_now_function_in_mapping.

@Test
void delete_simple_key_json_when_using_now_function_in_mapping() {
    // First insert a row...
    session.execute("INSERT INTO pk_value_with_timeuuid (my_pk, my_value, loaded_at) VALUES (1234567, true, now())");
    List<Row> results = session.execute("SELECT * FROM pk_value_with_timeuuid").all();
    assertThat(results.size()).isEqualTo(1);
    // now() function call is ignored when null value is send - DELETE will be performed
    taskConfigs.add(makeConnectorProperties("my_pk=value.my_pk, my_value=value.my_value, loaded_at=now()", "pk_value_with_timeuuid", null));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("my_value").type(SchemaType.STRING);
    builder.field("my_pk").type(SchemaType.DOUBLE);
    Schema recordType = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    // Set up records for "mytopic"
    String json = "{\"my_pk\": 1234567, \"my_value\": null}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, json, Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was deleted from the database.
    results = session.execute("SELECT * FROM pk_value_with_timeuuid").all();
    assertThat(results.size()).isEqualTo(0);
}