Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 26 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method simple_json_value_only.

@Test
void simple_json_value_only() {
    // Since the well-established JSON converter codecs do all the heavy lifting,
    // we don't test json very deeply here.
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "booleancol=value.boolean, " + "doublecol=value.double, " + "floatcol=value.float, " + "intcol=value.int, " + "smallintcol=value.smallint, " + "textcol=value.text, " + "tinyintcol=value.tinyint"));
    Long baseValue = 1234567L;
    String value = String.format("{\"bigint\": %d, " + "\"boolean\": %b, " + "\"double\": %f, " + "\"float\": %f, " + "\"int\": %d, " + "\"smallint\": %d, " + "\"text\": \"%s\", " + "\"tinyint\": %d}", baseValue, (baseValue.intValue() & 1) == 1, (double) baseValue + 0.123, baseValue.floatValue() + 0.987f, baseValue.intValue(), baseValue.shortValue(), baseValue.toString(), baseValue.byteValue());
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseValue + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
    assertThat(row.getShort("smallintcol")).isEqualTo(baseValue.shortValue());
    assertThat(row.getString("textcol")).isEqualTo(baseValue.toString());
    assertThat(row.getByte("tinyintcol")).isEqualTo(baseValue.byteValue());
}

Example 27 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method update_counter_table.

@Test
void update_counter_table() {
    session.execute("CREATE TABLE IF NOT EXISTS mycounter " + "(c1 int, c2 int, c3 counter, c4 counter, PRIMARY KEY (c1, c2))");
    session.execute("TRUNCATE mycounter");
    taskConfigs.add(makeConnectorProperties("bigintcol=value.key", ImmutableMap.of(String.format("topic.ctr.%s.mycounter.mapping", keyspaceName), "c1=value.f1, c2=value.f2, c3=value.f3, c4=value.f4")));
    String value = "{" + "\"f1\": 1, " + "\"f2\": 2, " + "\"f3\": 3, " + "\"f4\": 4" + "}";
    PulsarRecordImpl record = new PulsarRecordImpl(// NO TIMESTAMP
    "persistent://tn/ns/ctr", // NO TIMESTAMP
    null, // NO TIMESTAMP
    value, // NO TIMESTAMP
    Schema.STRING, // NO TIMESTAMP
    null);
    // Insert the record twice; the counter columns should accrue.
    runTaskWithRecords(record, record);
    // Verify...
    List<Row> results = session.execute("SELECT * FROM mycounter").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("c3")).isEqualTo(6);
    assertThat(row.getLong("c4")).isEqualTo(8);
}

Example 28 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method complex_json_value_only.

@Test
void complex_json_value_only() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.f1, mapcol=value.f2"));
    String value = "{\"f1\": 42, \"f2\": {\"sub1\": 37, \"sub2\": 96}}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(42);
    Map<String, Integer> mapcol = row.getMap("mapcol", String.class, Integer.class);
    assert mapcol != null;
    assertThat(mapcol.size()).isEqualTo(2);
    assertThat(mapcol).containsEntry("sub1", 37).containsEntry("sub2", 96);
}

Example 29 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_fail_when_use_query_to_partially_update_frozen_udt.

@Test
void should_fail_when_use_query_to_partially_update_frozen_udt() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen - here we are using frozen so the execption will be thrown
    String.format("UPDATE %s.types_with_frozen set udtCol.udtmem1=:udtcol1, udtCol.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    assertThatThrownBy(() -> runTaskWithRecords(record)).hasCauseInstanceOf(InvalidQueryException.class).hasStackTraceContaining("for frozen UDT column udtcol");
}

Example 30 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_non_frozen_udt_when_null_to_unset.

@Test
void should_use_query_to_partially_update_non_frozen_udt_when_null_to_unset() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen
    String.format("UPDATE %s.types_with_frozen set udtColNotFrozen.udtmem1=:udtcol1, udtColNotFrozen.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).withAttachmentPoint(session.getContext()).build();
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
    // insert record with only one column from udt - udtmem2 is null
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", null), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    row = extractAndAssertThatOneRowInResult(results);
    // default for topic is nullToUnset, so the udtmem2 field was not updated, the value was not
    // overridden
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
}