Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 31 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_struct_with_query_parameter.

@Test
void should_insert_struct_with_query_parameter() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigint_col, :int_col) USING TIMESTAMP :timestamp and TTL 1000", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigint_col=value.bigint, int_col=value.int, timestamp=value.int", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234567L).put("int", 1000), recordTypeJson, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol), ttl(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getInt("intcol")).isEqualTo(1000);
    assertThat(row.getLong(2)).isEqualTo(1000L);
    assertTtl(row.getInt(3), 1000);
}

Example 32 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_update_query_on_non_frozen_udt_and_override_with_null_when_null_to_unset_false.

@Test
void should_use_update_query_on_non_frozen_udt_and_override_with_null_when_null_to_unset_false() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen
    String.format("UPDATE %s.types_with_frozen set udtColNotFrozen.udtmem1=:udtcol1, udtColNotFrozen.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "false").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).withAttachmentPoint(session.getContext()).build();
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
    // insert record with only one column from udt - udtmem2 is null
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", null), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    row = extractAndAssertThatOneRowInResult(results);
    // nullToUnset for this topic was set to false, so the udtmem2 field was updated, the value was
    // overridden with null
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, null));
}

Example 33 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_allow_insert_json_using_query_parameter_with_bound_variables_different_than_cql_columns.

@Test
void should_allow_insert_json_using_query_parameter_with_bound_variables_different_than_cql_columns() {
    // when providing custom query, the connector is not validating bound variables from prepared
    // statements user needs to take care of the query requirements on their own.
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:some_name, :some_name_2)", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("some_name=value.bigint, some_name_2=value.int", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
}

Example 34 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_fail_insert_json_using_query_parameter_with_deletes_enabled.

@Test
void should_fail_insert_json_using_query_parameter_with_deletes_enabled() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol)", keyspaceName)).put(deletesEnabled()).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int", extras));
    Long recordTimestamp = 123456L;
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson, recordTimestamp);
    assertThatThrownBy(() -> runTaskWithRecords(record)).isInstanceOf(ConfigException.class).hasMessageContaining("If you want to provide own query, set the deletesEnabled to false.");
}

Example 35 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter_and_timestamp.

@Test
void should_insert_json_using_query_parameter_and_timestamp() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(// when user provide own query, the timestampTimeUnit is ignored
    String.format("topic.mytopic.%s.%s.timestampTimeUnit", keyspaceName, "types"), "HOURS").put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol) USING TIMESTAMP :timestamp", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, timestamp=value.timestamp", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000).put("timestamp", 100000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    assertThat(row.getLong(2)).isEqualTo(100000L);
}