Examples with GenericRecordImpl com.datastax.oss.sink.pulsar.GenericRecordImpl used on opensource projects

Example 21 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_fail_when_use_query_to_partially_update_frozen_udt.

@Test
void should_fail_when_use_query_to_partially_update_frozen_udt() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen - here we are using frozen so the execption will be thrown
    String.format("UPDATE %s.types_with_frozen set udtCol.udtmem1=:udtcol1, udtCol.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    assertThatThrownBy(() -> runTaskWithRecords(record)).hasCauseInstanceOf(InvalidQueryException.class).hasStackTraceContaining("for frozen UDT column udtcol");
}

Example 22 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_query_to_partially_update_non_frozen_udt_when_null_to_unset.

@Test
void should_use_query_to_partially_update_non_frozen_udt_when_null_to_unset() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen
    String.format("UPDATE %s.types_with_frozen set udtColNotFrozen.udtmem1=:udtcol1, udtColNotFrozen.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "true").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).withAttachmentPoint(session.getContext()).build();
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
    // insert record with only one column from udt - udtmem2 is null
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", null), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    row = extractAndAssertThatOneRowInResult(results);
    // default for topic is nullToUnset, so the udtmem2 field was not updated, the value was not
    // overridden
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
}

Example 23 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_struct_with_query_parameter.

@Test
void should_insert_struct_with_query_parameter() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigint_col, :int_col) USING TIMESTAMP :timestamp and TTL 1000", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigint_col=value.bigint, int_col=value.int, timestamp=value.int", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234567L).put("int", 1000), recordTypeJson, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol), ttl(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getInt("intcol")).isEqualTo(1000);
    assertThat(row.getLong(2)).isEqualTo(1000L);
    assertTtl(row.getInt(3), 1000);
}

Example 24 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_use_update_query_on_non_frozen_udt_and_override_with_null_when_null_to_unset_false.

@Test
void should_use_update_query_on_non_frozen_udt_and_override_with_null_when_null_to_unset_false() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types_with_frozen"), // not frozen
    String.format("UPDATE %s.types_with_frozen set udtColNotFrozen.udtmem1=:udtcol1, udtColNotFrozen.udtmem2=:udtcol2 where bigintCol=:bigintcol", keyspaceName)).put(deletesDisabled("types_with_frozen")).put(String.format("topic.mytopic.%s.types_with_frozen.nullToUnset", keyspaceName), "false").build();
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol1=value.udtmem1, udtcol2=value.udtmem2", "types_with_frozen", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer"), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    Row row = extractAndAssertThatOneRowInResult(results);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).withAttachmentPoint(session.getContext()).build();
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, "the answer"));
    // insert record with only one column from udt - udtmem2 is null
    record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", null), UDT_SCHEMA, 153000987L);
    runTaskWithRecords(record);
    results = session.execute("SELECT bigintcol, udtColNotFrozen FROM types_with_frozen").all();
    row = extractAndAssertThatOneRowInResult(results);
    // nullToUnset for this topic was set to false, so the udtmem2 field was updated, the value was
    // overridden with null
    assertThat(row.getUdtValue("udtColNotFrozen")).isEqualTo(udt.newValue(42, null));
}

Example 25 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_allow_insert_json_using_query_parameter_with_bound_variables_different_than_cql_columns.

@Test
void should_allow_insert_json_using_query_parameter_with_bound_variables_different_than_cql_columns() {
    // when providing custom query, the connector is not validating bound variables from prepared
    // statements user needs to take care of the query requirements on their own.
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:some_name, :some_name_2)", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("some_name=value.bigint, some_name_2=value.int", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
}