Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 11 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class WriteTimestampAndTtlCCMIT method should_insert_record_with_timestamp_provided_via_mapping_and_validate_timestamp_of_table.

/**
 * Test for KAF-46.
 */
@ParameterizedTest(name = "[{index}] schema={0}, timestampValue={1}, expectedTimestampValue={2}")
@MethodSource("timestampColProvider")
void should_insert_record_with_timestamp_provided_via_mapping_and_validate_timestamp_of_table(Schema schema, Number timestampValue, Number expectedTimestampValue) {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double, __timestamp = value.timestampcol", ImmutableMap.of(String.format("topic.mytopic.%s.%s.timestampTimeUnit", keyspaceName, "types"), "MILLISECONDS")));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("double", 42.0).put("timestampcol", timestampValue);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, doublecol, writetime(doublecol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getDouble("doublecol")).isEqualTo(42.0);
    assertThat(row.getLong(2)).isEqualTo(expectedTimestampValue.longValue());
}

Example 12 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class WriteTimestampAndTtlCCMIT method timestamp.

@Test
void timestamp() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double"));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("bigint").type(SchemaType.INT64);
    builder.field("double").type(SchemaType.DOUBLE);
    Schema schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("double", 42.0);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, doublecol, writetime(doublecol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getDouble("doublecol")).isEqualTo(42.0);
    assertThat(row.getLong(2)).isEqualTo(153000987000L);
}

Example 13 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class WriteTimestampAndTtlCCMIT method should_insert_record_with_ttl_provided_via_mapping_and_validate_ttl_of_table.

/**
 * Test for KAF-107.
 */
@ParameterizedTest(name = "[{index}] schema={0}, ttlValue={1}, expectedTtlValue={2}")
@MethodSource("ttlColProvider")
void should_insert_record_with_ttl_provided_via_mapping_and_validate_ttl_of_table(Schema schema, Number ttlValue, Number expectedTtlValue) {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double, __ttl = value.ttlcol", ImmutableMap.of(String.format("topic.mytopic.%s.%s.ttlTimeUnit", keyspaceName, "types"), "MILLISECONDS")));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("double", 42.0).put("ttlcol", ttlValue);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema, 153000987L);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, doublecol, ttl(doublecol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getDouble("doublecol")).isEqualTo(42.0);
    assertTtl(row.getInt(2), expectedTtlValue);
}

Example 14 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class WriteTimestampAndTtlCCMIT method should_extract_ttl_from_json_and_use_as_ttl_column.

@Test
void should_extract_ttl_from_json_and_use_as_ttl_column() {
    // given
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double, __ttl = value.ttlcol", ImmutableMap.of(String.format("topic.mytopic.%s.%s.ttlTimeUnit", keyspaceName, "types"), "MILLISECONDS")));
    // when
    String json = "{\"bigint\": 1234567, \"double\": 42.0, \"ttlcol\": 1000000}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, json, Schema.STRING);
    runTaskWithRecords(record);
    // then
    List<Row> results = session.execute("SELECT bigintcol, doublecol, ttl(doublecol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getDouble("doublecol")).isEqualTo(42.0);
    assertTtl(row.getInt(2), 1000);
}

Example 15 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class WriteTimestampAndTtlCCMIT method should_extract_timestamp_from_json_and_use_existing_column_as_timestamp.

/**
 * Test for KAF-46.
 */
@Test
void should_extract_timestamp_from_json_and_use_existing_column_as_timestamp() {
    // given
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double, __timestamp = value.double", ImmutableMap.of(String.format("topic.mytopic.%s.%s.ttlTimeUnit", keyspaceName, "types"), "MILLISECONDS", String.format("topic.mytopic.%s.%s.timestampTimeUnit", keyspaceName, "types"), "MILLISECONDS")));
    // when
    String json = "{\"bigint\": 1234567, \"double\": 1000.0}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, json, Schema.STRING);
    runTaskWithRecords(record);
    // then
    List<Row> results = session.execute("SELECT bigintcol, doublecol, writetime(doublecol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getDouble("doublecol")).isEqualTo(1000.0);
    assertThat(row.getLong(2)).isEqualTo(1_000_000L);
}