Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 51 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method raw_udt_value_from_json.

@Test
void raw_udt_value_from_json() {
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol=value"));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", "{\"udtmem1\": 42, \"udtmem2\": \"the answer\"}", Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(98761234L);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).build();
    udt.attach(session.getContext());
    assertThat(row.getUdtValue("udtcol")).isEqualTo(udt.newValue(42, "the answer"));
}

Example 52 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method null_in_json.

@Test
void null_in_json() {
    // Make a row with some value for textcol to start with.
    session.execute("INSERT INTO types (bigintcol, textcol) VALUES (1234567, 'got here')");
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, textcol=value.text"));
    String json = "{\"bigint\": 1234567, \"text\": null}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, json, Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database; textcol should be unchanged.
    List<Row> results = session.execute("SELECT bigintcol, textcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.getString("textcol")).isEqualTo("got here");
}

Example 53 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method timezone_and_locale_ISO_ZONED_DATE_TIME.

@Test
void timezone_and_locale_ISO_ZONED_DATE_TIME() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.key, " + "datecol=value.vdate, " + "timecol=value.vtime, " + "timestampcol=value.vtimestamp", ImmutableMap.<String, String>builder().put("topic.mytopic.codec.timeZone", "Europe/Paris").put("topic.mytopic.codec.locale", "fr_FR").put("topic.mytopic.codec.date", "cccc, d MMMM uuuu").put("topic.mytopic.codec.time", "HHmmssSSS").put("topic.mytopic.codec.timestamp", "ISO_ZONED_DATE_TIME").put("topic.mytopic.codec.unit", "SECONDS").build()));
    String value = "{\n" + "  \"key\": 4376,\n" + "  \"vdate\": \"vendredi, 9 mars 2018\",\n" + "  \"vtime\": 171232584,\n" + "  \"vtimestamp\": \"2018-03-09T17:12:32.584+01:00[Europe/Paris]\"\n" + "}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, Schema.STRING);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT datecol, timecol, timestampcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLocalDate("datecol")).isEqualTo(LocalDate.of(2018, 3, 9));
    assertThat(row.getLocalTime("timecol")).isEqualTo(LocalTime.of(17, 12, 32, 584_000_000));
    assertThat(row.getInstant("timestampcol")).isEqualTo(Instant.parse("2018-03-09T16:12:32.584Z"));
}

Example 54 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter_and_ttl.

@Test
void should_insert_json_using_query_parameter_and_ttl() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(// when user provide own query, the ttlTimeUnit is ignored
    String.format("topic.mytopic.%s.%s.ttlTimeUnit", keyspaceName, "types"), "HOURS").put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol) USING TTL :ttl", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, ttl=value.ttl", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000).put("ttl", 100000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, ttl(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    assertTtl(row.getInt(2), 100000);
}

Example 55 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter.

@Test
void should_insert_json_using_query_parameter() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol)", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int", extras));
    String value = "{\"bigint\": 1234, \"int\": 10000}";
    Long recordTimestamp = 123456L;
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson, recordTimestamp);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    // timestamp from record is ignored with user provided queries
    assertThat(row.getLong(2)).isGreaterThan(recordTimestamp);
}