Examples with GenericRecordImpl com.datastax.oss.sink.pulsar.GenericRecordImpl used on opensource projects

Example 36 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class JsonEndToEndCCMIT method json_key_struct_value.

@Test
void json_key_struct_value() {
    // Map various fields from the key and value to columns.
    taskConfigs.add(makeConnectorProperties("bigintcol=key.bigint, " + "booleancol=value.boolean, " + "doublecol=key.double, " + "floatcol=value.float, " + "intcol=key.int, " + "textcol=key.text "));
    // Use a Struct for the value.
    Long baseValue = 98761234L;
    GenericRecordImpl structValue = new GenericRecordImpl().put("bigint", baseValue).put("boolean", (baseValue.intValue() & 1) == 1).put("double", (double) baseValue + 0.123).put("float", baseValue.floatValue() + 0.987f).put("int", baseValue.intValue()).put("text", baseValue.toString());
    // Use JSON for the key.
    Long baseKey = 1234567L;
    String jsonKey = String.format("{\"bigint\": %d, " + "\"boolean\": %b, " + "\"double\": %f, " + "\"float\": %f, " + "\"int\": %d, " + "\"smallint\": %d, " + "\"text\": \"%s\", " + "\"tinyint\": %d}", baseKey, (baseKey.intValue() & 1) == 1, (double) baseKey + 0.123, baseKey.floatValue() + 0.987f, baseKey.intValue(), baseKey.shortValue(), baseKey.toString(), baseKey.byteValue());
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", jsonKey, structValue, recordType);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseKey);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseKey + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseKey.intValue());
    assertThat(row.getString("textcol")).isEqualTo(baseKey.toString());
}

Example 37 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter_and_ttl.

@Test
void should_insert_json_using_query_parameter_and_ttl() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(// when user provide own query, the ttlTimeUnit is ignored
    String.format("topic.mytopic.%s.%s.ttlTimeUnit", keyspaceName, "types"), "HOURS").put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol) USING TTL :ttl", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, ttl=value.ttl", extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000).put("ttl", 100000), recordTypeJson);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, ttl(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    assertTtl(row.getInt(2), 100000);
}

Example 38 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_insert_json_using_query_parameter.

@Test
void should_insert_json_using_query_parameter() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("INSERT INTO %s.types (bigintCol, intCol) VALUES (:bigintcol, :intcol)", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int", extras));
    String value = "{\"bigint\": 1234, \"int\": 10000}";
    Long recordTimestamp = 123456L;
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234).put("int", 10000), recordTypeJson, recordTimestamp);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, intcol, writetime(intcol) FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getInt("intcol")).isEqualTo(10000);
    // timestamp from record is ignored with user provided queries
    assertThat(row.getLong(2)).isGreaterThan(recordTimestamp);
}

Example 39 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class ProvidedQueryCCMIT method should_update_json_using_query_parameter.

@Test
void should_update_json_using_query_parameter() {
    ImmutableMap<String, String> extras = ImmutableMap.<String, String>builder().put(queryParameter("types"), String.format("UPDATE %s.types SET listCol = listCol + [1] where bigintcol = :pkey", keyspaceName)).put(deletesDisabled("types")).build();
    taskConfigs.add(makeConnectorProperties("pkey=value.pkey, newitem=value.newitem", extras));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("pkey").type(SchemaType.INT32);
    builder.field("newitem").type(SchemaType.INT32);
    Schema schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pkey", 1234).put("newitem", 1), schema);
    PulsarRecordImpl record2 = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("pkey", 1234).put("newitem", 1), schema);
    runTaskWithRecords(record, record2);
    // Verify that two values were append to listcol
    List<Row> results = session.execute("SELECT * FROM types where bigintcol = 1234").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234);
    assertThat(row.getList("listcol", Integer.class)).isEqualTo(ImmutableList.of(1, 1));
}

Example 40 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class DeleteCCMIT method delete_simple_key.

@Test
void delete_simple_key() {
    // First insert a row...
    session.execute("INSERT INTO pk_value (my_pk, my_value) VALUES (1234567, true)");
    List<Row> results = session.execute("SELECT * FROM pk_value").all();
    assertThat(results.size()).isEqualTo(1);
    taskConfigs.add(makeConnectorProperties("my_pk=key, my_value=value.my_value", "pk_value", null));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "1234567", new GenericRecordImpl(), recordType);
    runTaskWithRecords(record);
    // Verify that the record was deleted from the database.
    results = session.execute("SELECT * FROM pk_value").all();
    assertThat(results.size()).isEqualTo(0);
}