Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 6 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_missing.

@Test
void struct_optional_fields_missing() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, doublecol=value.double"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("int", baseValue.intValue());
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
}

Example 7 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_with_values.

@Test
void struct_optional_fields_with_values() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "booleancol=value.boolean, " + "doublecol=value.double, " + "floatcol=value.float, " + "intcol=value.int, " + "textcol=value.text, " + "blobcol=value.blob"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    byte[] blobValue = new byte[] { 12, 22, 32 };
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("boolean", (baseValue.intValue() & 1) == 1).put("double", (double) baseValue + 0.123).put("float", baseValue.floatValue() + 0.987f).put("int", baseValue.intValue()).put("smallint", baseValue.shortValue()).put("text", baseValue.toString()).put("blob", blobValue);
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseValue + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
    assertThat(row.getString("textcol")).isEqualTo(baseValue.toString());
    ByteBuffer blobcol = row.getByteBuffer("blobcol");
    assertThat(blobcol).isNotNull();
    assertThat(Bytes.getArray(blobcol)).isEqualTo(blobValue);
}

Example 8 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method single_record_multiple_tables.

@Test
void single_record_multiple_tables() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, booleancol=value.boolean, intcol=value.int", ImmutableMap.of(String.format("topic.mytopic.%s.small_simple.mapping", keyspaceName), "bigintcol=value.bigint, intcol=value.int")));
    // Set up records for "mytopic"
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("bigint").type(SchemaType.INT64);
    builder.field("boolean").type(SchemaType.BOOLEAN);
    builder.field("int").type(SchemaType.INT32);
    Schema schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("boolean", true).put("int", 5725);
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema);
    runTaskWithRecords(record);
    // Verify that a record was inserted in each of small_simple and types tables.
    {
        List<Row> results = session.execute("SELECT * FROM small_simple").all();
        assertThat(results.size()).isEqualTo(1);
        Row row = results.get(0);
        assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
        assertThat(row.get("booleancol", GenericType.BOOLEAN)).isNull();
        assertThat(row.getInt("intcol")).isEqualTo(5725);
    }
    {
        List<Row> results = session.execute("SELECT * FROM types").all();
        assertThat(results.size()).isEqualTo(1);
        Row row = results.get(0);
        assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
        assertThat(row.getBoolean("booleancol")).isTrue();
        assertThat(row.getInt("intcol")).isEqualTo(5725);
    }
}

Example 9 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method multiple_records_multiple_topics.

@Test
void multiple_records_multiple_topics() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double", ImmutableMap.of(String.format("topic.yourtopic.%s.types.mapping", keyspaceName), "bigintcol=key, intcol=value.int")));
    // Set up records for "mytopic"
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("bigint").type(SchemaType.INT64);
    builder.field("double").type(SchemaType.DOUBLE);
    Schema recordTypeUtd = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl value1 = new GenericRecordImpl().put("bigint", 1234567L).put("double", 42.0);
    GenericRecordImpl value2 = new GenericRecordImpl().put("bigint", 9876543L).put("double", 21.0);
    PulsarRecordImpl record1 = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value1, recordTypeUtd);
    PulsarRecordImpl record2 = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value2, recordTypeUtd);
    // Set up a record for "yourtopic"
    PulsarRecordImpl record3 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "5555", new GenericRecordImpl().put("int", 3333), recordType);
    runTaskWithRecords(record1, record2, record3);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, doublecol, intcol FROM types").all();
    assertThat(results.size()).isEqualTo(3);
    for (Row row : results) {
        if (row.getLong("bigintcol") == 1234567L) {
            assertThat(row.getDouble("doublecol")).isEqualTo(42.0);
            assertThat(row.getObject("intcol")).isNull();
        } else if (row.getLong("bigintcol") == 9876543L) {
            assertThat(row.getDouble("doublecol")).isEqualTo(21.0);
            assertThat(row.getObject("intcol")).isNull();
        } else if (row.getLong("bigintcol") == 5555L) {
            assertThat(row.getObject("doublecol")).isNull();
            assertThat(row.getInt("intcol")).isEqualTo(3333);
        }
    }
}

Example 10 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method raw_udt_value_from_struct.

@Test
void raw_udt_value_from_struct() {
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol=value"));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("udtmem1").type(SchemaType.INT32);
    builder.field("udtmem2").type(SchemaType.STRING);
    Schema recordTypeUtd = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl value = new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer");
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", value, recordTypeUtd);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(98761234L);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).build();
    udt.attach(session.getContext());
    assertThat(row.getUdtValue("udtcol")).isEqualTo(udt.newValue(42, "the answer"));
}