Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 41 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class NowFunctionCCMIT method should_insert_value_using_now_function_for_two_dse_columns.

@Test
void should_insert_value_using_now_function_for_two_dse_columns() {
    // given
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, loaded_at=now(), loaded_at2=now()"));
    // when
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234567), recordTypeJson);
    runTaskWithRecords(record);
    // then
    List<Row> results = session.execute("SELECT bigintcol, loaded_at, loaded_at2 FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    UUID loadedAt = row.get("loaded_at", TypeCodecs.TIMEUUID);
    UUID loadedAt2 = row.get("loaded_at2", TypeCodecs.TIMEUUID);
    // columns inserted using now() should have different TIMEUUID values
    assertThat(loadedAt).isNotEqualTo(loadedAt2);
}

Example 42 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class HeadersCCMIT method should_use_values_from_header_in_mapping.

/**
 * Test for KAF-142
 */
@Test
void should_use_values_from_header_in_mapping() {
    // values used in this test are random and irrelevant for the test
    // given
    taskConfigs.add(makeConnectorProperties("bigintcol=header.bigint," + "doublecol=header.double," + "textcol=header.text," + "booleancol=header.boolean," + "floatcol=header.float," + "intcol=header.int"));
    Long baseValue = 1234567L;
    String json = "{\"bigint\": 1234567}";
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", json, 1L, Schema.INT64).setProperty("bigint", baseValue + "").setProperty("double", baseValue.doubleValue() + "").setProperty("text", "value").setProperty("boolean", "false").setProperty("float", baseValue.floatValue() + "").setProperty("int", baseValue.intValue() + "");
    // when
    runTaskWithRecords(record);
    // then
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue.longValue());
    assertThat(row.getDouble("doublecol")).isEqualTo(baseValue.doubleValue());
    assertThat(row.getString("textcol")).isEqualTo("value");
    assertThat(row.getBoolean("booleancol")).isEqualTo(false);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue());
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
}

Example 43 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_value_struct_field.

@Test
void struct_value_struct_field() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "udtcol=value.struct, " + "booleanudtcol=value.booleanstruct"));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBeanInt");
    builder.field("udtmem1").type(SchemaType.INT32);
    builder.field("udtmem2").type(SchemaType.STRING);
    GenericSchema fieldSchema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl fieldValue = new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer");
    RecordSchemaBuilder builder2 = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBeanBoolean");
    builder2.field("udtmem1").type(SchemaType.BOOLEAN);
    builder2.field("udtmem2").type(SchemaType.STRING);
    GenericSchema booleanFieldSchema = org.apache.pulsar.client.api.Schema.generic(builder2.build(SchemaType.AVRO));
    GenericRecordImpl booleanFieldValue = new GenericRecordImpl().put("udtmem1", true).put("udtmem2", "the answer");
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBeanRoot");
    builderRoot.field("bigint").type(SchemaType.INT64);
    builderRoot.field("struct", fieldSchema).type(SchemaType.AVRO);
    builderRoot.field("booleanstruct", booleanFieldSchema).type(SchemaType.AVRO);
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", 1234567L).put("struct", fieldValue).put("booleanstruct", booleanFieldValue);
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).build();
    udt.attach(session.getContext());
    assertThat(row.getUdtValue("udtcol")).isEqualTo(udt.newValue(42, "the answer"));
    UserDefinedType booleanUdt = new UserDefinedTypeBuilder(keyspaceName, "mybooleanudt").withField("udtmem1", DataTypes.BOOLEAN).withField("udtmem2", DataTypes.TEXT).build();
    booleanUdt.attach(session.getContext());
    assertThat(row.getUdtValue("booleanudtcol")).isEqualTo(booleanUdt.newValue(true, "the answer"));
}

Example 44 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_value_only.

@Test
void struct_value_only() throws ParseException {
    String withDateRange = hasDateRange ? "daterangecol=value.daterange, " : "";
    String withGeotypes = ccm.getClusterType() == DSE ? "pointcol=value.point, linestringcol=value.linestring, polygoncol=value.polygon, " : "";
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "booleancol=value.boolean, " + "doublecol=value.double, " + "floatcol=value.float, " + "intcol=value.int, " + "textcol=value.text, " + withGeotypes + withDateRange + "blobcol=value.blob"));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("bigint").type(SchemaType.INT64);
    builder.field("boolean").type(SchemaType.BOOLEAN);
    builder.field("double").type(SchemaType.DOUBLE);
    builder.field("float").type(SchemaType.FLOAT);
    builder.field("int").type(SchemaType.INT32);
    builder.field("text").type(SchemaType.STRING);
    builder.field("blob").type(SchemaType.BYTES);
    builder.field("point").type(SchemaType.STRING);
    builder.field("linestring").type(SchemaType.STRING);
    builder.field("polygon").type(SchemaType.STRING);
    builder.field("daterange").type(SchemaType.STRING);
    Schema schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    byte[] blobValue = new byte[] { 12, 22, 32 };
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("boolean", (baseValue.intValue() & 1) == 1).put("double", (double) baseValue + 0.123).put("float", baseValue.floatValue() + 0.987f).put("int", baseValue.intValue()).put("text", baseValue.toString()).put("blob", blobValue).put("point", "POINT (32.0 64.0)").put("linestring", "LINESTRING (32.0 64.0, 48.5 96.5)").put("polygon", "POLYGON ((0.0 0.0, 20.0 0.0, 25.0 25.0, 0.0 25.0, 0.0 0.0))").put("daterange", "[* TO 2014-12-01]");
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseValue + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
    assertThat(row.getString("textcol")).isEqualTo(baseValue.toString());
    ByteBuffer blobcol = row.getByteBuffer("blobcol");
    assertThat(blobcol).isNotNull();
    assertThat(Bytes.getArray(blobcol)).isEqualTo(blobValue);
    if (ccm.getClusterType() == DSE) {
        assertThat(row.get("pointcol", GenericType.of(Point.class))).isEqualTo(new DefaultPoint(32.0, 64.0));
        assertThat(row.get("linestringcol", GenericType.of(LineString.class))).isEqualTo(new DefaultLineString(new DefaultPoint(32.0, 64.0), new DefaultPoint(48.5, 96.5)));
        assertThat(row.get("polygoncol", GenericType.of(Polygon.class))).isEqualTo(new DefaultPolygon(new DefaultPoint(0, 0), new DefaultPoint(20, 0), new DefaultPoint(25, 25), new DefaultPoint(0, 25), new DefaultPoint(0, 0)));
    }
    if (hasDateRange) {
        assertThat(row.get("daterangecol", GenericType.of(DateRange.class))).isEqualTo(DateRange.parse("[* TO 2014-12-01]"));
    }
}

Example 45 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method single_map_quoted_fields_to_quoted_columns.

/**
 * Test for KAF-83 (case-sensitive fields and columns). In Pulsar we cannot have field names with
 * dots and spaces, so this test is less significant than the one for Pulsar sink
 */
@Test
void single_map_quoted_fields_to_quoted_columns() {
    session.execute(SimpleStatement.builder("CREATE TABLE \"CASE_SENSITIVE\" (" + "\"bigint col\" bigint, " + "\"boolean-col\" boolean, " + "\"INT COL\" int," + "\"TEXT.COL\" text," + "PRIMARY KEY (\"bigint col\", \"boolean-col\")" + ")").setTimeout(Duration.ofSeconds(10)).build());
    taskConfigs.add(makeConnectorProperties("\"bigint col\" = \"key.bigint field\", " + "\"boolean-col\" = \"key.boolean-field\", " + "\"INT COL\" = \"value.INTFIELD\", " + "\"TEXT.COL\" = \"value.TEXTFIELD\"", "CASE_SENSITIVE", null));
    // Set up records for "mytopic"
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("INTFIELD").type(SchemaType.INT64);
    builder.field("TEXTFIELD").type(SchemaType.STRING);
    Schema schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.JSON));
    String key = "{\"bigint field\":1234567,\"boolean-field\":true}";
    GenericRecordImpl value = new GenericRecordImpl().put("INTFIELD", 5725).put("TEXTFIELD", "foo");
    // Note: with the current mapping grammar, it is not possible to distinguish f1.f2 (i.e. a field
    // "f1" containing a nested field "f2") from a field named "f1.f2".
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", key, value, schema);
    runTaskWithRecords(record);
    // Verify that a record was inserted
    List<Row> results = session.execute("SELECT * FROM \"CASE_SENSITIVE\"").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("\"bigint col\"")).isEqualTo(1234567L);
    assertThat(row.getBoolean("\"boolean-col\"")).isTrue();
    assertThat(row.getInt("\"INT COL\"")).isEqualTo(5725);
    assertThat(row.getString("\"TEXT.COL\"")).isEqualTo("foo");
}