Examples with RecordSchemaBuilder org.apache.pulsar.client.api.schema.RecordSchemaBuilder used on opensource projects

Example 1 with RecordSchemaBuilder

use of org.apache.pulsar.client.api.schema.RecordSchemaBuilder in project pulsar by apache.

the class JSONSchemaTest method testJsonGenericRecordBuilder.

@Test
public void testJsonGenericRecordBuilder() {
    JSONSchema<Seller> sellerJsonSchema = JSONSchema.of(Seller.class);
    RecordSchemaBuilder sellerSchemaBuilder = SchemaBuilder.record("seller");
    sellerSchemaBuilder.field("state").type(SchemaType.STRING);
    sellerSchemaBuilder.field("street").type(SchemaType.STRING);
    sellerSchemaBuilder.field("zipCode").type(SchemaType.INT64);
    SchemaInfo sellerSchemaInfo = sellerSchemaBuilder.build(SchemaType.JSON);
    GenericSchemaImpl sellerGenericSchema = GenericSchemaImpl.of(sellerSchemaInfo);
    JSONSchema<PC> pcJsonSchema = JSONSchema.of(PC.class);
    RecordSchemaBuilder pcSchemaBuilder = SchemaBuilder.record("pc");
    pcSchemaBuilder.field("brand").type(SchemaType.STRING);
    pcSchemaBuilder.field("model").type(SchemaType.STRING);
    pcSchemaBuilder.field("gpu").type(SchemaType.STRING);
    pcSchemaBuilder.field("year").type(SchemaType.INT64);
    pcSchemaBuilder.field("seller", sellerGenericSchema).type(SchemaType.JSON).optional();
    SchemaInfo pcGenericSchemaInfo = pcSchemaBuilder.build(SchemaType.JSON);
    GenericSchemaImpl pcGenericSchema = GenericSchemaImpl.of(pcGenericSchemaInfo);
    Seller seller = new Seller("USA", "oakstreet", 9999);
    PC pc = new PC("dell", "g3", 2020, GPU.AMD, seller);
    byte[] bytes = pcJsonSchema.encode(pc);
    Assert.assertTrue(bytes.length > 0);
    Object pc2 = pcJsonSchema.decode(bytes);
    assertEquals(pc, pc2);
    GenericRecord sellerRecord = sellerGenericSchema.newRecordBuilder().set("state", "USA").set("street", "oakstreet").set("zipCode", 9999).build();
    GenericRecord pcRecord = pcGenericSchema.newRecordBuilder().set("brand", "dell").set("model", "g3").set("year", 2020).set("gpu", GPU.AMD).set("seller", sellerRecord).build();
    byte[] bytes3 = pcGenericSchema.encode(pcRecord);
    Assert.assertTrue(bytes3.length > 0);
    GenericRecord pc3Record = pcGenericSchema.decode(bytes3);
    for (Field field : pc3Record.getFields()) {
        assertTrue(pcGenericSchema.getFields().contains(field));
    }
    assertEquals("dell", pc3Record.getField("brand"));
    assertEquals("g3", pc3Record.getField("model"));
    assertEquals(2020, pc3Record.getField("year"));
    assertEquals(GPU.AMD.toString(), pc3Record.getField("gpu"));
    GenericRecord seller3Record = (GenericRecord) pc3Record.getField("seller");
    assertEquals("USA", seller3Record.getField("state"));
    assertEquals("oakstreet", seller3Record.getField("street"));
    assertEquals(9999, seller3Record.getField("zipCode"));
    assertTrue(pc3Record instanceof GenericJsonRecord);
    Assertions.assertThatCode(() -> pc3Record.getField("I_DO_NOT_EXIST")).doesNotThrowAnyException();
}

Example 2 with RecordSchemaBuilder

use of org.apache.pulsar.client.api.schema.RecordSchemaBuilder in project pulsar by apache.

the class ElasticSearchExtractTests method testKeyValueGenericRecord.

@Test(dataProvider = "schemaType")
public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("a", "1").set("b", 1).build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "[\"1\",1]");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertNull(pair2.getLeft());
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // test null value
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair3 = elasticSearchSink.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return new GenericObject() {

                @Override
                public SchemaType getSchemaType() {
                    return SchemaType.KEY_VALUE;
                }

                @Override
                public Object getNativeObject() {
                    return new KeyValue<>(keyGenericRecord, null);
                }
            };
        }
    });
    assertEquals(pair3.getLeft(), "[\"1\",1]");
    assertNull(pair3.getRight());
}

Example 3 with RecordSchemaBuilder

use of org.apache.pulsar.client.api.schema.RecordSchemaBuilder in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method raw_udt_value_and_cherry_pick_from_struct.

@Test
void raw_udt_value_and_cherry_pick_from_struct() {
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol=value, intcol=value.udtmem1"));
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("udtmem1").type(SchemaType.INT32);
    builder.field("udtmem2").type(SchemaType.STRING);
    Schema recordTypeUtd = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecordImpl value = new GenericRecordImpl().put("udtmem1", 42).put("udtmem2", "the answer");
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", "98761234", value, recordTypeUtd);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, udtcol, intcol FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(98761234L);
    UserDefinedType udt = new UserDefinedTypeBuilder(keyspaceName, "myudt").withField("udtmem1", DataTypes.INT).withField("udtmem2", DataTypes.TEXT).build();
    udt.attach(session.getContext());
    assertThat(row.getUdtValue("udtcol")).isEqualTo(udt.newValue(42, "the answer"));
    assertThat(row.getInt("intcol")).isEqualTo(42);
}

Example 4 with RecordSchemaBuilder

use of org.apache.pulsar.client.api.schema.RecordSchemaBuilder in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_missing.

@Test
void struct_optional_fields_missing() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, doublecol=value.double"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("int", baseValue.intValue());
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
}

Example 5 with RecordSchemaBuilder

use of org.apache.pulsar.client.api.schema.RecordSchemaBuilder in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_with_values.

@Test
void struct_optional_fields_with_values() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "booleancol=value.boolean, " + "doublecol=value.double, " + "floatcol=value.float, " + "intcol=value.int, " + "textcol=value.text, " + "blobcol=value.blob"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    byte[] blobValue = new byte[] { 12, 22, 32 };
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("boolean", (baseValue.intValue() & 1) == 1).put("double", (double) baseValue + 0.123).put("float", baseValue.floatValue() + 0.987f).put("int", baseValue.intValue()).put("smallint", baseValue.shortValue()).put("text", baseValue.toString()).put("blob", blobValue);
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseValue + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
    assertThat(row.getString("textcol")).isEqualTo(baseValue.toString());
    ByteBuffer blobcol = row.getByteBuffer("blobcol");
    assertThat(blobcol).isNotNull();
    assertThat(Bytes.getArray(blobcol)).isEqualTo(blobValue);
}