Examples with GenericSchema org.apache.pulsar.client.api.schema.GenericSchema used on opensource projects

Example 1 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by apache.

the class FieldSchemaBuilderImpl method build.

Field build() {
    requireNonNull(type, "Schema type is not provided");
    // verify the default value and object
    SchemaUtils.validateFieldSchema(fieldName, type, defaultVal);
    final Schema baseSchema;
    switch(type) {
        case INT32:
            baseSchema = SchemaBuilder.builder().intType();
            break;
        case INT64:
            baseSchema = SchemaBuilder.builder().longType();
            break;
        case STRING:
            baseSchema = SchemaBuilder.builder().stringType();
            break;
        case FLOAT:
            baseSchema = SchemaBuilder.builder().floatType();
            break;
        case DOUBLE:
            baseSchema = SchemaBuilder.builder().doubleType();
            break;
        case BOOLEAN:
            baseSchema = SchemaBuilder.builder().booleanType();
            break;
        case BYTES:
            baseSchema = SchemaBuilder.builder().bytesType();
            break;
        // DATE, TIME, TIMESTAMP support from generic record
        case DATE:
            baseSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIME:
            baseSchema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIMESTAMP:
            baseSchema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
            break;
        case JSON:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.JSON, "The field is expected to be using JSON schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            AvroBaseStructSchema genericJsonSchema = (AvroBaseStructSchema) genericSchema;
            baseSchema = genericJsonSchema.getAvroSchema();
            break;
        case AVRO:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.AVRO, "The field is expected to be using AVRO schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            GenericAvroSchema genericAvroSchema = (GenericAvroSchema) genericSchema;
            baseSchema = genericAvroSchema.getAvroSchema();
            break;
        default:
            throw new RuntimeException("Schema `" + type + "` is not supported to be used as a field for now");
    }
    for (Map.Entry<String, String> entry : properties.entrySet()) {
        baseSchema.addProp(entry.getKey(), entry.getValue());
    }
    if (null != aliases) {
        for (String alias : aliases) {
            baseSchema.addAlias(alias);
        }
    }
    final Schema finalSchema;
    if (optional) {
        if (defaultVal != null) {
            finalSchema = SchemaBuilder.builder().unionOf().type(baseSchema).and().nullType().endUnion();
        } else {
            finalSchema = SchemaBuilder.builder().unionOf().nullType().and().type(baseSchema).endUnion();
        }
    } else {
        finalSchema = baseSchema;
    }
    final Object finalDefaultValue;
    if (defaultVal != null) {
        finalDefaultValue = SchemaUtils.toAvroObject(defaultVal);
    } else {
        if (optional) {
            finalDefaultValue = JsonProperties.NULL_VALUE;
        } else {
            finalDefaultValue = null;
        }
    }
    return new Field(fieldName, finalSchema, doc, finalDefaultValue);
}

Example 2 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by apache.

the class GenericJsonRecordTest method testEncodeAndDecodeObject.

@Test
public void testEncodeAndDecodeObject() throws JsonProcessingException {
    // test case from issue https://github.com/apache/pulsar/issues/9605
    JSONSchema<PC> jsonSchema = JSONSchema.of(SchemaDefinition.<PC>builder().withPojo(PC.class).build());
    GenericSchema genericJsonSchema = GenericJsonSchema.of(jsonSchema.getSchemaInfo());
    PC pc = new PC("dell", "alienware", 2021, GPU.AMD, new Seller("WA", "street", 98004));
    JsonNode jsonNode = ObjectMapperFactory.getThreadLocal().valueToTree(pc);
    GenericJsonRecord genericJsonRecord = new GenericJsonRecord(null, null, jsonNode, genericJsonSchema.getSchemaInfo());
    byte[] encoded = genericJsonSchema.encode(genericJsonRecord);
    PC roundtrippedPc = jsonSchema.decode(encoded);
    assertEquals(roundtrippedPc, pc);
}

Example 3 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by apache.

the class GenericSchemaImplTest method testKeyValueSchema.

@Test
public void testKeyValueSchema() {
    // configure the schema info provider
    MultiVersionSchemaInfoProvider multiVersionSchemaInfoProvider = mock(MultiVersionSchemaInfoProvider.class);
    List<Schema<Foo>> encodeSchemas = Lists.newArrayList(Schema.JSON(Foo.class), Schema.AVRO(Foo.class));
    for (Schema<Foo> keySchema : encodeSchemas) {
        for (Schema<Foo> valueSchema : encodeSchemas) {
            // configure encode schema
            Schema<KeyValue<Foo, Foo>> kvSchema = KeyValueSchemaImpl.of(keySchema, valueSchema);
            // configure decode schema
            Schema<KeyValue<GenericRecord, GenericRecord>> decodeSchema = KeyValueSchemaImpl.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME());
            decodeSchema.configureSchemaInfo("test-topic", "topic", kvSchema.getSchemaInfo());
            GenericSchema genericAvroSchema = GenericSchemaImpl.of(Schema.AVRO(Foo.class).getSchemaInfo());
            when(multiVersionSchemaInfoProvider.getSchemaByVersion(any(byte[].class))).thenReturn(CompletableFuture.completedFuture(KeyValueSchemaInfo.encodeKeyValueSchemaInfo(keySchema, valueSchema, KeyValueEncodingType.INLINE)));
            decodeSchema.setSchemaInfoProvider(multiVersionSchemaInfoProvider);
            testEncodeAndDecodeKeyValues(kvSchema, decodeSchema);
        }
    }
}

Example 4 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by apache.

the class AbstractGenericSchemaTest method testGenericProtobufNativeSchema.

@Test
public void testGenericProtobufNativeSchema() {
    Schema<org.apache.pulsar.client.schema.proto.Test.TestMessage> encodeSchema = Schema.PROTOBUF_NATIVE(org.apache.pulsar.client.schema.proto.Test.TestMessage.class);
    GenericSchema decodeSchema = GenericProtobufNativeSchema.of(encodeSchema.getSchemaInfo());
    testEncodeAndDecodeGenericRecord(encodeSchema, decodeSchema);
}

Example 5 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by apache.

the class ElasticSearchExtractTests method testKeyValueGenericRecord.

@Test(dataProvider = "schemaType")
public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("a", "1").set("b", 1).build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "[\"1\",1]");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertNull(pair2.getLeft());
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // test null value
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair3 = elasticSearchSink.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return new GenericObject() {

                @Override
                public SchemaType getSchemaType() {
                    return SchemaType.KEY_VALUE;
                }

                @Override
                public Object getNativeObject() {
                    return new KeyValue<>(keyGenericRecord, null);
                }
            };
        }
    });
    assertEquals(pair3.getLeft(), "[\"1\",1]");
    assertNull(pair3.getRight());
}