Search in sources :

Example 1 with PayloadSerializer

use of org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer in project beam by apache.

the class KafkaTableProvider method buildBeamSqlTable.

@Override
public BeamSqlTable buildBeamSqlTable(Table table) {
    Schema schema = table.getSchema();
    JSONObject properties = table.getProperties();
    Optional<ParsedLocation> parsedLocation = Optional.empty();
    if (!Strings.isNullOrEmpty(table.getLocation())) {
        parsedLocation = Optional.of(parseLocation(checkArgumentNotNull(table.getLocation())));
    }
    List<String> topics = mergeParam(parsedLocation.map(loc -> loc.topic), properties.getJSONArray("topics"));
    List<String> allBootstrapServers = mergeParam(parsedLocation.map(loc -> loc.brokerLocation), properties.getJSONArray("bootstrap_servers"));
    String bootstrapServers = String.join(",", allBootstrapServers);
    Optional<String> payloadFormat = properties.containsKey("format") ? Optional.of(properties.getString("format")) : Optional.empty();
    if (Schemas.isNestedSchema(schema)) {
        Optional<PayloadSerializer> serializer = payloadFormat.map(format -> PayloadSerializers.getSerializer(format, checkArgumentNotNull(schema.getField(PAYLOAD_FIELD).getType().getRowSchema()), properties.getInnerMap()));
        return new NestedPayloadKafkaTable(schema, bootstrapServers, topics, serializer);
    } else {
        /*
       * CSV is handled separately because multiple rows can be produced from a single message, which
       * adds complexity to payload extraction. It remains here and as the default because it is the
       * historical default, but it will not be extended to support attaching extended attributes to
       * rows.
       */
        if (payloadFormat.orElse("csv").equals("csv")) {
            return new BeamKafkaCSVTable(schema, bootstrapServers, topics);
        }
        PayloadSerializer serializer = PayloadSerializers.getSerializer(payloadFormat.get(), schema, properties.getInnerMap());
        return new PayloadSerializerKafkaTable(schema, bootstrapServers, topics, serializer);
    }
}
Also used : InMemoryMetaTableProvider(org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider) Splitter(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter) Schema(org.apache.beam.sdk.schemas.Schema) PayloadSerializer(org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer) PAYLOAD_FIELD(org.apache.beam.sdk.extensions.sql.meta.provider.kafka.Schemas.PAYLOAD_FIELD) Preconditions.checkArgumentNotNull(org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull) Table(org.apache.beam.sdk.extensions.sql.meta.Table) List(java.util.List) PayloadSerializers(org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers) BeamSqlTable(org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable) Strings(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings) AutoService(com.google.auto.service.AutoService) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) Preconditions.checkArgument(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument) Optional(java.util.Optional) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) JSONObject(com.alibaba.fastjson.JSONObject) TableProvider(org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider) Nullable(org.checkerframework.checker.nullness.qual.Nullable) PayloadSerializer(org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer) Schema(org.apache.beam.sdk.schemas.Schema) JSONObject(com.alibaba.fastjson.JSONObject)

Aggregations

JSONObject (com.alibaba.fastjson.JSONObject)1 AutoService (com.google.auto.service.AutoService)1 List (java.util.List)1 Optional (java.util.Optional)1 BeamSqlTable (org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable)1 Table (org.apache.beam.sdk.extensions.sql.meta.Table)1 InMemoryMetaTableProvider (org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider)1 TableProvider (org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider)1 PAYLOAD_FIELD (org.apache.beam.sdk.extensions.sql.meta.provider.kafka.Schemas.PAYLOAD_FIELD)1 Schema (org.apache.beam.sdk.schemas.Schema)1 PayloadSerializer (org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer)1 PayloadSerializers (org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers)1 Preconditions.checkArgumentNotNull (org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull)1 Preconditions.checkArgument (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument)1 Splitter (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter)1 Strings (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings)1 ImmutableList (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList)1 Iterables (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables)1 Nullable (org.checkerframework.checker.nullness.qual.Nullable)1