Examples with Table org.apache.beam.sdk.extensions.sql.meta.Table used on opensource projects

Example 61 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigtableTableCreationFailuresTest method testCreatePropertiesCountNotEqualSchemaFields.

@Test
public void testCreatePropertiesCountNotEqualSchemaFields() {
    String createTable = "CREATE EXTERNAL TABLE fail(key VARCHAR, q BIGINT, qq BINARY) \n" + "TYPE bigtable \n" + "LOCATION '" + location() + "' \n" + "TBLPROPERTIES '{\"columnsMapping\": \"f:q\"}'";
    cli.execute(createTable);
    Table table = metaStore.getTables().get("fail");
    IllegalStateException e = assertThrows(IllegalStateException.class, () -> tableProvider.buildBeamSqlTable(table));
    checkMessage(e.getMessage(), "Schema fields count: '2' does not fit columnsMapping count: '1'");
}

Example 62 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigtableTableWithRowsTest method testCreatesSchemaCorrectly.

@Test
public void testCreatesSchemaCorrectly() {
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    metaStore.registerProvider(new BigtableTableProvider());
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute(createFullTableString(TABLE, location()));
    Table table = metaStore.getTables().get("beamTable");
    assertNotNull(table);
    assertEquals(TEST_SCHEMA, table.getSchema());
}

Example 63 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BeamDDLNestedTypesTest method supportsPrimitiveTypes.

@Property
public void supportsPrimitiveTypes(@From(PrimitiveTypes.class) FieldType fieldType) throws SqlParseException {
    String fieldTypeDeclaration = unparse(fieldType);
    Table table = executeCreateTableWith(fieldTypeDeclaration);
    Schema expectedSchema = newSimpleSchemaWith(fieldType);
    assertEquals(expectedSchema, table.getSchema());
}

Example 64 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BeamDDLNestedTypesTest method supportsNestedTypes.

@Property
public void supportsNestedTypes(@From(AnyFieldType.class) FieldType generatedFieldType) throws SqlParseException {
    String fieldTypeDeclaration = unparse(generatedFieldType);
    Table table = executeCreateTableWith(fieldTypeDeclaration);
    Schema expectedSchema = newSimpleSchemaWith(generatedFieldType);
    assertEquals(expectedSchema, table.getSchema());
}

Example 65 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class KafkaTableProvider method buildBeamSqlTable.

@Override
public BeamSqlTable buildBeamSqlTable(Table table) {
    Schema schema = table.getSchema();
    JSONObject properties = table.getProperties();
    Optional<ParsedLocation> parsedLocation = Optional.empty();
    if (!Strings.isNullOrEmpty(table.getLocation())) {
        parsedLocation = Optional.of(parseLocation(checkArgumentNotNull(table.getLocation())));
    }
    List<String> topics = mergeParam(parsedLocation.map(loc -> loc.topic), properties.getJSONArray("topics"));
    List<String> allBootstrapServers = mergeParam(parsedLocation.map(loc -> loc.brokerLocation), properties.getJSONArray("bootstrap_servers"));
    String bootstrapServers = String.join(",", allBootstrapServers);
    Optional<String> payloadFormat = properties.containsKey("format") ? Optional.of(properties.getString("format")) : Optional.empty();
    if (Schemas.isNestedSchema(schema)) {
        Optional<PayloadSerializer> serializer = payloadFormat.map(format -> PayloadSerializers.getSerializer(format, checkArgumentNotNull(schema.getField(PAYLOAD_FIELD).getType().getRowSchema()), properties.getInnerMap()));
        return new NestedPayloadKafkaTable(schema, bootstrapServers, topics, serializer);
    } else {
        /*
       * CSV is handled separately because multiple rows can be produced from a single message, which
       * adds complexity to payload extraction. It remains here and as the default because it is the
       * historical default, but it will not be extended to support attaching extended attributes to
       * rows.
       */
        if (payloadFormat.orElse("csv").equals("csv")) {
            return new BeamKafkaCSVTable(schema, bootstrapServers, topics);
        }
        PayloadSerializer serializer = PayloadSerializers.getSerializer(payloadFormat.get(), schema, properties.getInnerMap());
        return new PayloadSerializerKafkaTable(schema, bootstrapServers, topics, serializer);
    }
}