Examples with Schema org.apache.flink.table.api.Schema used on opensource projects

Example 6 with Schema

use of org.apache.flink.table.api.Schema in project flink by apache.

the class SchemaTranslator method createConsumingResult.

/**
 * Converts the given {@link DataType} and an optional declared {@link Schema} (possibly
 * incomplete) into the final {@link ConsumingResult}.
 *
 * <p>This method serves three types of use cases:
 *
 * <ul>
 *   <li>1. Derive physical columns from the input data type.
 *   <li>2. Derive physical columns but merge them with declared computed columns and other
 *       schema information.
 *   <li>3. Derive and enrich physical columns and merge other schema information (only if
 *       {@param mergePhysicalSchema} is set to {@code true}).
 * </ul>
 */
public static ConsumingResult createConsumingResult(DataTypeFactory dataTypeFactory, DataType inputDataType, @Nullable Schema declaredSchema, boolean mergePhysicalSchema) {
    final LogicalType inputType = inputDataType.getLogicalType();
    // we don't allow modifying the number of columns during enrichment, therefore we preserve
    // whether the original type was qualified as a top-level record or not
    final boolean isTopLevelRecord = LogicalTypeChecks.isCompositeType(inputType);
    // the schema will be entirely derived from the input
    if (declaredSchema == null) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, null);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    final List<UnresolvedColumn> declaredColumns = declaredSchema.getColumns();
    final UnresolvedPrimaryKey declaredPrimaryKey = declaredSchema.getPrimaryKey().orElse(null);
    // thus, it only enriches the non-physical column parts
    if (declaredColumns.stream().noneMatch(SchemaTranslator::isPhysical)) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, declaredPrimaryKey);
        builder.fromSchema(declaredSchema);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    if (!mergePhysicalSchema) {
        return new ConsumingResult(inputDataType, isTopLevelRecord, declaredSchema, null);
    }
    // the declared schema enriches the physical data type and the derived schema,
    // it possibly projects the result
    final DataType patchedDataType = patchDataTypeFromDeclaredSchema(dataTypeFactory, inputDataType, declaredColumns);
    final Schema patchedSchema = createPatchedSchema(isTopLevelRecord, patchedDataType, declaredSchema);
    final List<String> projections = extractProjections(patchedSchema, declaredSchema);
    return new ConsumingResult(patchedDataType, isTopLevelRecord, patchedSchema, projections);
}

Example 7 with Schema

use of org.apache.flink.table.api.Schema in project flink by apache.

the class DataStreamJavaITCase method testFromAndToChangelogStreamEventTime.

@Test
public void testFromAndToChangelogStreamEventTime() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final DataStream<Tuple3<Long, Integer, String>> dataStream = getWatermarkedDataStream();
    final DataStream<Row> changelogStream = dataStream.map(t -> Row.ofKind(RowKind.INSERT, t.f1, t.f2)).returns(Types.ROW(Types.INT, Types.STRING));
    // derive physical columns and add a rowtime
    final Table table = tableEnv.fromChangelogStream(changelogStream, Schema.newBuilder().columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("computed", $("f1").upperCase()).watermark("rowtime", sourceWatermark()).build());
    tableEnv.createTemporaryView("t", table);
    // access and reorder columns
    final Table reordered = tableEnv.sqlQuery("SELECT computed, rowtime, f0 FROM t");
    // write out the rowtime column with fully declared schema
    final DataStream<Row> result = tableEnv.toChangelogStream(reordered, Schema.newBuilder().column("f1", STRING()).columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("ignored", $("f1").upperCase()).column("f0", INT()).build());
    // test event time window and field access
    testResult(result.keyBy(k -> k.getField("f1")).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).<Row>apply((key, window, input, out) -> {
        int sum = 0;
        for (Row row : input) {
            sum += row.<Integer>getFieldAs("f0");
        }
        out.collect(Row.of(key, sum));
    }).returns(Types.ROW(Types.STRING, Types.INT)), Row.of("A", 47), Row.of("C", 1000), Row.of("C", 1000));
}

Example 8 with Schema

use of org.apache.flink.table.api.Schema in project flink by apache.

the class AbstractJdbcCatalog method getTable.

// ------ tables and views ------
@Override
public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException {
    if (!tableExists(tablePath)) {
        throw new TableNotExistException(getName(), tablePath);
    }
    String dbUrl = baseUrl + tablePath.getDatabaseName();
    try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd)) {
        DatabaseMetaData metaData = conn.getMetaData();
        Optional<UniqueConstraint> primaryKey = getPrimaryKey(metaData, getSchemaName(tablePath), getTableName(tablePath));
        PreparedStatement ps = conn.prepareStatement(String.format("SELECT * FROM %s;", getSchemaTableName(tablePath)));
        ResultSetMetaData resultSetMetaData = ps.getMetaData();
        String[] columnNames = new String[resultSetMetaData.getColumnCount()];
        DataType[] types = new DataType[resultSetMetaData.getColumnCount()];
        for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
            columnNames[i - 1] = resultSetMetaData.getColumnName(i);
            types[i - 1] = fromJDBCType(tablePath, resultSetMetaData, i);
            if (resultSetMetaData.isNullable(i) == ResultSetMetaData.columnNoNulls) {
                types[i - 1] = types[i - 1].notNull();
            }
        }
        Schema.Builder schemaBuilder = Schema.newBuilder().fromFields(columnNames, types);
        primaryKey.ifPresent(pk -> schemaBuilder.primaryKeyNamed(pk.getName(), pk.getColumns()));
        Schema tableSchema = schemaBuilder.build();
        Map<String, String> props = new HashMap<>();
        props.put(CONNECTOR.key(), IDENTIFIER);
        props.put(URL.key(), dbUrl);
        props.put(USERNAME.key(), username);
        props.put(PASSWORD.key(), pwd);
        props.put(TABLE_NAME.key(), getSchemaTableName(tablePath));
        return CatalogTable.of(tableSchema, null, Lists.newArrayList(), props);
    } catch (Exception e) {
        throw new CatalogException(String.format("Failed getting table %s", tablePath.getFullName()), e);
    }
}

Example 9 with Schema

use of org.apache.flink.table.api.Schema in project flink by apache.

the class HiveCatalogITCase method testViewSchema.

@Test
public void testViewSchema() throws Exception {
    TableEnvironment tableEnv = HiveTestUtils.createTableEnvInBatchMode(SqlDialect.DEFAULT);
    tableEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog);
    tableEnv.useCatalog(hiveCatalog.getName());
    tableEnv.executeSql("create database db1");
    try {
        tableEnv.useDatabase("db1");
        tableEnv.executeSql("create table src(x int,ts timestamp(3)) with ('connector'='datagen','number-of-rows'='10')");
        tableEnv.executeSql("create view v1 as select x,ts from src order by x limit 3");
        CatalogView catalogView = (CatalogView) hiveCatalog.getTable(new ObjectPath("db1", "v1"));
        Schema viewSchema = catalogView.getUnresolvedSchema();
        assertThat(viewSchema).isEqualTo(Schema.newBuilder().fromFields(new String[] { "x", "ts" }, new AbstractDataType[] { DataTypes.INT(), DataTypes.TIMESTAMP(3) }).build());
        List<Row> results = CollectionUtil.iteratorToList(tableEnv.executeSql("select x from v1").collect());
        assertThat(results).hasSize(3);
        tableEnv.executeSql("create view v2 (v2_x,v2_ts) comment 'v2 comment' as select x,cast(ts as timestamp_ltz(3)) from v1");
        catalogView = (CatalogView) hiveCatalog.getTable(new ObjectPath("db1", "v2"));
        assertThat(catalogView.getUnresolvedSchema()).isEqualTo(Schema.newBuilder().fromFields(new String[] { "v2_x", "v2_ts" }, new AbstractDataType[] { DataTypes.INT(), DataTypes.TIMESTAMP_LTZ(3) }).build());
        assertThat(catalogView.getComment()).isEqualTo("v2 comment");
        results = CollectionUtil.iteratorToList(tableEnv.executeSql("select * from v2").collect());
        assertThat(results).hasSize(3);
    } finally {
        tableEnv.executeSql("drop database db1 cascade");
    }
}

Example 10 with Schema

use of org.apache.flink.table.api.Schema in project flink by apache.

the class SchemaResolutionTest method testGeneratedConstraintName.

@Test
public void testGeneratedConstraintName() {
    final Schema schema = Schema.newBuilder().column("a", DataTypes.INT()).column("b", DataTypes.STRING()).column("c", DataTypes.STRING()).primaryKey("b", "a").build();
    assertThat(schema.getPrimaryKey().orElseThrow(IllegalStateException::new).getConstraintName(), equalTo("PK_b_a"));
}