Examples with StreamTableSource org.apache.flink.table.sources.StreamTableSource used on opensource projects

Example 1 with StreamTableSource

use of org.apache.flink.table.sources.StreamTableSource in project flink by apache.

the class HiveTableFactoryTest method testGenericTable.

@Test
public void testGenericTable() throws Exception {
    final TableSchema schema = TableSchema.builder().field("name", DataTypes.STRING()).field("age", DataTypes.INT()).build();
    catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
    final Map<String, String> options = Collections.singletonMap(FactoryUtil.CONNECTOR.key(), "COLLECTION");
    final CatalogTable table = new CatalogTableImpl(schema, options, "csv table");
    catalog.createTable(new ObjectPath("mydb", "mytable"), table, true);
    final Optional<TableFactory> tableFactoryOpt = catalog.getTableFactory();
    assertTrue(tableFactoryOpt.isPresent());
    final HiveTableFactory tableFactory = (HiveTableFactory) tableFactoryOpt.get();
    final TableSource tableSource = tableFactory.createTableSource(new TableSourceFactoryContextImpl(ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration(), false));
    assertTrue(tableSource instanceof StreamTableSource);
    final TableSink tableSink = tableFactory.createTableSink(new TableSinkFactoryContextImpl(ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration(), true, false));
    assertTrue(tableSink instanceof StreamTableSink);
}

Example 2 with StreamTableSource

use of org.apache.flink.table.sources.StreamTableSource in project flink by apache.

the class SimpleCatalogFactory method createCatalog.

@Override
public Catalog createCatalog(Context context) {
    final Configuration configuration = Configuration.fromMap(context.getOptions());
    final String database = configuration.getString(DEFAULT_DATABASE);
    final String tableName = configuration.getString(TABLE_NAME);
    final GenericInMemoryCatalog genericInMemoryCatalog = new GenericInMemoryCatalog(context.getName(), database);
    StreamTableSource<Row> tableSource = new StreamTableSource<Row>() {

        @Override
        public DataStream<Row> getDataStream(StreamExecutionEnvironment execEnv) {
            return execEnv.fromCollection(TABLE_CONTENTS).returns(new RowTypeInfo(new TypeInformation[] { Types.INT(), Types.STRING() }, new String[] { "id", "string" }));
        }

        @Override
        public TableSchema getTableSchema() {
            return TableSchema.builder().field("id", DataTypes.INT()).field("string", DataTypes.STRING()).build();
        }

        @Override
        public DataType getProducedDataType() {
            return DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT()), DataTypes.FIELD("string", DataTypes.STRING())).notNull();
        }
    };
    try {
        genericInMemoryCatalog.createTable(new ObjectPath(database, tableName), ConnectorCatalogTable.source(tableSource, false), false);
    } catch (Exception e) {
        throw new WrappingRuntimeException(e);
    }
    return genericInMemoryCatalog;
}

Example 3 with StreamTableSource

use of org.apache.flink.table.sources.StreamTableSource in project flink by apache.

the class CommonExecLegacyTableSourceScan method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final Transformation<?> sourceTransform;
    final StreamExecutionEnvironment env = planner.getExecEnv();
    if (tableSource instanceof InputFormatTableSource) {
        InputFormatTableSource<Object> inputFormat = (InputFormatTableSource<Object>) tableSource;
        TypeInformation<Object> typeInfo = (TypeInformation<Object>) fromDataTypeToTypeInfo(inputFormat.getProducedDataType());
        InputFormat<Object, ?> format = inputFormat.getInputFormat();
        sourceTransform = createInput(env, format, typeInfo);
    } else if (tableSource instanceof StreamTableSource) {
        sourceTransform = ((StreamTableSource<?>) tableSource).getDataStream(env).getTransformation();
    } else {
        throw new UnsupportedOperationException(tableSource.getClass().getSimpleName() + " is unsupported.");
    }
    final TypeInformation<?> inputType = sourceTransform.getOutputType();
    final DataType producedDataType = tableSource.getProducedDataType();
    // check that declared and actual type of table source DataStream are identical
    if (!inputType.equals(TypeInfoDataTypeConverter.fromDataTypeToTypeInfo(producedDataType))) {
        throw new TableException(String.format("TableSource of type %s " + "returned a DataStream of data type %s that does not match with the " + "data type %s declared by the TableSource.getProducedDataType() method. " + "Please validate the implementation of the TableSource.", tableSource.getClass().getCanonicalName(), inputType, producedDataType));
    }
    final RowType outputType = (RowType) getOutputType();
    final RelDataType relDataType = FlinkTypeFactory.INSTANCE().buildRelNodeRowType(outputType);
    // get expression to extract rowtime attribute
    final Optional<RexNode> rowtimeExpression = JavaScalaConversionUtil.toJava(TableSourceUtil.getRowtimeAttributeDescriptor(tableSource, relDataType)).map(desc -> TableSourceUtil.getRowtimeExtractionExpression(desc.getTimestampExtractor(), producedDataType, planner.getRelBuilder(), getNameRemapping()));
    return createConversionTransformationIfNeeded(planner.getExecEnv(), config, sourceTransform, rowtimeExpression.orElse(null));
}

Example 4 with StreamTableSource

use of org.apache.flink.table.sources.StreamTableSource in project flink by apache.

the class CatalogSchemaTable method findAndCreateTableSource.

private Optional<TableSource<?>> findAndCreateTableSource() {
    Optional<TableSource<?>> tableSource = Optional.empty();
    try {
        if (contextResolvedTable.getTable() instanceof CatalogTable) {
            // Use an empty config for TableSourceFactoryContextImpl since we can't fetch the
            // actual TableConfig here. And currently the empty config do not affect the logic.
            ReadableConfig config = new Configuration();
            TableSourceFactory.Context context = new TableSourceFactoryContextImpl(contextResolvedTable.getIdentifier(), contextResolvedTable.getTable(), config, contextResolvedTable.isTemporary());
            TableSource<?> source = TableFactoryUtil.findAndCreateTableSource(context);
            if (source instanceof StreamTableSource) {
                if (!isStreamingMode && !((StreamTableSource<?>) source).isBounded()) {
                    throw new ValidationException("Cannot query on an unbounded source in batch mode, but " + contextResolvedTable.getIdentifier().asSummaryString() + " is unbounded.");
                }
                tableSource = Optional.of(source);
            } else {
                throw new ValidationException("Catalog tables only support " + "StreamTableSource and InputFormatTableSource.");
            }
        }
    } catch (Exception e) {
        tableSource = Optional.empty();
    }
    return tableSource;
}