Search in sources :

Example 1 with TableFunctionProvider

use of org.apache.flink.table.connector.source.TableFunctionProvider in project flink by apache.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}
Also used : HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) HBaseRowDataLookupFunction(org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) HBaseTableSchema(org.apache.flink.connector.hbase.util.HBaseTableSchema) TableFunction(org.apache.flink.table.functions.TableFunction) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) Test(org.junit.Test)

Example 2 with TableFunctionProvider

use of org.apache.flink.table.connector.source.TableFunctionProvider in project flink by apache.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}
Also used : HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) HBaseRowDataLookupFunction(org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) HBaseTableSchema(org.apache.flink.connector.hbase.util.HBaseTableSchema) TableFunction(org.apache.flink.table.functions.TableFunction) AsyncTableFunction(org.apache.flink.table.functions.AsyncTableFunction) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) Test(org.junit.Test)

Example 3 with TableFunctionProvider

use of org.apache.flink.table.connector.source.TableFunctionProvider in project flink by apache.

the class LookupJoinUtil method getLookupFunction.

/**
 * Gets LookupFunction from temporal table according to the given lookup keys.
 */
public static UserDefinedFunction getLookupFunction(RelOptTable temporalTable, Collection<Integer> lookupKeys) {
    int[] lookupKeyIndicesInOrder = getOrderedLookupKeys(lookupKeys);
    if (temporalTable instanceof TableSourceTable) {
        // TODO: support nested lookup keys in the future,
        // currently we only support top-level lookup keys
        int[][] indices = IntStream.of(lookupKeyIndicesInOrder).mapToObj(i -> new int[] { i }).toArray(int[][]::new);
        LookupTableSource tableSource = (LookupTableSource) ((TableSourceTable) temporalTable).tableSource();
        LookupRuntimeProviderContext providerContext = new LookupRuntimeProviderContext(indices);
        LookupTableSource.LookupRuntimeProvider provider = tableSource.getLookupRuntimeProvider(providerContext);
        if (provider instanceof TableFunctionProvider) {
            return ((TableFunctionProvider<?>) provider).createTableFunction();
        } else if (provider instanceof AsyncTableFunctionProvider) {
            return ((AsyncTableFunctionProvider<?>) provider).createAsyncTableFunction();
        }
    }
    if (temporalTable instanceof LegacyTableSourceTable) {
        String[] lookupFieldNamesInOrder = IntStream.of(lookupKeyIndicesInOrder).mapToObj(temporalTable.getRowType().getFieldNames()::get).toArray(String[]::new);
        LegacyTableSourceTable<?> legacyTableSourceTable = (LegacyTableSourceTable<?>) temporalTable;
        LookupableTableSource<?> tableSource = (LookupableTableSource<?>) legacyTableSourceTable.tableSource();
        if (tableSource.isAsyncEnabled()) {
            return tableSource.getAsyncLookupFunction(lookupFieldNamesInOrder);
        } else {
            return tableSource.getLookupFunction(lookupFieldNamesInOrder);
        }
    }
    throw new TableException(String.format("table %s is neither TableSourceTable not LegacyTableSourceTable", temporalTable.getQualifiedName()));
}
Also used : IntStream(java.util.stream.IntStream) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) JsonCreator(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator) LegacyTableSourceTable(org.apache.flink.table.planner.plan.schema.LegacyTableSourceTable) JsonSubTypes(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes) RexLiteral(org.apache.calcite.rex.RexLiteral) Collection(java.util.Collection) TableException(org.apache.flink.table.api.TableException) UserDefinedFunction(org.apache.flink.table.functions.UserDefinedFunction) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable) LookupableTableSource(org.apache.flink.table.sources.LookupableTableSource) JsonProperty(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty) RelOptTable(org.apache.calcite.plan.RelOptTable) ArrayList(java.util.ArrayList) Objects(java.util.Objects) List(java.util.List) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) LogicalType(org.apache.flink.table.types.logical.LogicalType) JsonTypeInfo(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo) Internal(org.apache.flink.annotation.Internal) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) JsonIgnoreProperties(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) JsonTypeName(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName) TableException(org.apache.flink.table.api.TableException) LegacyTableSourceTable(org.apache.flink.table.planner.plan.schema.LegacyTableSourceTable) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) LookupableTableSource(org.apache.flink.table.sources.LookupableTableSource) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) LegacyTableSourceTable(org.apache.flink.table.planner.plan.schema.LegacyTableSourceTable) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable)

Aggregations

LookupTableSource (org.apache.flink.table.connector.source.LookupTableSource)3 TableFunctionProvider (org.apache.flink.table.connector.source.TableFunctionProvider)3 LookupRuntimeProviderContext (org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext)3 HBaseRowDataLookupFunction (org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction)2 HBaseTableSchema (org.apache.flink.connector.hbase.util.HBaseTableSchema)2 ResolvedSchema (org.apache.flink.table.catalog.ResolvedSchema)2 AsyncTableFunctionProvider (org.apache.flink.table.connector.source.AsyncTableFunctionProvider)2 DynamicTableSource (org.apache.flink.table.connector.source.DynamicTableSource)2 TableFunction (org.apache.flink.table.functions.TableFunction)2 Test (org.junit.Test)2 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 List (java.util.List)1 Objects (java.util.Objects)1 IntStream (java.util.stream.IntStream)1 RelOptTable (org.apache.calcite.plan.RelOptTable)1 RexLiteral (org.apache.calcite.rex.RexLiteral)1 Internal (org.apache.flink.annotation.Internal)1 HBaseDynamicTableSource (org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource)1 HBaseDynamicTableSource (org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource)1