Search in sources :

Example 6 with HBaseDynamicTableSource

use of org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource in project flink-mirror by flink-ci.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}
Also used : HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) HBaseRowDataLookupFunction(org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) HBaseTableSchema(org.apache.flink.connector.hbase.util.HBaseTableSchema) TableFunction(org.apache.flink.table.functions.TableFunction) AsyncTableFunction(org.apache.flink.table.functions.AsyncTableFunction) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) Test(org.junit.Test)

Example 7 with HBaseDynamicTableSource

use of org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource in project flink-mirror by flink-ci.

the class HBaseDynamicTableFactoryTest method testLookupAsync.

@Test
public void testLookupAsync() {
    Map<String, String> options = getAllOptions();
    options.put("lookup.async", "true");
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()), Column.physical(FAMILY1, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, INT()))));
    DynamicTableSource source = createTableSource(schema, options);
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 0 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof AsyncTableFunctionProvider);
    AsyncTableFunction asyncTableFunction = ((AsyncTableFunctionProvider) lookupProvider).createAsyncTableFunction();
    assertTrue(asyncTableFunction instanceof HBaseRowDataAsyncLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataAsyncLookupFunction) asyncTableFunction).getHTableName());
}
Also used : HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) HBaseRowDataAsyncLookupFunction(org.apache.flink.connector.hbase2.source.HBaseRowDataAsyncLookupFunction) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) AsyncTableFunctionProvider(org.apache.flink.table.connector.source.AsyncTableFunctionProvider) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) AsyncTableFunction(org.apache.flink.table.functions.AsyncTableFunction) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) Test(org.junit.Test)

Example 8 with HBaseDynamicTableSource

use of org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource in project flink-mirror by flink-ci.

the class HBase2DynamicTableFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    TableFactoryHelper helper = createTableFactoryHelper(this, context);
    helper.validateExcept(PROPERTIES_PREFIX);
    final ReadableConfig tableOptions = helper.getOptions();
    validatePrimaryKey(context.getPhysicalRowDataType(), context.getPrimaryKeyIndexes());
    String tableName = tableOptions.get(TABLE_NAME);
    Configuration hbaseConf = getHBaseConfiguration(tableOptions);
    HBaseLookupOptions lookupOptions = getHBaseLookupOptions(tableOptions);
    String nullStringLiteral = tableOptions.get(NULL_STRING_LITERAL);
    HBaseTableSchema hbaseSchema = HBaseTableSchema.fromDataType(context.getPhysicalRowDataType());
    return new HBaseDynamicTableSource(hbaseConf, tableName, hbaseSchema, nullStringLiteral, lookupOptions);
}
Also used : HBaseLookupOptions(org.apache.flink.connector.hbase.options.HBaseLookupOptions) HBaseConnectorOptionsUtil.getHBaseLookupOptions(org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseLookupOptions) HBaseDynamicTableSource(org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource) ReadableConfig(org.apache.flink.configuration.ReadableConfig) Configuration(org.apache.hadoop.conf.Configuration) HBaseConnectorOptionsUtil.getHBaseConfiguration(org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseConfiguration) TableFactoryHelper(org.apache.flink.table.factories.FactoryUtil.TableFactoryHelper) FactoryUtil.createTableFactoryHelper(org.apache.flink.table.factories.FactoryUtil.createTableFactoryHelper) HBaseTableSchema(org.apache.flink.connector.hbase.util.HBaseTableSchema)

Example 9 with HBaseDynamicTableSource

use of org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource in project flink-mirror by flink-ci.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}
Also used : HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) HBaseRowDataLookupFunction(org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction) LookupRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext) LookupTableSource(org.apache.flink.table.connector.source.LookupTableSource) HBaseTableSchema(org.apache.flink.connector.hbase.util.HBaseTableSchema) TableFunction(org.apache.flink.table.functions.TableFunction) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) TableFunctionProvider(org.apache.flink.table.connector.source.TableFunctionProvider) Test(org.junit.Test)

Example 10 with HBaseDynamicTableSource

use of org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource in project flink-mirror by flink-ci.

the class HBaseDynamicTableFactoryTest method testLookupOptions.

@Test
public void testLookupOptions() {
    Map<String, String> options = getAllOptions();
    options.put("lookup.cache.max-rows", "1000");
    options.put("lookup.cache.ttl", "10s");
    options.put("lookup.max-retries", "10");
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()), Column.physical(FAMILY1, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, INT()))));
    DynamicTableSource source = createTableSource(schema, options);
    HBaseLookupOptions actual = ((HBaseDynamicTableSource) source).getLookupOptions();
    HBaseLookupOptions expected = HBaseLookupOptions.builder().setCacheMaxSize(1000).setCacheExpireMs(10_000).setMaxRetryTimes(10).build();
    assertEquals(expected, actual);
}
Also used : HBaseLookupOptions(org.apache.flink.connector.hbase.options.HBaseLookupOptions) HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) HBaseDynamicTableSource(org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource) Test(org.junit.Test)

Aggregations

ResolvedSchema (org.apache.flink.table.catalog.ResolvedSchema)10 DynamicTableSource (org.apache.flink.table.connector.source.DynamicTableSource)10 Test (org.junit.Test)10 HBaseTableSchema (org.apache.flink.connector.hbase.util.HBaseTableSchema)8 HBaseDynamicTableSource (org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource)8 HBaseLookupOptions (org.apache.flink.connector.hbase.options.HBaseLookupOptions)6 HBaseDynamicTableSource (org.apache.flink.connector.hbase1.source.HBaseDynamicTableSource)6 LookupTableSource (org.apache.flink.table.connector.source.LookupTableSource)6 LookupRuntimeProviderContext (org.apache.flink.table.runtime.connector.source.LookupRuntimeProviderContext)6 ReadableConfig (org.apache.flink.configuration.ReadableConfig)4 HBaseRowDataLookupFunction (org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction)4 HBaseConnectorOptionsUtil.getHBaseConfiguration (org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseConfiguration)4 AsyncTableFunctionProvider (org.apache.flink.table.connector.source.AsyncTableFunctionProvider)4 TableFunctionProvider (org.apache.flink.table.connector.source.TableFunctionProvider)4 TableFactoryHelper (org.apache.flink.table.factories.FactoryUtil.TableFactoryHelper)4 FactoryUtil.createTableFactoryHelper (org.apache.flink.table.factories.FactoryUtil.createTableFactoryHelper)4 AsyncTableFunction (org.apache.flink.table.functions.AsyncTableFunction)4 TableFunction (org.apache.flink.table.functions.TableFunction)4 Configuration (org.apache.hadoop.conf.Configuration)4 HBaseConnectorOptionsUtil.getHBaseLookupOptions (org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseLookupOptions)2