Examples with HBaseTableSchema org.apache.flink.connector.hbase.util.HBaseTableSchema used on opensource projects

Example 6 with HBaseTableSchema

use of org.apache.flink.connector.hbase.util.HBaseTableSchema in project flink by apache.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}

Example 7 with HBaseTableSchema

use of org.apache.flink.connector.hbase.util.HBaseTableSchema in project flink by apache.

the class HBaseDynamicTableFactoryTest method testTableSinkFactory.

@Test
public void testTableSinkFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()), Column.physical(FAMILY1, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL3, BIGINT()))), Column.physical(FAMILY3, ROW(FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSink sink = createTableSink(schema, getAllOptions());
    assertTrue(sink instanceof HBaseDynamicTableSink);
    HBaseDynamicTableSink hbaseSink = (HBaseDynamicTableSink) sink;
    HBaseTableSchema hbaseSchema = hbaseSink.getHBaseTableSchema();
    assertEquals(0, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(STRING()), hbaseSchema.getRowKeyDataType());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c3" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { DOUBLE(), INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
    // verify hadoop Configuration
    org.apache.hadoop.conf.Configuration expectedConfiguration = HBaseConfigurationUtil.getHBaseConfiguration();
    expectedConfiguration.set(HConstants.ZOOKEEPER_QUORUM, "localhost:2181");
    expectedConfiguration.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/flink");
    expectedConfiguration.set("hbase.security.authentication", "kerberos");
    org.apache.hadoop.conf.Configuration actualConfiguration = hbaseSink.getConfiguration();
    assertEquals(IteratorUtils.toList(expectedConfiguration.iterator()), IteratorUtils.toList(actualConfiguration.iterator()));
    // verify tableName
    assertEquals("testHBastTable", hbaseSink.getTableName());
    HBaseWriteOptions expectedWriteOptions = HBaseWriteOptions.builder().setBufferFlushMaxRows(1000).setBufferFlushIntervalMillis(1000).setBufferFlushMaxSizeInBytes(2 * 1024 * 1024).build();
    HBaseWriteOptions actualWriteOptions = hbaseSink.getWriteOptions();
    assertEquals(expectedWriteOptions, actualWriteOptions);
}

Example 8 with HBaseTableSchema

use of org.apache.flink.connector.hbase.util.HBaseTableSchema in project flink by apache.

the class HBaseRowDataAsyncLookupFunctionTest method buildRowDataAsyncLookupFunction.

private HBaseRowDataAsyncLookupFunction buildRowDataAsyncLookupFunction() {
    HBaseLookupOptions lookupOptions = HBaseLookupOptions.builder().build();
    if (useCache) {
        lookupOptions = HBaseLookupOptions.builder().setCacheMaxSize(4).setCacheExpireMs(10000).build();
    }
    DataType dataType = ROW(FIELD(ROW_KEY, INT()), FIELD(FAMILY1, ROW(FIELD(F1COL1, INT()))), FIELD(FAMILY2, ROW(FIELD(F2COL1, STRING()), FIELD(F2COL2, BIGINT()))), FIELD(FAMILY3, ROW(FIELD(F3COL1, DOUBLE()), FIELD(F3COL2, DataTypes.BOOLEAN()), FIELD(F3COL3, STRING()))));
    HBaseTableSchema hbaseSchema = HBaseTableSchema.fromDataType(dataType);
    return new HBaseRowDataAsyncLookupFunction(getConf(), TEST_TABLE_1, hbaseSchema, "null", lookupOptions);
}

Example 9 with HBaseTableSchema

use of org.apache.flink.connector.hbase.util.HBaseTableSchema in project flink by apache.

the class HBaseConnectorITCase method testTableInputFormatOpenClose.

@Test
public void testTableInputFormatOpenClose() throws IOException {
    HBaseTableSchema tableSchema = new HBaseTableSchema();
    tableSchema.addColumn(FAMILY1, F1COL1, byte[].class);
    AbstractTableInputFormat<?> inputFormat = new HBaseRowDataInputFormat(getConf(), TEST_TABLE_1, tableSchema, "null");
    inputFormat.open(inputFormat.createInputSplits(1)[0]);
    assertNotNull(inputFormat.getConnection());
    assertNotNull(inputFormat.getConnection().getTable(TableName.valueOf(TEST_TABLE_1)));
    inputFormat.close();
    assertNull(inputFormat.getConnection());
}

Example 10 with HBaseTableSchema

use of org.apache.flink.connector.hbase.util.HBaseTableSchema in project flink by apache.

the class HBase2DynamicTableFactory method createDynamicTableSink.

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
    TableFactoryHelper helper = createTableFactoryHelper(this, context);
    helper.validateExcept(PROPERTIES_PREFIX);
    final ReadableConfig tableOptions = helper.getOptions();
    validatePrimaryKey(context.getPhysicalRowDataType(), context.getPrimaryKeyIndexes());
    String tableName = tableOptions.get(TABLE_NAME);
    Configuration hbaseConf = getHBaseConfiguration(tableOptions);
    HBaseWriteOptions hBaseWriteOptions = getHBaseWriteOptions(tableOptions);
    String nullStringLiteral = tableOptions.get(NULL_STRING_LITERAL);
    HBaseTableSchema hbaseSchema = HBaseTableSchema.fromDataType(context.getPhysicalRowDataType());
    return new HBaseDynamicTableSink(tableName, hbaseSchema, hbaseConf, hBaseWriteOptions, nullStringLiteral);
}