Examples with CatalogTable org.apache.flink.table.catalog.CatalogTable used on opensource projects

Example 1 with CatalogTable

use of org.apache.flink.table.catalog.CatalogTable in project flink by apache.

the class HiveDynamicTableFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    final ReadableConfig configuration = Configuration.fromMap(context.getCatalogTable().getOptions());
    final boolean isHiveTable = HiveCatalog.isHiveTable(context.getCatalogTable().getOptions());
    // we don't support temporary hive tables yet
    if (!isHiveTable || context.isTemporary()) {
        DynamicTableSource source = FactoryUtil.createDynamicTableSource(null, context.getObjectIdentifier(), context.getCatalogTable(), context.getConfiguration(), context.getClassLoader(), context.isTemporary());
        if (source instanceof RequireCatalogLock) {
            ((RequireCatalogLock) source).setLockFactory(HiveCatalogLock.createFactory(hiveConf));
        }
        return source;
    }
    final CatalogTable catalogTable = Preconditions.checkNotNull(context.getCatalogTable());
    final boolean isStreamingSource = configuration.get(STREAMING_SOURCE_ENABLE);
    final boolean includeAllPartition = STREAMING_SOURCE_PARTITION_INCLUDE.defaultValue().equals(configuration.get(STREAMING_SOURCE_PARTITION_INCLUDE));
    final JobConf jobConf = JobConfUtils.createJobConfWithCredentials(hiveConf);
    // hive table source that has not lookup ability
    if (isStreamingSource && includeAllPartition) {
        return new HiveTableSource(jobConf, context.getConfiguration(), context.getObjectIdentifier().toObjectPath(), catalogTable);
    } else {
        // hive table source that has scan and lookup ability
        return new HiveLookupTableSource(jobConf, context.getConfiguration(), context.getObjectIdentifier().toObjectPath(), catalogTable);
    }
}

Example 2 with CatalogTable

use of org.apache.flink.table.catalog.CatalogTable in project flink by apache.

the class HiveTableUtil method instantiateHiveTable.

public static Table instantiateHiveTable(ObjectPath tablePath, CatalogBaseTable table, HiveConf hiveConf, boolean managedTable) {
    final boolean isView = table instanceof CatalogView;
    // let Hive set default parameters for us, e.g. serialization.format
    Table hiveTable = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(tablePath.getDatabaseName(), tablePath.getObjectName());
    hiveTable.setCreateTime((int) (System.currentTimeMillis() / 1000));
    Map<String, String> properties = new HashMap<>(table.getOptions());
    if (managedTable) {
        properties.put(CONNECTOR.key(), ManagedTableFactory.DEFAULT_IDENTIFIER);
    }
    // Table comment
    if (table.getComment() != null) {
        properties.put(HiveCatalogConfig.COMMENT, table.getComment());
    }
    boolean isHiveTable = HiveCatalog.isHiveTable(properties);
    // Hive table's StorageDescriptor
    StorageDescriptor sd = hiveTable.getSd();
    HiveTableUtil.setDefaultStorageFormat(sd, hiveConf);
    // because hive cannot understand the expanded query anyway
    if (isHiveTable && !isView) {
        HiveTableUtil.initiateTableFromProperties(hiveTable, properties, hiveConf);
        List<FieldSchema> allColumns = HiveTableUtil.createHiveColumns(table.getSchema());
        // Table columns and partition keys
        if (table instanceof CatalogTable) {
            CatalogTable catalogTable = (CatalogTable) table;
            if (catalogTable.isPartitioned()) {
                int partitionKeySize = catalogTable.getPartitionKeys().size();
                List<FieldSchema> regularColumns = allColumns.subList(0, allColumns.size() - partitionKeySize);
                List<FieldSchema> partitionColumns = allColumns.subList(allColumns.size() - partitionKeySize, allColumns.size());
                sd.setCols(regularColumns);
                hiveTable.setPartitionKeys(partitionColumns);
            } else {
                sd.setCols(allColumns);
                hiveTable.setPartitionKeys(new ArrayList<>());
            }
        } else {
            sd.setCols(allColumns);
        }
        // Table properties
        hiveTable.getParameters().putAll(properties);
    } else {
        DescriptorProperties tableSchemaProps = new DescriptorProperties(true);
        tableSchemaProps.putTableSchema(Schema.SCHEMA, table.getSchema());
        if (table instanceof CatalogTable) {
            tableSchemaProps.putPartitionKeys(((CatalogTable) table).getPartitionKeys());
        }
        properties.putAll(tableSchemaProps.asMap());
        properties = maskFlinkProperties(properties);
        // 2. when creating views which don't have connector properties
        if (isView || (!properties.containsKey(FLINK_PROPERTY_PREFIX + CONNECTOR.key()) && !properties.containsKey(FLINK_PROPERTY_PREFIX + CONNECTOR_TYPE))) {
            properties.put(IS_GENERIC, "true");
        }
        hiveTable.setParameters(properties);
    }
    if (isView) {
        // TODO: [FLINK-12398] Support partitioned view in catalog API
        hiveTable.setPartitionKeys(new ArrayList<>());
        CatalogView view = (CatalogView) table;
        hiveTable.setViewOriginalText(view.getOriginalQuery());
        hiveTable.setViewExpandedText(view.getExpandedQuery());
        hiveTable.setTableType(TableType.VIRTUAL_VIEW.name());
    }
    return hiveTable;
}

Example 3 with CatalogTable

use of org.apache.flink.table.catalog.CatalogTable in project flink by apache.

the class HivePartitionFetcherTest method testIgnoreNonExistPartition.

@Test
public void testIgnoreNonExistPartition() throws Exception {
    // it's possible a partition path exists but the partition is not added to HMS, e.g. the
    // partition is still being loaded, or the path is simply misplaced
    // make sure the fetch can ignore such paths
    HiveCatalog hiveCatalog = HiveTestUtils.createHiveCatalog();
    hiveCatalog.open();
    // create test table
    String[] fieldNames = new String[] { "i", "date" };
    DataType[] fieldTypes = new DataType[] { DataTypes.INT(), DataTypes.STRING() };
    TableSchema schema = TableSchema.builder().fields(fieldNames, fieldTypes).build();
    List<String> partitionKeys = Collections.singletonList("date");
    Map<String, String> options = new HashMap<>();
    options.put("connector", "hive");
    CatalogTable catalogTable = new CatalogTableImpl(schema, partitionKeys, options, null);
    ObjectPath tablePath = new ObjectPath("default", "test");
    hiveCatalog.createTable(tablePath, catalogTable, false);
    // add a valid partition path
    Table hiveTable = hiveCatalog.getHiveTable(tablePath);
    Path path = new Path(hiveTable.getSd().getLocation(), "date=2021-06-18");
    FileSystem fs = path.getFileSystem(hiveCatalog.getHiveConf());
    fs.mkdirs(path);
    // test partition-time order
    Configuration flinkConf = new Configuration();
    flinkConf.set(STREAMING_SOURCE_PARTITION_ORDER, HiveOptions.PartitionOrder.PARTITION_TIME);
    HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveCatalog.getHiveVersion());
    JobConfWrapper jobConfWrapper = new JobConfWrapper(new JobConf(hiveCatalog.getHiveConf()));
    String defaultPartName = "__HIVE_DEFAULT_PARTITION__";
    MyHivePartitionFetcherContext fetcherContext = new MyHivePartitionFetcherContext(tablePath, hiveShim, jobConfWrapper, partitionKeys, fieldTypes, fieldNames, flinkConf, defaultPartName);
    fetcherContext.open();
    assertEquals(0, fetcherContext.getComparablePartitionValueList().size());
    // test create-time order
    flinkConf.set(STREAMING_SOURCE_PARTITION_ORDER, HiveOptions.PartitionOrder.CREATE_TIME);
    fetcherContext = new MyHivePartitionFetcherContext(tablePath, hiveShim, jobConfWrapper, partitionKeys, fieldTypes, fieldNames, flinkConf, defaultPartName);
    fetcherContext.open();
    assertEquals(0, fetcherContext.getComparablePartitionValueList().size());
    // test partition-name order
    flinkConf.set(STREAMING_SOURCE_PARTITION_ORDER, HiveOptions.PartitionOrder.PARTITION_NAME);
    fetcherContext = new MyHivePartitionFetcherContext(tablePath, hiveShim, jobConfWrapper, partitionKeys, fieldTypes, fieldNames, flinkConf, defaultPartName);
    fetcherContext.open();
    assertEquals(0, fetcherContext.getComparablePartitionValueList().size());
}

Example 4 with CatalogTable

use of org.apache.flink.table.catalog.CatalogTable in project flink by apache.

the class HiveCatalogDataTypeTest method verifyDataTypes.

private void verifyDataTypes(DataType[] types) throws Exception {
    CatalogTable table = createCatalogTable(types);
    catalog.createDatabase(db1, createDb(), false);
    catalog.createTable(path1, table, false);
    assertEquals(table.getSchema(), catalog.getTable(path1).getSchema());
}

Example 5 with CatalogTable

use of org.apache.flink.table.catalog.CatalogTable in project flink by apache.

the class HiveCatalogDataTypeTest method testNonSupportedBinaryDataTypes.

@Test
public void testNonSupportedBinaryDataTypes() throws Exception {
    DataType[] types = new DataType[] { DataTypes.BINARY(BinaryType.MAX_LENGTH) };
    CatalogTable table = createCatalogTable(types);
    catalog.createDatabase(db1, createDb(), false);
    exception.expect(UnsupportedOperationException.class);
    catalog.createTable(path1, table, false);
}