Search in sources :

Example 81 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerLocalScan method testCreateTableWithColumnSpecificationMultilevelPartitioned.

@Test
public void testCreateTableWithColumnSpecificationMultilevelPartitioned() throws IOException {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).identity("first_name").identity("last_name").build();
    Map<StructLike, List<Record>> data = ImmutableMap.of(Row.of("Alice", "Brown"), Collections.singletonList(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.get(0)), Row.of("Bob", "Green"), Collections.singletonList(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.get(1)), Row.of("Trudy", "Pink"), Collections.singletonList(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.get(2)));
    String createSql = "CREATE EXTERNAL TABLE " + identifier + " (customer_id BIGINT) " + "PARTITIONED BY (first_name STRING COMMENT 'This is first name', " + "last_name STRING COMMENT 'This is last name') " + "STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + testTables.propertiesForCreateTableSQL(ImmutableMap.of());
    runCreateAndReadTest(identifier, createSql, HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, data);
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) ArrayList(java.util.ArrayList) ImmutableList(org.apache.iceberg.relocated.com.google.common.collect.ImmutableList) List(java.util.List) StructLike(org.apache.iceberg.StructLike) PartitionSpec(org.apache.iceberg.PartitionSpec) Test(org.junit.Test)

Example 82 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project presto by prestodb.

the class IcebergHadoopMetadata method renameColumn.

@Override
public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) {
    TableIdentifier tableIdentifier = toIcebergTableIdentifier(((IcebergTableHandle) tableHandle).getSchemaTableName());
    Table icebergTable = resourceFactory.getCatalog(session).loadTable(tableIdentifier);
    IcebergColumnHandle columnHandle = (IcebergColumnHandle) source;
    icebergTable.updateSchema().renameColumn(columnHandle.getName(), target).commit();
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table)

Example 83 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project presto by prestodb.

the class IcebergHadoopMetadata method dropColumn.

@Override
public void dropColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column) {
    TableIdentifier tableIdentifier = toIcebergTableIdentifier(((IcebergTableHandle) tableHandle).getSchemaTableName());
    Table icebergTable = resourceFactory.getCatalog(session).loadTable(tableIdentifier);
    IcebergColumnHandle handle = (IcebergColumnHandle) column;
    icebergTable.updateSchema().deleteColumn(handle.getName()).commit();
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table)

Example 84 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project presto by prestodb.

the class IcebergHadoopMetadata method getTableHandle.

@Override
public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) {
    IcebergTableName name = IcebergTableName.from(tableName.getTableName());
    verify(name.getTableType() == DATA, "Wrong table type: " + name.getTableType());
    TableIdentifier tableIdentifier = toIcebergTableIdentifier(tableName.getSchemaName(), name.getTableName());
    Table table;
    try {
        table = resourceFactory.getCatalog(session).loadTable(tableIdentifier);
    } catch (NoSuchTableException e) {
        // return null to throw
        return null;
    }
    return new IcebergTableHandle(tableName.getSchemaName(), name.getTableName(), name.getTableType(), resolveSnapshotIdByName(table, name), TupleDomain.all());
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table) NoSuchTableException(org.apache.iceberg.exceptions.NoSuchTableException)

Example 85 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project incubator-gobblin by apache.

the class IcebergMetadataWriter method createTable.

protected Table createTable(GobblinMetadataChangeEvent gmce, HiveSpec spec) throws IOException {
    String schema = gmce.getTableSchema();
    org.apache.hadoop.hive.metastore.api.Table table = HiveMetaStoreUtils.getTable(spec.getTable());
    IcebergUtils.IcebergDataAndPartitionSchema schemas = IcebergUtils.getIcebergSchema(schema, table);
    TableIdentifier tid = TableIdentifier.of(table.getDbName(), table.getTableName());
    Schema tableSchema = schemas.tableSchema;
    Preconditions.checkState(tableSchema != null, "Table schema cannot be null when creating a table");
    PartitionSpec partitionSpec = IcebergUtils.getPartitionSpec(tableSchema, schemas.partitionSchema);
    Table icebergTable = null;
    String tableLocation = null;
    if (useDataLocationAsTableLocation) {
        tableLocation = gmce.getDatasetIdentifier().getNativeName() + String.format(TABLE_LOCATION_SUFFIX, table.getDbName());
        // Set the path permission
        Path tablePath = new Path(tableLocation);
        WriterUtils.mkdirsWithRecursivePermission(tablePath.getFileSystem(conf), tablePath, permission);
    }
    try (Timer.Context context = metricContext.timer(CREATE_TABLE_TIME).time()) {
        icebergTable = catalog.createTable(tid, tableSchema, partitionSpec, tableLocation, IcebergUtils.getTableProperties(table));
        log.info("Created table {}, schema: {} partition spec: {}", tid, tableSchema, partitionSpec);
    } catch (AlreadyExistsException e) {
        log.warn("table {} already exist, there may be some other process try to create table concurrently", tid);
    }
    return icebergTable;
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) Path(org.apache.hadoop.fs.Path) Table(org.apache.iceberg.Table) AlreadyExistsException(org.apache.iceberg.exceptions.AlreadyExistsException) IcebergUtils(org.apache.gobblin.iceberg.Utils.IcebergUtils) Schema(org.apache.iceberg.Schema) PartitionSpec(org.apache.iceberg.PartitionSpec) Timer(com.codahale.metrics.Timer)

Aggregations

TableIdentifier (org.apache.iceberg.catalog.TableIdentifier)87 Test (org.junit.Test)69 Table (org.apache.iceberg.Table)56 PartitionSpec (org.apache.iceberg.PartitionSpec)27 Schema (org.apache.iceberg.Schema)25 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)16 BaseTable (org.apache.iceberg.BaseTable)15 UpdateSchema (org.apache.iceberg.UpdateSchema)15 List (java.util.List)13 NoSuchTableException (org.apache.iceberg.exceptions.NoSuchTableException)13 ArrayList (java.util.ArrayList)11 ImmutableList (org.apache.iceberg.relocated.com.google.common.collect.ImmutableList)11 IOException (java.io.IOException)10 Map (java.util.Map)10 Types (org.apache.iceberg.types.Types)10 HashMap (java.util.HashMap)9 Path (org.apache.hadoop.fs.Path)9 TableProperties (org.apache.iceberg.TableProperties)9 Collections (java.util.Collections)8 Properties (java.util.Properties)8