Search in sources :

Example 6 with Table

use of org.apache.iceberg.Table in project presto by prestodb.

the class IcebergHadoopMetadata method getSystemTable.

@Override
public Optional<SystemTable> getSystemTable(ConnectorSession session, SchemaTableName tableName) {
    IcebergTableName name = IcebergTableName.from(tableName.getTableName());
    TableIdentifier tableIdentifier = toIcebergTableIdentifier(tableName.getSchemaName(), name.getTableName());
    Table table;
    try {
        table = resourceFactory.getCatalog(session).loadTable(tableIdentifier);
    } catch (NoSuchTableException e) {
        return Optional.empty();
    }
    if (name.getSnapshotId().isPresent() && table.snapshot(name.getSnapshotId().get()) == null) {
        throw new PrestoException(ICEBERG_INVALID_SNAPSHOT_ID, format("Invalid snapshot [%s] for table: %s", name.getSnapshotId().get(), table));
    }
    return getIcebergSystemTable(tableName, table);
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table) NoSuchTableException(org.apache.iceberg.exceptions.NoSuchTableException) PrestoException(com.facebook.presto.spi.PrestoException)

Example 7 with Table

use of org.apache.iceberg.Table in project presto by prestodb.

the class IcebergHadoopMetadata method beginCreateTable.

@Override
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout) {
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    Schema schema = toIcebergSchema(tableMetadata.getColumns());
    PartitionSpec partitionSpec = parsePartitionFields(schema, getPartitioning(tableMetadata.getProperties()));
    ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.builder();
    FileFormat fileFormat = getFileFormat(tableMetadata.getProperties());
    propertiesBuilder.put(DEFAULT_FILE_FORMAT, fileFormat.toString());
    if (tableMetadata.getComment().isPresent()) {
        propertiesBuilder.put(TABLE_COMMENT, tableMetadata.getComment().get());
    }
    String formatVersion = getFormatVersion(tableMetadata.getProperties());
    if (formatVersion != null) {
        propertiesBuilder.put(FORMAT_VERSION, formatVersion);
    }
    try {
        transaction = resourceFactory.getCatalog(session).newCreateTableTransaction(toIcebergTableIdentifier(schemaTableName), schema, partitionSpec, propertiesBuilder.build());
    } catch (AlreadyExistsException e) {
        throw new TableAlreadyExistsException(schemaTableName);
    }
    Table icebergTable = transaction.table();
    return new IcebergWritableTableHandle(schemaName, tableName, SchemaParser.toJson(icebergTable.schema()), PartitionSpecParser.toJson(icebergTable.spec()), getColumns(icebergTable.schema(), typeManager), icebergTable.location(), fileFormat, icebergTable.properties());
}
Also used : TableAlreadyExistsException(com.facebook.presto.hive.TableAlreadyExistsException) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table) AlreadyExistsException(org.apache.iceberg.exceptions.AlreadyExistsException) TableAlreadyExistsException(com.facebook.presto.hive.TableAlreadyExistsException) Schema(org.apache.iceberg.Schema) FileFormat(org.apache.iceberg.FileFormat) IcebergTableProperties.getFileFormat(com.facebook.presto.iceberg.IcebergTableProperties.getFileFormat) SchemaTableName(com.facebook.presto.spi.SchemaTableName) PartitionSpec(org.apache.iceberg.PartitionSpec) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap)

Example 8 with Table

use of org.apache.iceberg.Table in project presto by prestodb.

the class IcebergHadoopMetadata method addColumn.

@Override
public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) {
    TableIdentifier tableIdentifier = toIcebergTableIdentifier(((IcebergTableHandle) tableHandle).getSchemaTableName());
    Table icebergTable = resourceFactory.getCatalog(session).loadTable(tableIdentifier);
    icebergTable.updateSchema().addColumn(column.getName(), toIcebergType(column.getType())).commit();
}
Also used : TableIdentifier(org.apache.iceberg.catalog.TableIdentifier) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table)

Example 9 with Table

use of org.apache.iceberg.Table in project presto by prestodb.

the class IcebergHadoopMetadata method getTableMetadata.

@Override
protected ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName table) {
    Table icebergTable;
    try {
        icebergTable = getHadoopIcebergTable(resourceFactory, session, table);
    } catch (NoSuchTableException e) {
        throw new TableNotFoundException(table);
    }
    List<ColumnMetadata> columns = getColumnMetadatas(icebergTable);
    return new ConnectorTableMetadata(table, columns, createMetadataProperties(icebergTable), getTableComment(icebergTable));
}
Also used : TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) SystemTable(com.facebook.presto.spi.SystemTable) IcebergUtil.getHadoopIcebergTable(com.facebook.presto.iceberg.IcebergUtil.getHadoopIcebergTable) Table(org.apache.iceberg.Table) NoSuchTableException(org.apache.iceberg.exceptions.NoSuchTableException) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata)

Example 10 with Table

use of org.apache.iceberg.Table in project presto by prestodb.

the class IcebergUtil method getTableScan.

public static TableScan getTableScan(TupleDomain<IcebergColumnHandle> predicates, Optional<Long> snapshotId, Table icebergTable) {
    Expression expression = ExpressionConverter.toIcebergExpression(predicates);
    TableScan tableScan = icebergTable.newScan().filter(expression);
    return snapshotId.map(id -> isSnapshot(icebergTable, id) ? tableScan.useSnapshot(id) : tableScan.asOfTime(id)).orElse(tableScan);
}
Also used : HdfsEnvironment(com.facebook.presto.hive.HdfsEnvironment) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) ICEBERG_TABLE_TYPE_VALUE(org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE) PrestoException(com.facebook.presto.spi.PrestoException) WRITE_LOCATION_PROVIDER_IMPL(org.apache.iceberg.TableProperties.WRITE_LOCATION_PROVIDER_IMPL) TABLE_TYPE_PROP(org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP) PartitionField(org.apache.iceberg.PartitionField) LocationProvider(org.apache.iceberg.io.LocationProvider) TableOperations(org.apache.iceberg.TableOperations) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Expression(org.apache.iceberg.expressions.Expression) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) HistoryEntry(org.apache.iceberg.HistoryEntry) Locale(java.util.Locale) TypeManager(com.facebook.presto.common.type.TypeManager) Map(java.util.Map) TABLE_COMMENT(com.facebook.presto.hive.HiveMetadata.TABLE_COMMENT) DEFAULT_FILE_FORMAT(org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT) HdfsContext(com.facebook.presto.hive.HdfsContext) LocationProviders.locationsFor(org.apache.iceberg.LocationProviders.locationsFor) TypeConverter.toPrestoType(com.facebook.presto.iceberg.TypeConverter.toPrestoType) HiveColumnConverterProvider(com.facebook.presto.hive.HiveColumnConverterProvider) BaseTable(org.apache.iceberg.BaseTable) ImmutableMap(com.google.common.collect.ImmutableMap) Table(org.apache.iceberg.Table) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScan(org.apache.iceberg.TableScan) Schema(org.apache.iceberg.Schema) FileFormat(org.apache.iceberg.FileFormat) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) Streams.stream(com.google.common.collect.Streams.stream) List(java.util.List) IcebergPrestoModelConverters.toIcebergTableIdentifier(com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) PartitionSpec(org.apache.iceberg.PartitionSpec) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) DEFAULT_FILE_FORMAT_DEFAULT(org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT) ICEBERG_INVALID_SNAPSHOT_ID(com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_SNAPSHOT_ID) Lists.reverse(com.google.common.collect.Lists.reverse) Snapshot(org.apache.iceberg.Snapshot) TableScan(org.apache.iceberg.TableScan) Expression(org.apache.iceberg.expressions.Expression)

Aggregations

Table (org.apache.iceberg.Table)188 Test (org.junit.Test)132 Schema (org.apache.iceberg.Schema)66 TableIdentifier (org.apache.iceberg.catalog.TableIdentifier)56 Record (org.apache.iceberg.data.Record)56 PartitionSpec (org.apache.iceberg.PartitionSpec)51 IOException (java.io.IOException)27 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)27 List (java.util.List)22 Map (java.util.Map)20 DataFile (org.apache.iceberg.DataFile)19 NoSuchTableException (org.apache.iceberg.exceptions.NoSuchTableException)19 Collectors (java.util.stream.Collectors)18 BaseTable (org.apache.iceberg.BaseTable)18 Types (org.apache.iceberg.types.Types)18 Properties (java.util.Properties)17 Configuration (org.apache.hadoop.conf.Configuration)17 Path (org.apache.hadoop.fs.Path)17 FileFormat (org.apache.iceberg.FileFormat)16 ArrayList (java.util.ArrayList)15