Search in sources :

Example 36 with TableNotFoundException

use of io.trino.spi.connector.TableNotFoundException in project trino by trinodb.

the class BigQueryMetadata method getViewDefinitionSystemTable.

private Optional<SystemTable> getViewDefinitionSystemTable(ConnectorSession session, SchemaTableName viewDefinitionTableName, SchemaTableName sourceTableName) {
    BigQueryClient client = bigQueryClientFactory.create(session);
    String projectId = getProjectId(client);
    String remoteSchemaName = client.toRemoteDataset(projectId, sourceTableName.getSchemaName()).map(RemoteDatabaseObject::getOnlyRemoteName).orElseThrow(() -> new TableNotFoundException(viewDefinitionTableName));
    String remoteTableName = client.toRemoteTable(projectId, remoteSchemaName, sourceTableName.getTableName()).map(RemoteDatabaseObject::getOnlyRemoteName).orElseThrow(() -> new TableNotFoundException(viewDefinitionTableName));
    TableInfo tableInfo = client.getTable(TableId.of(projectId, remoteSchemaName, remoteTableName)).orElseThrow(() -> new TableNotFoundException(viewDefinitionTableName));
    if (!(tableInfo.getDefinition() instanceof ViewDefinition)) {
        throw new TableNotFoundException(viewDefinitionTableName);
    }
    List<ColumnMetadata> columns = ImmutableList.of(new ColumnMetadata("query", VarcharType.VARCHAR));
    List<Type> types = columns.stream().map(ColumnMetadata::getType).collect(toImmutableList());
    Optional<String> query = Optional.ofNullable(((ViewDefinition) tableInfo.getDefinition()).getQuery());
    Iterable<List<Object>> propertyValues = ImmutableList.of(ImmutableList.of(query.orElse("NULL")));
    return Optional.of(createSystemTable(new ConnectorTableMetadata(sourceTableName, columns), constraint -> new InMemoryRecordSet(types, propertyValues).cursor()));
}
Also used : ViewDefinition(com.google.cloud.bigquery.ViewDefinition) StandardTableDefinition(com.google.cloud.bigquery.StandardTableDefinition) TableId(com.google.cloud.bigquery.TableId) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) VIEW(com.google.cloud.bigquery.TableDefinition.Type.VIEW) Schema(com.google.cloud.bigquery.Schema) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) Map(java.util.Map) ProjectionApplicationResult(io.trino.spi.connector.ProjectionApplicationResult) RemoteDatabaseObject(io.trino.plugin.bigquery.BigQueryClient.RemoteDatabaseObject) ENGLISH(java.util.Locale.ENGLISH) TABLE(com.google.cloud.bigquery.TableDefinition.Type.TABLE) Field(com.google.cloud.bigquery.Field) TableDefinition(com.google.cloud.bigquery.TableDefinition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TrinoException(io.trino.spi.TrinoException) Streams(com.google.common.collect.Streams) SchemaTableName(io.trino.spi.connector.SchemaTableName) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) BIGQUERY_LISTING_DATASET_ERROR(io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_LISTING_DATASET_ERROR) TrinoPrincipal(io.trino.spi.security.TrinoPrincipal) SchemaTablePrefix(io.trino.spi.connector.SchemaTablePrefix) Assignment(io.trino.spi.connector.Assignment) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) ConnectorMetadata(io.trino.spi.connector.ConnectorMetadata) SystemTable(io.trino.spi.connector.SystemTable) Constraint(io.trino.spi.connector.Constraint) Logger(io.airlift.log.Logger) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) Type(io.trino.spi.type.Type) BigQueryException(com.google.cloud.bigquery.BigQueryException) BigQueryType.toField(io.trino.plugin.bigquery.BigQueryType.toField) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) DatasetId(com.google.cloud.bigquery.DatasetId) Function(java.util.function.Function) Inject(javax.inject.Inject) VarcharType(io.trino.spi.type.VarcharType) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) ColumnHandle(io.trino.spi.connector.ColumnHandle) Table(com.google.cloud.bigquery.Table) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) RecordCursor(io.trino.spi.connector.RecordCursor) ConnectorSession(io.trino.spi.connector.ConnectorSession) TupleDomain(io.trino.spi.predicate.TupleDomain) InMemoryRecordSet(io.trino.spi.connector.InMemoryRecordSet) ConnectorTableProperties(io.trino.spi.connector.ConnectorTableProperties) ConnectorExpression(io.trino.spi.expression.ConnectorExpression) DatasetInfo(com.google.cloud.bigquery.DatasetInfo) TableInfo(com.google.cloud.bigquery.TableInfo) ConnectorTransactionHandle(io.trino.spi.connector.ConnectorTransactionHandle) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) ViewDefinition(com.google.cloud.bigquery.ViewDefinition) InMemoryRecordSet(io.trino.spi.connector.InMemoryRecordSet) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Type(io.trino.spi.type.Type) VarcharType(io.trino.spi.type.VarcharType) TableInfo(com.google.cloud.bigquery.TableInfo) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata)

Example 37 with TableNotFoundException

use of io.trino.spi.connector.TableNotFoundException in project trino by trinodb.

the class BigQueryMetadata method listTableColumns.

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) {
    log.debug("listTableColumns(session=%s, prefix=%s)", session, prefix);
    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    List<SchemaTableName> tables = prefix.toOptionalSchemaTableName().<List<SchemaTableName>>map(ImmutableList::of).orElseGet(() -> listTables(session, prefix.getSchema()));
    for (SchemaTableName tableName : tables) {
        try {
            Optional.ofNullable(getTableHandleIgnoringConflicts(session, tableName)).ifPresent(tableHandle -> columns.put(tableName, getTableMetadata(session, tableHandle).getColumns()));
        } catch (TableNotFoundException e) {
        // table disappeared during listing operation
        }
    }
    return columns.buildOrThrow();
}
Also used : TableNotFoundException(io.trino.spi.connector.TableNotFoundException) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) SchemaTableName(io.trino.spi.connector.SchemaTableName) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap)

Example 38 with TableNotFoundException

use of io.trino.spi.connector.TableNotFoundException in project trino by trinodb.

the class BaseIcebergConnectorTest method assertFilterPushdown.

private void assertFilterPushdown(QualifiedObjectName tableName, Map<String, Domain> filter, Map<String, Domain> expectedEnforcedPredicate, Map<String, Domain> expectedUnenforcedPredicate) {
    Metadata metadata = getQueryRunner().getMetadata();
    newTransaction().execute(getSession(), session -> {
        TableHandle table = metadata.getTableHandle(session, tableName).orElseThrow(() -> new TableNotFoundException(tableName.asSchemaTableName()));
        Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, table);
        TupleDomain<ColumnHandle> domains = TupleDomain.withColumnDomains(filter.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue)));
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, table, new Constraint(domains));
        assertTrue(result.isEmpty() == (expectedUnenforcedPredicate == null && expectedEnforcedPredicate == null));
        if (result.isPresent()) {
            IcebergTableHandle newTable = (IcebergTableHandle) result.get().getHandle().getConnectorHandle();
            assertEquals(newTable.getEnforcedPredicate(), TupleDomain.withColumnDomains(expectedEnforcedPredicate.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue))));
            assertEquals(newTable.getUnenforcedPredicate(), TupleDomain.withColumnDomains(expectedUnenforcedPredicate.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue))));
        }
    });
}
Also used : SkipException(org.testng.SkipException) FileSystem(org.apache.hadoop.fs.FileSystem) Test(org.testng.annotations.Test) TestTable(io.trino.testing.sql.TestTable) BROADCAST(io.trino.sql.planner.OptimizerConfig.JoinDistributionType.BROADCAST) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Matcher(java.util.regex.Matcher) Map(java.util.Map) MaterializedRow(io.trino.testing.MaterializedRow) Path(java.nio.file.Path) Assert.assertFalse(org.testng.Assert.assertFalse) Assert.assertEquals(io.trino.testing.assertions.Assert.assertEquals) GenericDatumWriter(org.apache.avro.generic.GenericDatumWriter) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Schema(org.apache.avro.Schema) DataProviders(io.trino.testing.DataProviders) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) DataFileWriter(org.apache.avro.file.DataFileWriter) HDFS_ENVIRONMENT(io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT) Collectors.joining(java.util.stream.Collectors.joining) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) Stream(java.util.stream.Stream) ORC(io.trino.plugin.iceberg.IcebergFileFormat.ORC) Session(io.trino.Session) NullableValue(io.trino.spi.predicate.NullableValue) GenericData(org.apache.avro.generic.GenericData) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) String.join(java.lang.String.join) ColumnHandle(io.trino.spi.connector.ColumnHandle) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) LongStream(java.util.stream.LongStream) OperatorStats(io.trino.operator.OperatorStats) Language(org.intellij.lang.annotations.Language) Files(java.nio.file.Files) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) File(java.io.File) BaseConnectorTest(io.trino.testing.BaseConnectorTest) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Paths(java.nio.file.Paths) QueryRunner(io.trino.testing.QueryRunner) Domain.multipleValues(io.trino.spi.predicate.Domain.multipleValues) QueryId(io.trino.spi.QueryId) TransactionBuilder.transaction(io.trino.transaction.TransactionBuilder.transaction) IntStream.range(java.util.stream.IntStream.range) MaterializedResult(io.trino.testing.MaterializedResult) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS(io.trino.SystemSessionProperties.PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) ICEBERG_DOMAIN_COMPACTION_THRESHOLD(io.trino.plugin.iceberg.IcebergSplitManager.ICEBERG_DOMAIN_COMPACTION_THRESHOLD) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) QueryAssertions.assertEqualsIgnoreOrder(io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder) Locale(java.util.Locale) Iterables.concat(com.google.common.collect.Iterables.concat) TestingConnectorBehavior(io.trino.testing.TestingConnectorBehavior) TestTable.randomTableSuffix(io.trino.testing.sql.TestTable.randomTableSuffix) TpchTable(io.trino.tpch.TpchTable) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collections.nCopies(java.util.Collections.nCopies) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ICEBERG_CATALOG(io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG) DataSize(io.airlift.units.DataSize) List(java.util.List) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) BIGINT(io.trino.spi.type.BigintType.BIGINT) Domain.singleValue(io.trino.spi.predicate.Domain.singleValue) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) DataFileReader(org.apache.avro.file.DataFileReader) GenericDatumReader(org.apache.avro.generic.GenericDatumReader) IcebergQueryRunner.createIcebergQueryRunner(io.trino.plugin.iceberg.IcebergQueryRunner.createIcebergQueryRunner) LINE_ITEM(io.trino.tpch.TpchTable.LINE_ITEM) IntStream(java.util.stream.IntStream) Constraint(io.trino.spi.connector.Constraint) DataProvider(org.testng.annotations.DataProvider) PARQUET(io.trino.plugin.iceberg.IcebergFileFormat.PARQUET) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatistics(io.trino.spi.statistics.TableStatistics) NoSuchElementException(java.util.NoSuchElementException) VerifyException(com.google.common.base.VerifyException) OutputStream(java.io.OutputStream) ResultWithQueryId(io.trino.testing.ResultWithQueryId) TupleDomain(io.trino.spi.predicate.TupleDomain) Consumer(java.util.function.Consumer) Assert.assertEventually(io.trino.testing.assertions.Assert.assertEventually) JOIN_DISTRIBUTION_TYPE(io.trino.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) Metadata(io.trino.metadata.Metadata) Assert.assertTrue(org.testng.Assert.assertTrue) MaterializedResult.resultBuilder(io.trino.testing.MaterializedResult.resultBuilder) ColumnHandle(io.trino.spi.connector.ColumnHandle) Constraint(io.trino.spi.connector.Constraint) Metadata(io.trino.metadata.Metadata) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) TableHandle(io.trino.metadata.TableHandle) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 39 with TableNotFoundException

use of io.trino.spi.connector.TableNotFoundException in project trino by trinodb.

the class TrinoHiveCatalog method dropTable.

@Override
public void dropTable(ConnectorSession session, SchemaTableName schemaTableName) {
    BaseTable table = (BaseTable) loadTable(session, schemaTableName);
    TableMetadata metadata = table.operations().current();
    validateTableCanBeDropped(table);
    io.trino.plugin.hive.metastore.Table metastoreTable = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    metastore.dropTable(schemaTableName.getSchemaName(), schemaTableName.getTableName(), false);
    // Use the Iceberg routine for dropping the table data because the data files
    // of the Iceberg table may be located in different locations
    dropTableData(table.io(), metadata);
    deleteTableDirectory(session, schemaTableName, hdfsEnvironment, new Path(metastoreTable.getStorage().getLocation()));
}
Also used : ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) TableMetadata(org.apache.iceberg.TableMetadata) Path(org.apache.hadoop.fs.Path) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) BaseTable(org.apache.iceberg.BaseTable)

Example 40 with TableNotFoundException

use of io.trino.spi.connector.TableNotFoundException in project trino by trinodb.

the class GlueIcebergTableOperations method getRefreshedLocation.

@Override
protected String getRefreshedLocation() {
    Table table = getTable();
    if (isPrestoView(table.getParameters()) && isHiveOrPrestoView(table.getTableType())) {
        // this is a Presto Hive view, hence not a table
        throw new TableNotFoundException(getSchemaTableName());
    }
    if (!isIcebergTable(table.getParameters())) {
        throw new UnknownTableTypeException(getSchemaTableName());
    }
    String metadataLocation = table.getParameters().get(METADATA_LOCATION_PROP);
    if (metadataLocation == null) {
        throw new TrinoException(ICEBERG_INVALID_METADATA, format("Table is missing [%s] property: %s", METADATA_LOCATION_PROP, getSchemaTableName()));
    }
    return metadataLocation;
}
Also used : TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Table(com.amazonaws.services.glue.model.Table) HiveUtil.isIcebergTable(io.trino.plugin.hive.util.HiveUtil.isIcebergTable) TrinoException(io.trino.spi.TrinoException) UnknownTableTypeException(io.trino.plugin.iceberg.UnknownTableTypeException)

Aggregations

TableNotFoundException (io.trino.spi.connector.TableNotFoundException)84 SchemaTableName (io.trino.spi.connector.SchemaTableName)65 TrinoException (io.trino.spi.TrinoException)39 Table (io.trino.plugin.hive.metastore.Table)33 ImmutableMap (com.google.common.collect.ImmutableMap)27 ImmutableList (com.google.common.collect.ImmutableList)26 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)26 List (java.util.List)25 Optional (java.util.Optional)24 HdfsContext (io.trino.plugin.hive.HdfsEnvironment.HdfsContext)22 Path (org.apache.hadoop.fs.Path)22 ColumnHandle (io.trino.spi.connector.ColumnHandle)21 Map (java.util.Map)21 Objects.requireNonNull (java.util.Objects.requireNonNull)20 ConnectorSession (io.trino.spi.connector.ConnectorSession)19 ColumnMetadata (io.trino.spi.connector.ColumnMetadata)18 TupleDomain (io.trino.spi.predicate.TupleDomain)18 Set (java.util.Set)18 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)17 CatalogSchemaTableName (io.trino.spi.connector.CatalogSchemaTableName)17