Search in sources :

Example 26 with TableHandle

use of io.trino.metadata.TableHandle in project trino by trinodb.

the class BaseIcebergConnectorTest method assertFilterPushdown.

private void assertFilterPushdown(QualifiedObjectName tableName, Map<String, Domain> filter, Map<String, Domain> expectedEnforcedPredicate, Map<String, Domain> expectedUnenforcedPredicate) {
    Metadata metadata = getQueryRunner().getMetadata();
    newTransaction().execute(getSession(), session -> {
        TableHandle table = metadata.getTableHandle(session, tableName).orElseThrow(() -> new TableNotFoundException(tableName.asSchemaTableName()));
        Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, table);
        TupleDomain<ColumnHandle> domains = TupleDomain.withColumnDomains(filter.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue)));
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, table, new Constraint(domains));
        assertTrue(result.isEmpty() == (expectedUnenforcedPredicate == null && expectedEnforcedPredicate == null));
        if (result.isPresent()) {
            IcebergTableHandle newTable = (IcebergTableHandle) result.get().getHandle().getConnectorHandle();
            assertEquals(newTable.getEnforcedPredicate(), TupleDomain.withColumnDomains(expectedEnforcedPredicate.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue))));
            assertEquals(newTable.getUnenforcedPredicate(), TupleDomain.withColumnDomains(expectedUnenforcedPredicate.entrySet().stream().collect(toImmutableMap(entry -> columns.get(entry.getKey()), Map.Entry::getValue))));
        }
    });
}
Also used : SkipException(org.testng.SkipException) FileSystem(org.apache.hadoop.fs.FileSystem) Test(org.testng.annotations.Test) TestTable(io.trino.testing.sql.TestTable) BROADCAST(io.trino.sql.planner.OptimizerConfig.JoinDistributionType.BROADCAST) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Matcher(java.util.regex.Matcher) Map(java.util.Map) MaterializedRow(io.trino.testing.MaterializedRow) Path(java.nio.file.Path) Assert.assertFalse(org.testng.Assert.assertFalse) Assert.assertEquals(io.trino.testing.assertions.Assert.assertEquals) GenericDatumWriter(org.apache.avro.generic.GenericDatumWriter) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Schema(org.apache.avro.Schema) DataProviders(io.trino.testing.DataProviders) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) DataFileWriter(org.apache.avro.file.DataFileWriter) HDFS_ENVIRONMENT(io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT) Collectors.joining(java.util.stream.Collectors.joining) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) Stream(java.util.stream.Stream) ORC(io.trino.plugin.iceberg.IcebergFileFormat.ORC) Session(io.trino.Session) NullableValue(io.trino.spi.predicate.NullableValue) GenericData(org.apache.avro.generic.GenericData) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) String.join(java.lang.String.join) ColumnHandle(io.trino.spi.connector.ColumnHandle) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) LongStream(java.util.stream.LongStream) OperatorStats(io.trino.operator.OperatorStats) Language(org.intellij.lang.annotations.Language) Files(java.nio.file.Files) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) File(java.io.File) BaseConnectorTest(io.trino.testing.BaseConnectorTest) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Paths(java.nio.file.Paths) QueryRunner(io.trino.testing.QueryRunner) Domain.multipleValues(io.trino.spi.predicate.Domain.multipleValues) QueryId(io.trino.spi.QueryId) TransactionBuilder.transaction(io.trino.transaction.TransactionBuilder.transaction) IntStream.range(java.util.stream.IntStream.range) MaterializedResult(io.trino.testing.MaterializedResult) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS(io.trino.SystemSessionProperties.PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) ICEBERG_DOMAIN_COMPACTION_THRESHOLD(io.trino.plugin.iceberg.IcebergSplitManager.ICEBERG_DOMAIN_COMPACTION_THRESHOLD) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) QueryAssertions.assertEqualsIgnoreOrder(io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder) Locale(java.util.Locale) Iterables.concat(com.google.common.collect.Iterables.concat) TestingConnectorBehavior(io.trino.testing.TestingConnectorBehavior) TestTable.randomTableSuffix(io.trino.testing.sql.TestTable.randomTableSuffix) TpchTable(io.trino.tpch.TpchTable) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collections.nCopies(java.util.Collections.nCopies) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ICEBERG_CATALOG(io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG) DataSize(io.airlift.units.DataSize) List(java.util.List) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) BIGINT(io.trino.spi.type.BigintType.BIGINT) Domain.singleValue(io.trino.spi.predicate.Domain.singleValue) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) DataFileReader(org.apache.avro.file.DataFileReader) GenericDatumReader(org.apache.avro.generic.GenericDatumReader) IcebergQueryRunner.createIcebergQueryRunner(io.trino.plugin.iceberg.IcebergQueryRunner.createIcebergQueryRunner) LINE_ITEM(io.trino.tpch.TpchTable.LINE_ITEM) IntStream(java.util.stream.IntStream) Constraint(io.trino.spi.connector.Constraint) DataProvider(org.testng.annotations.DataProvider) PARQUET(io.trino.plugin.iceberg.IcebergFileFormat.PARQUET) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatistics(io.trino.spi.statistics.TableStatistics) NoSuchElementException(java.util.NoSuchElementException) VerifyException(com.google.common.base.VerifyException) OutputStream(java.io.OutputStream) ResultWithQueryId(io.trino.testing.ResultWithQueryId) TupleDomain(io.trino.spi.predicate.TupleDomain) Consumer(java.util.function.Consumer) Assert.assertEventually(io.trino.testing.assertions.Assert.assertEventually) JOIN_DISTRIBUTION_TYPE(io.trino.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) Metadata(io.trino.metadata.Metadata) Assert.assertTrue(org.testng.Assert.assertTrue) MaterializedResult.resultBuilder(io.trino.testing.MaterializedResult.resultBuilder) ColumnHandle(io.trino.spi.connector.ColumnHandle) Constraint(io.trino.spi.connector.Constraint) Metadata(io.trino.metadata.Metadata) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) TableHandle(io.trino.metadata.TableHandle) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 27 with TableHandle

use of io.trino.metadata.TableHandle in project trino by trinodb.

the class TestKuduIntegrationDynamicFilter method testIncompleteDynamicFilterTimeout.

@Test(timeOut = 30_000)
public void testIncompleteDynamicFilterTimeout() throws Exception {
    QueryRunner runner = getQueryRunner();
    TransactionManager transactionManager = runner.getTransactionManager();
    TransactionId transactionId = transactionManager.beginTransaction(false);
    Session session = Session.builder(getSession()).setCatalogSessionProperty("kudu", "dynamic_filtering_wait_timeout", "1s").build().beginTransactionId(transactionId, transactionManager, new AllowAllAccessControl());
    QualifiedObjectName tableName = new QualifiedObjectName("kudu", "tpch", "orders");
    Optional<TableHandle> tableHandle = runner.getMetadata().getTableHandle(session, tableName);
    assertTrue(tableHandle.isPresent());
    SplitSource splitSource = runner.getSplitManager().getSplits(session, tableHandle.get(), UNGROUPED_SCHEDULING, new IncompleteDynamicFilter(), alwaysTrue());
    List<Split> splits = new ArrayList<>();
    while (!splitSource.isFinished()) {
        splits.addAll(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000).get().getSplits());
    }
    splitSource.close();
    assertFalse(splits.isEmpty());
}
Also used : ArrayList(java.util.ArrayList) DistributedQueryRunner(io.trino.testing.DistributedQueryRunner) QueryRunner(io.trino.testing.QueryRunner) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) TransactionId(io.trino.transaction.TransactionId) TransactionManager(io.trino.transaction.TransactionManager) AllowAllAccessControl(io.trino.security.AllowAllAccessControl) TableHandle(io.trino.metadata.TableHandle) SplitSource(io.trino.split.SplitSource) Split(io.trino.metadata.Split) Session(io.trino.Session) Test(org.testng.annotations.Test)

Example 28 with TableHandle

use of io.trino.metadata.TableHandle in project trino by trinodb.

the class TestIcebergProjectionPushdownPlans method testDereferencePushdown.

@Test
public void testDereferencePushdown() {
    String testTable = "test_simple_projection_pushdown" + randomTableSuffix();
    QualifiedObjectName completeTableName = new QualifiedObjectName(CATALOG, SCHEMA, testTable);
    getQueryRunner().execute(format("CREATE TABLE %s (col0, col1) WITH (partitioning = ARRAY['col1']) AS" + " SELECT CAST(row(5, 6) AS row(x bigint, y bigint)) AS col0, 5 AS col1 WHERE false", testTable));
    Session session = getQueryRunner().getDefaultSession();
    Optional<TableHandle> tableHandle = getTableHandle(session, completeTableName);
    assertTrue(tableHandle.isPresent(), "expected the table handle to be present");
    Map<String, ColumnHandle> columns = getColumnHandles(session, completeTableName);
    IcebergColumnHandle column0Handle = (IcebergColumnHandle) columns.get("col0");
    IcebergColumnHandle column1Handle = (IcebergColumnHandle) columns.get("col1");
    IcebergColumnHandle columnX = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(0).getId()), BIGINT, Optional.empty());
    IcebergColumnHandle columnY = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(1).getId()), BIGINT, Optional.empty());
    // Simple Projection pushdown
    assertPlan("SELECT col0.x expr_x, col0.y expr_y FROM " + testTable, any(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(columnX, columnY))), TupleDomain.all(), ImmutableMap.of("col0#x", equalTo(columnX), "col0#y", equalTo(columnY)))));
    // Projection and predicate pushdown
    assertPlan(format("SELECT col0.x FROM %s WHERE col0.x = col1 + 3 and col0.y = 2", testTable), anyTree(filter("y = BIGINT '2' AND (x =  CAST((col1 + 3) AS BIGINT))", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column1Handle, columnX, columnY)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnY, Domain.singleValue(BIGINT, 2L))));
    }, TupleDomain.all(), ImmutableMap.of("y", equalTo(columnY), "x", equalTo(columnX), "col1", equalTo(column1Handle))))));
    // Projection and predicate pushdown with overlapping columns
    assertPlan(format("SELECT col0, col0.y expr_y FROM %s WHERE col0.x = 5", testTable), anyTree(filter("x = BIGINT '5'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column0Handle, columnX)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 5L))));
    }, TupleDomain.all(), ImmutableMap.of("col0", equalTo(column0Handle), "x", equalTo(columnX))))));
    // Projection and predicate pushdown with joins
    assertPlan(format("SELECT T.col0.x, T.col0, T.col0.y FROM %s T join %s S on T.col1 = S.col1 WHERE (T.col0.x = 2)", testTable, testTable), anyTree(project(ImmutableMap.of("expr_0_x", expression("expr_0[1]"), "expr_0", expression("expr_0"), "expr_0_y", expression("expr_0[2]")), join(INNER, ImmutableList.of(equiJoinClause("t_expr_1", "s_expr_1")), anyTree(filter("x = BIGINT '2'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        Set<IcebergColumnHandle> expectedProjections = ImmutableSet.of(column0Handle, column1Handle, columnX);
        TupleDomain<IcebergColumnHandle> expectedUnenforcedConstraint = TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 2L)));
        return icebergTableHandle.getProjectedColumns().equals(expectedProjections) && unenforcedConstraint.equals(expectedUnenforcedConstraint);
    }, TupleDomain.all(), ImmutableMap.of("x", equalTo(columnX), "expr_0", equalTo(column0Handle), "t_expr_1", equalTo(column1Handle))))), anyTree(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(column1Handle))), TupleDomain.all(), ImmutableMap.of("s_expr_1", equalTo(column1Handle))))))));
}
Also used : MoreFiles.deleteRecursively(com.google.common.io.MoreFiles.deleteRecursively) PlanMatchPattern.any(io.trino.sql.planner.assertions.PlanMatchPattern.any) INNER(io.trino.sql.planner.plan.JoinNode.Type.INNER) Database(io.trino.plugin.hive.metastore.Database) Test(org.testng.annotations.Test) PlanMatchPattern.filter(io.trino.sql.planner.assertions.PlanMatchPattern.filter) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) ImmutableList(com.google.common.collect.ImmutableList) Files(com.google.common.io.Files) BasePushdownPlanTest(io.trino.sql.planner.assertions.BasePushdownPlanTest) LocalQueryRunner(io.trino.testing.LocalQueryRunner) Map(java.util.Map) ColumnHandle(io.trino.spi.connector.ColumnHandle) FileHiveMetastore.createTestingFileHiveMetastore(io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore) PlanMatchPattern.equiJoinClause(io.trino.sql.planner.assertions.PlanMatchPattern.equiJoinClause) EMPTY_MODULE(com.google.inject.util.Modules.EMPTY_MODULE) PlanMatchPattern.join(io.trino.sql.planner.assertions.PlanMatchPattern.join) TestTable.randomTableSuffix(io.trino.testing.sql.TestTable.randomTableSuffix) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.expression(io.trino.sql.planner.assertions.PlanMatchPattern.expression) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) PrincipalType(io.trino.spi.security.PrincipalType) Domain(io.trino.spi.predicate.Domain) Set(java.util.Set) TupleDomain(io.trino.spi.predicate.TupleDomain) File(java.io.File) String.format(java.lang.String.format) Predicates.equalTo(com.google.common.base.Predicates.equalTo) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) TableHandle(io.trino.metadata.TableHandle) PlanMatchPattern.anyTree(io.trino.sql.planner.assertions.PlanMatchPattern.anyTree) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) BIGINT(io.trino.spi.type.BigintType.BIGINT) PlanMatchPattern.project(io.trino.sql.planner.assertions.PlanMatchPattern.project) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) PlanMatchPattern.tableScan(io.trino.sql.planner.assertions.PlanMatchPattern.tableScan) Session(io.trino.Session) ColumnHandle(io.trino.spi.connector.ColumnHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) TupleDomain(io.trino.spi.predicate.TupleDomain) TableHandle(io.trino.metadata.TableHandle) Session(io.trino.Session) Test(org.testng.annotations.Test) BasePushdownPlanTest(io.trino.sql.planner.assertions.BasePushdownPlanTest)

Example 29 with TableHandle

use of io.trino.metadata.TableHandle in project trino by trinodb.

the class TestMinimalFunctionality method testStreamExists.

@Test
public void testStreamExists() {
    QualifiedObjectName name = new QualifiedObjectName("kinesis", "default", streamName);
    transaction(queryRunner.getTransactionManager(), new AllowAllAccessControl()).singleStatement().execute(SESSION, session -> {
        Optional<TableHandle> handle = queryRunner.getServer().getMetadata().getTableHandle(session, name);
        assertTrue(handle.isPresent());
    });
}
Also used : AllowAllAccessControl(io.trino.security.AllowAllAccessControl) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Test(org.testng.annotations.Test)

Example 30 with TableHandle

use of io.trino.metadata.TableHandle in project trino by trinodb.

the class TestRecordAccess method testStreamExists.

@Test
public void testStreamExists() {
    QualifiedObjectName name = new QualifiedObjectName("kinesis", "default", dummyStreamName);
    transaction(queryRunner.getTransactionManager(), new AllowAllAccessControl()).singleStatement().execute(SESSION, session -> {
        Optional<TableHandle> handle = queryRunner.getServer().getMetadata().getTableHandle(session, name);
        assertTrue(handle.isPresent());
    });
    log.info("Completed first test (access table handle)");
}
Also used : AllowAllAccessControl(io.trino.security.AllowAllAccessControl) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Test(org.testng.annotations.Test)

Aggregations

TableHandle (io.trino.metadata.TableHandle)73 Session (io.trino.Session)37 QualifiedObjectName (io.trino.metadata.QualifiedObjectName)33 ColumnHandle (io.trino.spi.connector.ColumnHandle)31 ImmutableMap (com.google.common.collect.ImmutableMap)29 ImmutableList (com.google.common.collect.ImmutableList)28 Optional (java.util.Optional)26 Symbol (io.trino.sql.planner.Symbol)24 Test (org.testng.annotations.Test)24 CatalogName (io.trino.connector.CatalogName)23 Map (java.util.Map)21 TupleDomain (io.trino.spi.predicate.TupleDomain)18 List (java.util.List)17 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)16 BIGINT (io.trino.spi.type.BigintType.BIGINT)15 Type (io.trino.spi.type.Type)15 Assignments (io.trino.sql.planner.plan.Assignments)15 Expression (io.trino.sql.tree.Expression)15 ImmutableSet (com.google.common.collect.ImmutableSet)14 Metadata (io.trino.metadata.Metadata)14