Examples with HiveTableHandle io.trino.plugin.hive.HiveTableHandle used on opensource projects

Example 6 with HiveTableHandle

use of io.trino.plugin.hive.HiveTableHandle in project trino by trinodb.

the class TestHiveProjectionPushdownIntoTableScan method testDereferencePushdown.

@Test
public void testDereferencePushdown() {
    String testTable = "test_simple_projection_pushdown";
    QualifiedObjectName completeTableName = new QualifiedObjectName(HIVE_CATALOG_NAME, SCHEMA_NAME, testTable);
    getQueryRunner().execute(format("CREATE TABLE %s (col0, col1) AS" + " SELECT cast(row(5, 6) as row(x bigint, y bigint)) AS col0, 5 AS col1 WHERE false", testTable));
    Session session = getQueryRunner().getDefaultSession();
    Optional<TableHandle> tableHandle = getTableHandle(session, completeTableName);
    assertTrue(tableHandle.isPresent(), "expected the table handle to be present");
    Map<String, ColumnHandle> columns = getColumnHandles(session, completeTableName);
    HiveColumnHandle column0Handle = (HiveColumnHandle) columns.get("col0");
    HiveColumnHandle column1Handle = (HiveColumnHandle) columns.get("col1");
    HiveColumnHandle columnX = createProjectedColumnHandle(column0Handle, ImmutableList.of(0));
    HiveColumnHandle columnY = createProjectedColumnHandle(column0Handle, ImmutableList.of(1));
    // Simple Projection pushdown
    assertPlan("SELECT col0.x expr_x, col0.y expr_y FROM " + testTable, any(tableScan(((HiveTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(ImmutableSet.of(columnX, columnY))::equals, TupleDomain.all(), ImmutableMap.of("col0#x", columnX::equals, "col0#y", columnY::equals))));
    // Projection and predicate pushdown
    assertPlan(format("SELECT col0.x FROM %s WHERE col0.x = col1 + 3 and col0.y = 2", testTable), anyTree(filter("col0_y = BIGINT '2' AND (col0_x =  cast((col1 + 3) as BIGINT))", tableScan(table -> {
        HiveTableHandle hiveTableHandle = (HiveTableHandle) table;
        return hiveTableHandle.getCompactEffectivePredicate().equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnY, Domain.singleValue(BIGINT, 2L)))) && hiveTableHandle.getProjectedColumns().equals(ImmutableSet.of(column1Handle, columnX, columnY));
    }, TupleDomain.all(), ImmutableMap.of("col0_y", columnY::equals, "col0_x", columnX::equals, "col1", column1Handle::equals)))));
    // Projection and predicate pushdown with overlapping columns
    assertPlan(format("SELECT col0, col0.y expr_y FROM %s WHERE col0.x = 5", testTable), anyTree(filter("col0_x = BIGINT '5'", tableScan(table -> {
        HiveTableHandle hiveTableHandle = (HiveTableHandle) table;
        return hiveTableHandle.getCompactEffectivePredicate().equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 5L)))) && hiveTableHandle.getProjectedColumns().equals(ImmutableSet.of(column0Handle, columnX));
    }, TupleDomain.all(), ImmutableMap.of("col0", column0Handle::equals, "col0_x", columnX::equals)))));
    // Projection and predicate pushdown with joins
    assertPlan(format("SELECT T.col0.x, T.col0, T.col0.y FROM %s T join %s S on T.col1 = S.col1 WHERE (T.col0.x = 2)", testTable, testTable), anyTree(project(ImmutableMap.of("expr_0_x", expression("expr_0[1]"), "expr_0", expression("expr_0"), "expr_0_y", expression("expr_0[2]")), join(INNER, ImmutableList.of(equiJoinClause("t_expr_1", "s_expr_1")), anyTree(filter("expr_0_x = BIGINT '2'", tableScan(table -> ((HiveTableHandle) table).getCompactEffectivePredicate().getDomains().get().equals(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 2L))), TupleDomain.all(), ImmutableMap.of("expr_0_x", columnX::equals, "expr_0", column0Handle::equals, "t_expr_1", column1Handle::equals)))), anyTree(tableScan(((HiveTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(ImmutableSet.of(column1Handle))::equals, TupleDomain.all(), ImmutableMap.of("s_expr_1", column1Handle::equals)))))));
}

Example 7 with HiveTableHandle

use of io.trino.plugin.hive.HiveTableHandle in project trino by trinodb.

the class DropStatsProcedure method doDropStats.

private void doDropStats(ConnectorSession session, ConnectorAccessControl accessControl, String schema, String table, List<?> partitionValues) {
    TransactionalMetadata hiveMetadata = hiveMetadataFactory.create(session.getIdentity(), true);
    HiveTableHandle handle = (HiveTableHandle) hiveMetadata.getTableHandle(session, new SchemaTableName(schema, table));
    if (handle == null) {
        throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, format("Table '%s' does not exist", new SchemaTableName(schema, table)));
    }
    accessControl.checkCanInsertIntoTable(null, new SchemaTableName(schema, table));
    Map<String, ColumnHandle> columns = hiveMetadata.getColumnHandles(session, handle);
    List<String> partitionColumns = columns.values().stream().map(HiveColumnHandle.class::cast).filter(HiveColumnHandle::isPartitionKey).map(HiveColumnHandle::getName).collect(toImmutableList());
    HiveMetastoreClosure metastore = hiveMetadata.getMetastore().unsafeGetRawHiveMetastoreClosure();
    if (partitionValues != null) {
        // drop stats for specified partitions
        List<List<String>> partitionStringValues = partitionValues.stream().map(DropStatsProcedure::validateParameterType).collect(toImmutableList());
        validatePartitions(partitionStringValues, partitionColumns);
        partitionStringValues.forEach(values -> metastore.updatePartitionStatistics(schema, table, makePartName(partitionColumns, values), stats -> PartitionStatistics.empty()));
    } else {
        // no partition specified, so drop stats for the entire table
        if (partitionColumns.isEmpty()) {
            // for non-partitioned tables, just wipe table stats
            metastore.updateTableStatistics(schema, table, NO_ACID_TRANSACTION, stats -> PartitionStatistics.empty());
        } else {
            // the table is partitioned; remove stats for every partition
            metastore.getPartitionNamesByFilter(handle.getSchemaName(), handle.getTableName(), partitionColumns, TupleDomain.all()).ifPresent(partitions -> partitions.forEach(partitionName -> metastore.updatePartitionStatistics(schema, table, partitionName, stats -> PartitionStatistics.empty())));
        }
    }
    hiveMetadata.commit();
}

Example 8 with HiveTableHandle

use of io.trino.plugin.hive.HiveTableHandle in project trino by trinodb.

the class TestConnectorPushdownRulesWithHive method testPushdownWithDuplicateExpressions.

@Test
public void testPushdownWithDuplicateExpressions() {
    String tableName = "duplicate_expressions";
    tester().getQueryRunner().execute(format("CREATE TABLE  %s (struct_of_bigint, just_bigint) AS SELECT cast(row(5, 6) AS row(a bigint, b bigint)) AS struct_of_int, 5 AS just_bigint WHERE false", tableName));
    PushProjectionIntoTableScan pushProjectionIntoTableScan = new PushProjectionIntoTableScan(tester().getPlannerContext(), tester().getTypeAnalyzer(), new ScalarStatsCalculator(tester().getPlannerContext(), tester().getTypeAnalyzer()));
    HiveTableHandle hiveTable = new HiveTableHandle(SCHEMA_NAME, tableName, ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty());
    TableHandle table = new TableHandle(new CatalogName(HIVE_CATALOG_NAME), hiveTable, new HiveTransactionHandle(false));
    HiveColumnHandle bigintColumn = createBaseColumn("just_bigint", 1, toHiveType(BIGINT), BIGINT, REGULAR, Optional.empty());
    HiveColumnHandle partialColumn = new HiveColumnHandle("struct_of_bigint", 0, toHiveType(ROW_TYPE), ROW_TYPE, Optional.of(new HiveColumnProjectionInfo(ImmutableList.of(0), ImmutableList.of("a"), toHiveType(BIGINT), BIGINT)), REGULAR, Optional.empty());
    // Test projection pushdown with duplicate column references
    tester().assertThat(pushProjectionIntoTableScan).on(p -> {
        SymbolReference column = p.symbol("just_bigint", BIGINT).toSymbolReference();
        Expression negation = new ArithmeticUnaryExpression(MINUS, column);
        return p.project(Assignments.of(// The column reference is part of both the assignments
        p.symbol("column_ref", BIGINT), column, p.symbol("negated_column_ref", BIGINT), negation), p.tableScan(table, ImmutableList.of(p.symbol("just_bigint", BIGINT)), ImmutableMap.of(p.symbol("just_bigint", BIGINT), bigintColumn)));
    }).matches(project(ImmutableMap.of("column_ref", expression("just_bigint_0"), "negated_column_ref", expression("- just_bigint_0")), tableScan(hiveTable.withProjectedColumns(ImmutableSet.of(bigintColumn))::equals, TupleDomain.all(), ImmutableMap.of("just_bigint_0", bigintColumn::equals))));
    // Test Dereference pushdown
    tester().assertThat(pushProjectionIntoTableScan).on(p -> {
        SubscriptExpression subscript = new SubscriptExpression(p.symbol("struct_of_bigint", ROW_TYPE).toSymbolReference(), new LongLiteral("1"));
        Expression sum = new ArithmeticBinaryExpression(ADD, subscript, new LongLiteral("2"));
        return p.project(Assignments.of(// The subscript expression instance is part of both the assignments
        p.symbol("expr_deref", BIGINT), subscript, p.symbol("expr_deref_2", BIGINT), sum), p.tableScan(table, ImmutableList.of(p.symbol("struct_of_bigint", ROW_TYPE)), ImmutableMap.of(p.symbol("struct_of_bigint", ROW_TYPE), partialColumn.getBaseColumn())));
    }).matches(project(ImmutableMap.of("expr_deref", expression(new SymbolReference("struct_of_bigint#a")), "expr_deref_2", expression(new ArithmeticBinaryExpression(ADD, new SymbolReference("struct_of_bigint#a"), new LongLiteral("2")))), tableScan(hiveTable.withProjectedColumns(ImmutableSet.of(partialColumn))::equals, TupleDomain.all(), ImmutableMap.of("struct_of_bigint#a", partialColumn::equals))));
    metastore.dropTable(SCHEMA_NAME, tableName, true);
}

Example 9 with HiveTableHandle

use of io.trino.plugin.hive.HiveTableHandle in project trino by trinodb.

the class TestConnectorPushdownRulesWithHive method testProjectionPushdown.

@Test
public void testProjectionPushdown() {
    String tableName = "projection_test";
    PushProjectionIntoTableScan pushProjectionIntoTableScan = new PushProjectionIntoTableScan(tester().getPlannerContext(), tester().getTypeAnalyzer(), new ScalarStatsCalculator(tester().getPlannerContext(), tester().getTypeAnalyzer()));
    tester().getQueryRunner().execute(format("CREATE TABLE  %s (struct_of_int) AS " + "SELECT cast(row(5, 6) as row(a bigint, b bigint)) as struct_of_int where false", tableName));
    Type baseType = ROW_TYPE;
    HiveColumnHandle partialColumn = new HiveColumnHandle("struct_of_int", 0, toHiveType(baseType), baseType, Optional.of(new HiveColumnProjectionInfo(ImmutableList.of(0), ImmutableList.of("a"), toHiveType(BIGINT), BIGINT)), REGULAR, Optional.empty());
    HiveTableHandle hiveTable = new HiveTableHandle(SCHEMA_NAME, tableName, ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty());
    TableHandle table = new TableHandle(new CatalogName(HIVE_CATALOG_NAME), hiveTable, new HiveTransactionHandle(false));
    HiveColumnHandle fullColumn = partialColumn.getBaseColumn();
    // Test projected columns pushdown to HiveTableHandle in case of full column references
    tester().assertThat(pushProjectionIntoTableScan).on(p -> p.project(Assignments.of(p.symbol("struct_of_int", baseType), p.symbol("struct_of_int", baseType).toSymbolReference()), p.tableScan(table, ImmutableList.of(p.symbol("struct_of_int", baseType)), ImmutableMap.of(p.symbol("struct_of_int", baseType), fullColumn)))).matches(project(ImmutableMap.of("expr", expression("col")), tableScan(hiveTable.withProjectedColumns(ImmutableSet.of(fullColumn))::equals, TupleDomain.all(), ImmutableMap.of("col", fullColumn::equals))));
    // Rule should return Optional.empty after projected ColumnHandles have been added to HiveTableHandle
    tester().assertThat(pushProjectionIntoTableScan).on(p -> p.project(Assignments.of(p.symbol("struct_of_int", baseType), p.symbol("struct_of_int", baseType).toSymbolReference()), p.tableScan(new TableHandle(new CatalogName(HIVE_CATALOG_NAME), hiveTable.withProjectedColumns(ImmutableSet.of(fullColumn)), new HiveTransactionHandle(false)), ImmutableList.of(p.symbol("struct_of_int", baseType)), ImmutableMap.of(p.symbol("struct_of_int", baseType), fullColumn)))).doesNotFire();
    // Test Dereference pushdown
    tester().assertThat(pushProjectionIntoTableScan).on(p -> p.project(Assignments.of(p.symbol("expr_deref", BIGINT), new SubscriptExpression(p.symbol("struct_of_int", baseType).toSymbolReference(), new LongLiteral("1"))), p.tableScan(table, ImmutableList.of(p.symbol("struct_of_int", baseType)), ImmutableMap.of(p.symbol("struct_of_int", baseType), fullColumn)))).matches(project(ImmutableMap.of("expr_deref", expression(new SymbolReference("struct_of_int#a"))), tableScan(hiveTable.withProjectedColumns(ImmutableSet.of(partialColumn))::equals, TupleDomain.all(), ImmutableMap.of("struct_of_int#a", partialColumn::equals))));
    metastore.dropTable(SCHEMA_NAME, tableName, true);
}