Examples with TableMetadata io.prestosql.metadata.TableMetadata used on opensource projects

Example 26 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class CubeOptimizer method parseNodeRecursively.

private void parseNodeRecursively(PlanNode node) {
    if (node instanceof ProjectNode) {
        ProjectNode projection = ((ProjectNode) node);
        validateProjection(projection);
        parseNodeRecursively(projection.getSource());
        handleProjection(projection);
    } else if (node instanceof JoinNode) {
        JoinNode localJoinNode = (JoinNode) node;
        parseNodeRecursively(localJoinNode.getLeft());
        parseNodeRecursively(localJoinNode.getRight());
        localJoinNode.getOutputSymbols().stream().map(Symbol::getName).forEach(symbol -> originalPlanMappings.put(symbol, originalPlanMappings.get(symbol)));
        localJoinNode.getCriteria().forEach(equiJoinClause -> {
            // Join condition(s) must be defined on column from Source table
            ColumnWithTable leftColumn = originalPlanMappings.get(equiJoinClause.getLeft().getName());
            ColumnWithTable rightColumn = originalPlanMappings.get(equiJoinClause.getRight().getName());
            ColumnWithTable sourceTableColumn = leftColumn.getFQTableName().equalsIgnoreCase(sourceTableName) ? leftColumn : rightColumn;
            dimensions.add(sourceTableColumn.getColumnName());
            groupBy.add(sourceTableColumn.getColumnName());
            matchingMetadataList.removeIf(metadata -> {
                // Retain Cube metadata, only if one of the join column is part of Cube
                return !metadata.getDimensions().contains(sourceTableColumn.getColumnName());
            });
        });
    } else if (node instanceof TableScanNode) {
        TableScanNode scanNode = (TableScanNode) node;
        TableMetadata tableMetadata = metadata.getTableMetadata(context.getSession(), scanNode.getTable());
        scanNode.getOutputSymbols().forEach(output -> {
            ColumnWithTable columnWithTable = new ColumnWithTable(tableMetadata.getQualifiedName().toString(), scanNode.getAssignments().get(output).getColumnName());
            originalPlanMappings.put(output.getName(), columnWithTable);
        });
        // Assumption: Cubes are defined on only of the tables involved in Join. That table will be considered Source Table.
        List<CubeMetadata> metadataList = cubeMetaStore.getMetadataList(tableMetadata.getQualifiedName().toString());
        if (sourceTableScanNode == null && !metadataList.isEmpty()) {
            sourceTableScanNode = scanNode;
            sourceTableMetadata = tableMetadata;
            sourceTableName = sourceTableMetadata.getQualifiedName().toString();
            sourceTableHandle = sourceTableScanNode.getTable();
            matchingMetadataList.addAll(metadataList);
            sourceTableColumnMap.putAll(metadata.getColumnHandles(context.getSession(), sourceTableHandle));
        }
    } else {
        throw new UnsupportedOperationException("Unexpected plan node. Expected TableScan, JoinNode or ProjectNode. Actual is " + node.getClass());
    }
}

Example 27 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class StarTreeAggregationRule method optimize.

public Result optimize(AggregationNode aggregationNode, final PlanNode filterNode, TableScanNode tableScanNode, Map<String, Object> symbolMapping, Session session, PlanSymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
    TableHandle tableHandle = tableScanNode.getTable();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
    String tableName = tableMetadata.getQualifiedName().toString();
    CubeStatement statement = CubeStatementGenerator.generate(tableName, aggregationNode, symbolMapping);
    // Don't use star-tree for non-aggregate queries
    if (statement.getAggregations().isEmpty()) {
        return Result.empty();
    }
    boolean hasDistinct = statement.getAggregations().stream().anyMatch(AggregationSignature::isDistinct);
    // Since cube is pre-aggregated, utilising it for such queries could return incorrect result
    if (aggregationNode.hasEmptyGroupingSet() && hasDistinct) {
        return Result.empty();
    }
    List<CubeMetadata> cubeMetadataList = CubeMetadata.filter(this.cubeMetaStore.getMetadataList(statement.getFrom()), statement);
    // Compare FilterNode predicate with Cube predicates to evaluate which cube can be used.
    List<CubeMetadata> matchedCubeMetadataList = cubeMetadataList.stream().filter(cubeMetadata -> filterPredicateMatches((FilterNode) filterNode, cubeMetadata, session, symbolAllocator.getTypes())).collect(Collectors.toList());
    // Match based on filter conditions
    if (matchedCubeMetadataList.isEmpty()) {
        return Result.empty();
    }
    LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
    if (lastModifiedTimeSupplier == null) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
        return Result.empty();
    }
    // Filter out cubes that were created before the source table was updated
    long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
    // There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
    if (lastModifiedTime == -1L) {
        return Result.empty();
    }
    matchedCubeMetadataList = matchedCubeMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
    if (matchedCubeMetadataList.isEmpty()) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
        return Result.empty();
    }
    // If multiple cubes are matching then lets select the recent built cube
    // so sort the cube based on the last updated time stamp
    matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
    CubeMetadata matchedCubeMetadata = matchedCubeMetadataList.get(0);
    AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadata);
    return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, rewriteByRemovingSourceFilter(filterNode, matchedCubeMetadata)));
}

Example 28 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project boostkit-bigdata by kunpengcompute.

the class TestHiveIntegrationSmokeTest method testFullUnifyVacuum2.

@Test
public void testFullUnifyVacuum2() {
    String table = "tab_fm_vacuum_2";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc')", schema, table));
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (3, 4)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (5, 6)", schema, table), 1);
    assertUpdate(String.format("VACUUM TABLE %s.%s AND WAIT", schema, table), 3);
    assertUpdate(String.format("VACUUM TABLE %s.%s FULL UNIFY AND WAIT", schema, table), 3);
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = ((String) tableMetadata.getMetadata().getProperties().get("location"));
    assertFilesAfterCleanup(tablePath, 1);
    assertUpdate(String.format("DROP TABLE %s.%s", schema, table));
}

Example 29 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project boostkit-bigdata by kunpengcompute.

the class TestHiveIntegrationSmokeTest method testInsert.

private void testInsert(Session session, HiveStorageFormat storageFormat) {
    @Language("SQL") String createTable = "" + "CREATE TABLE test_insert_format_table " + "(" + "  _string VARCHAR," + "  _varchar VARCHAR(65535)," + "  _char CHAR(10)," + "  _bigint BIGINT," + "  _integer INTEGER," + "  _smallint SMALLINT," + "  _tinyint TINYINT," + "  _real REAL," + "  _double DOUBLE," + "  _boolean BOOLEAN," + "  _decimal_short DECIMAL(3,2)," + "  _decimal_long DECIMAL(30,10)" + ") " + "WITH (format = '" + storageFormat + "') ";
    if (storageFormat == HiveStorageFormat.AVRO) {
        createTable = createTable.replace(" _smallint SMALLINT,", " _smallint INTEGER,");
        createTable = createTable.replace(" _tinyint TINYINT,", " _tinyint INTEGER,");
    }
    assertUpdate(session, createTable);
    TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_insert_format_table");
    assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
    assertColumnType(tableMetadata, "_string", createUnboundedVarcharType());
    assertColumnType(tableMetadata, "_varchar", createVarcharType(65535));
    assertColumnType(tableMetadata, "_char", createCharType(10));
    @Language("SQL") String select = "SELECT" + " 'foo' _string" + ", 'bar' _varchar" + ", CAST('boo' AS CHAR(10)) _char" + ", 1 _bigint" + ", CAST(42 AS INTEGER) _integer" + ", CAST(43 AS SMALLINT) _smallint" + ", CAST(44 AS TINYINT) _tinyint" + ", CAST('123.45' AS REAL) _real" + ", CAST('3.14' AS DOUBLE) _double" + ", true _boolean" + ", CAST('3.14' AS DECIMAL(3,2)) _decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _decimal_long";
    if (storageFormat == HiveStorageFormat.AVRO) {
        select = select.replace(" CAST (43 AS SMALLINT) _smallint,", " 3 _smallint,");
        select = select.replace(" CAST (44 AS TINYINT) _tinyint,", " 4 _tinyint,");
    }
    assertUpdate(session, "INSERT INTO test_insert_format_table " + select, 1);
    assertQuery(session, "SELECT * from test_insert_format_table", select);
    assertUpdate(session, "INSERT INTO test_insert_format_table (_tinyint, _smallint, _integer, _bigint, _real, _double) SELECT CAST(1 AS TINYINT), CAST(2 AS SMALLINT), 3, 4, cast(14.3E0 as REAL), 14.3E0", 1);
    assertQuery(session, "SELECT * from test_insert_format_table where _bigint = 4", "SELECT null, null, null, 4, 3, 2, 1, 14.3, 14.3, null, null, null");
    assertQuery(session, "SELECT * from test_insert_format_table where _real = CAST(14.3 as REAL)", "SELECT null, null, null, 4, 3, 2, 1, 14.3, 14.3, null, null, null");
    assertUpdate(session, "INSERT INTO test_insert_format_table (_double, _bigint) SELECT 2.72E0, 3", 1);
    assertQuery(session, "SELECT * from test_insert_format_table where _bigint = 3", "SELECT null, null, null, 3, null, null, null, null, 2.72, null, null, null");
    assertUpdate(session, "INSERT INTO test_insert_format_table (_decimal_short, _decimal_long) SELECT DECIMAL '2.72', DECIMAL '98765432101234567890.0123456789'", 1);
    assertQuery(session, "SELECT * from test_insert_format_table where _decimal_long = DECIMAL '98765432101234567890.0123456789'", "SELECT null, null, null, null, null, null, null, null, null, null, 2.72, 98765432101234567890.0123456789");
    assertUpdate(session, "DROP TABLE test_insert_format_table");
    assertFalse(getQueryRunner().tableExists(session, "test_insert_format_table"));
}

Example 30 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project boostkit-bigdata by kunpengcompute.

the class TestHiveIntegrationSmokeTest method testVacuumOnPartitionedTable2.

@Test
public void testVacuumOnPartitionedTable2() {
    String table = "tab6";
    String schema = "default";
    assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
    String partitionedColumn = "b";
    assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc', partitioned_by=Array['%s'])", schema, table, partitionedColumn));
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 1)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 1)", schema, table), 1);
    assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 2)", schema, table), 1);
    assertUpdate(String.format("VACUUM TABLE %s.%s PARTITION '%s=1' AND WAIT", schema, table, partitionedColumn), 2);
    TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
    String tablePath = (String) tableMetadata.getMetadata().getProperties().get("location");
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1"), 1);
    assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("2"), 2);
}