Examples with CubeMetadata io.hetu.core.spi.cube.CubeMetadata used on opensource projects

Example 6 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class CubeOptimizer method optimize.

public Optional<PlanNode> optimize() {
    parse();
    // Match based on filter conditions
    if (matchingMetadataList.isEmpty()) {
        return Optional.empty();
    }
    LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(context.getSession(), sourceTableHandle);
    if (lastModifiedTimeSupplier == null) {
        context.getWarningCollector().add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + sourceTableMetadata.getTable().toString() + ". Ignoring star tree cubes."));
        return Optional.empty();
    }
    // Filter out cubes that were created before the source table was updated
    long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
    // There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
    if (lastModifiedTime == -1L) {
        return Optional.empty();
    }
    matchingMetadataList = matchingMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
    if (matchingMetadataList.isEmpty()) {
        context.getWarningCollector().add(new PrestoWarning(EXPIRED_CUBE, sourceTableMetadata.getTable().toString() + " has been modified after creating cubes. Ignoring expired cubes."));
        return Optional.empty();
    }
    // If multiple matching cubes found, then cube with fewer group by columns (could also mean least number of rows) and pick the most recent cube
    Comparator<CubeMetadata> byGroupSize = Comparator.comparingInt(cm -> cm.getGroup().size());
    Comparator<CubeMetadata> byLastModifiedTime = Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed();
    matchingMetadataList.sort(byGroupSize.thenComparing(byLastModifiedTime));
    cubeMetadata = matchingMetadataList.get(0);
    return Optional.ofNullable(rewrite());
}

Example 7 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class CubeOptimizer method parseNodeRecursively.

private void parseNodeRecursively(PlanNode node) {
    if (node instanceof ProjectNode) {
        ProjectNode projection = ((ProjectNode) node);
        validateProjection(projection);
        parseNodeRecursively(projection.getSource());
        handleProjection(projection);
    } else if (node instanceof JoinNode) {
        JoinNode localJoinNode = (JoinNode) node;
        parseNodeRecursively(localJoinNode.getLeft());
        parseNodeRecursively(localJoinNode.getRight());
        localJoinNode.getOutputSymbols().stream().map(Symbol::getName).forEach(symbol -> originalPlanMappings.put(symbol, originalPlanMappings.get(symbol)));
        localJoinNode.getCriteria().forEach(equiJoinClause -> {
            // Join condition(s) must be defined on column from Source table
            ColumnWithTable leftColumn = originalPlanMappings.get(equiJoinClause.getLeft().getName());
            ColumnWithTable rightColumn = originalPlanMappings.get(equiJoinClause.getRight().getName());
            ColumnWithTable sourceTableColumn = leftColumn.getFQTableName().equalsIgnoreCase(sourceTableName) ? leftColumn : rightColumn;
            dimensions.add(sourceTableColumn.getColumnName());
            groupBy.add(sourceTableColumn.getColumnName());
            matchingMetadataList.removeIf(metadata -> {
                // Retain Cube metadata, only if one of the join column is part of Cube
                return !metadata.getDimensions().contains(sourceTableColumn.getColumnName());
            });
        });
    } else if (node instanceof TableScanNode) {
        TableScanNode scanNode = (TableScanNode) node;
        TableMetadata tableMetadata = metadata.getTableMetadata(context.getSession(), scanNode.getTable());
        scanNode.getOutputSymbols().forEach(output -> {
            ColumnWithTable columnWithTable = new ColumnWithTable(tableMetadata.getQualifiedName().toString(), scanNode.getAssignments().get(output).getColumnName());
            originalPlanMappings.put(output.getName(), columnWithTable);
        });
        // Assumption: Cubes are defined on only of the tables involved in Join. That table will be considered Source Table.
        List<CubeMetadata> metadataList = cubeMetaStore.getMetadataList(tableMetadata.getQualifiedName().toString());
        if (sourceTableScanNode == null && !metadataList.isEmpty()) {
            sourceTableScanNode = scanNode;
            sourceTableMetadata = tableMetadata;
            sourceTableName = sourceTableMetadata.getQualifiedName().toString();
            sourceTableHandle = sourceTableScanNode.getTable();
            matchingMetadataList.addAll(metadataList);
            sourceTableColumnMap.putAll(metadata.getColumnHandles(context.getSession(), sourceTableHandle));
        }
    } else {
        throw new UnsupportedOperationException("Unexpected plan node. Expected TableScan, JoinNode or ProjectNode. Actual is " + node.getClass());
    }
}

Example 8 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class StarTreeAggregationRule method optimize.

public Result optimize(AggregationNode aggregationNode, final PlanNode filterNode, TableScanNode tableScanNode, Map<String, Object> symbolMapping, Session session, PlanSymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
    TableHandle tableHandle = tableScanNode.getTable();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
    String tableName = tableMetadata.getQualifiedName().toString();
    CubeStatement statement = CubeStatementGenerator.generate(tableName, aggregationNode, symbolMapping);
    // Don't use star-tree for non-aggregate queries
    if (statement.getAggregations().isEmpty()) {
        return Result.empty();
    }
    boolean hasDistinct = statement.getAggregations().stream().anyMatch(AggregationSignature::isDistinct);
    // Since cube is pre-aggregated, utilising it for such queries could return incorrect result
    if (aggregationNode.hasEmptyGroupingSet() && hasDistinct) {
        return Result.empty();
    }
    List<CubeMetadata> cubeMetadataList = CubeMetadata.filter(this.cubeMetaStore.getMetadataList(statement.getFrom()), statement);
    // Compare FilterNode predicate with Cube predicates to evaluate which cube can be used.
    List<CubeMetadata> matchedCubeMetadataList = cubeMetadataList.stream().filter(cubeMetadata -> filterPredicateMatches((FilterNode) filterNode, cubeMetadata, session, symbolAllocator.getTypes())).collect(Collectors.toList());
    // Match based on filter conditions
    if (matchedCubeMetadataList.isEmpty()) {
        return Result.empty();
    }
    LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
    if (lastModifiedTimeSupplier == null) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
        return Result.empty();
    }
    // Filter out cubes that were created before the source table was updated
    long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
    // There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
    if (lastModifiedTime == -1L) {
        return Result.empty();
    }
    matchedCubeMetadataList = matchedCubeMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
    if (matchedCubeMetadataList.isEmpty()) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
        return Result.empty();
    }
    // If multiple cubes are matching then lets select the recent built cube
    // so sort the cube based on the last updated time stamp
    matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
    CubeMetadata matchedCubeMetadata = matchedCubeMetadataList.get(0);
    AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadata);
    return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, rewriteByRemovingSourceFilter(filterNode, matchedCubeMetadata)));
}

Example 9 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class DropCubeTask method execute.

@Override
public ListenableFuture<?> execute(DropCube statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters, HeuristicIndexerManager heuristicIndexerManager) {
    Session session = stateMachine.getSession();
    Optional<CubeMetaStore> optionalCubeMetaStore = this.cubeManager.getMetaStore(STAR_TREE);
    if (!optionalCubeMetaStore.isPresent()) {
        throw new RuntimeException("HetuMetastore is not initialized");
    }
    CubeMetaStore cubeMetaStore = optionalCubeMetaStore.get();
    QualifiedObjectName fullObjectName = createQualifiedObjectName(session, statement, statement.getCubeName());
    QualifiedName cubeTableName = QualifiedName.of(fullObjectName.getCatalogName(), fullObjectName.getSchemaName(), fullObjectName.getObjectName());
    try {
        Optional<CubeMetadata> matchedCube = cubeMetaStore.getMetadataFromCubeName(cubeTableName.toString());
        if (!matchedCube.isPresent()) {
            if (!statement.isExists()) {
                throw new SemanticException(MISSING_CUBE, statement, "Cube '%s' does not exist", cubeTableName);
            }
            return immediateFuture(null);
        }
        Optional<TableHandle> tableHandle = metadata.getTableHandle(session, fullObjectName);
        tableHandle.ifPresent(handle -> {
            accessControl.checkCanDropTable(session.getRequiredTransactionId(), session.getIdentity(), fullObjectName);
            metadata.dropTable(session, handle);
        });
        cubeMetaStore.removeCube(matchedCube.get());
        return immediateFuture(null);
    } catch (TableNotFoundException s) {
        throw new SemanticException(MISSING_CUBE, statement, "Cube '%s' is not Found", cubeTableName.toString());
    }
}

Example 10 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class TestStarTreeAggregationRule method setupBeforeClass.

@BeforeClass
public void setupBeforeClass() {
    PlanSymbolAllocator symbolAllocator = new PlanSymbolAllocator();
    columnOrderkey = symbolAllocator.newSymbol("orderkey", BIGINT);
    columnOrderDate = symbolAllocator.newSymbol("orderdate", DATE);
    columnCustkey = symbolAllocator.newSymbol("custkey", BIGINT);
    columnTotalprice = symbolAllocator.newSymbol("totalprice", DOUBLE);
    orderkeyHandle = new TpchColumnHandle("orderkey", BIGINT);
    orderdateHandle = new TpchColumnHandle("orderdate", DATE);
    custkeyHandle = new TpchColumnHandle("custkey", BIGINT);
    totalpriceHandle = new TpchColumnHandle("totalprice", DOUBLE);
    ColumnMetadata orderKeyColumnMetadata = new ColumnMetadata(orderkeyHandle.getColumnName(), orderkeyHandle.getType());
    ColumnMetadata orderDateColumnMetadata = new ColumnMetadata(orderdateHandle.getColumnName(), orderdateHandle.getType());
    ColumnMetadata custKeyColumnMetadata = new ColumnMetadata(custkeyHandle.getColumnName(), custkeyHandle.getType());
    ColumnMetadata totalPriceColumnMetadata = new ColumnMetadata(totalpriceHandle.getColumnName(), totalpriceHandle.getType());
    output = symbolAllocator.newSymbol("output", DOUBLE);
    assignments = ImmutableMap.<Symbol, ColumnHandle>builder().put(columnOrderkey, orderkeyHandle).put(columnOrderDate, orderdateHandle).put(columnCustkey, custkeyHandle).put(columnTotalprice, totalpriceHandle).build();
    TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
    ordersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
    baseTableScan = new TableScanNode(newId(), ordersTableHandle, ImmutableList.copyOf(assignments.keySet()), assignments, TupleDomain.all(), Optional.empty(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
    QualifiedObjectName baseTableName = QualifiedObjectName.valueOf(baseTableScan.getTable().getFullyQualifiedName());
    baseTableMetadata = new TableMetadata(ordersTableHandle.getCatalogName(), new ConnectorTableMetadata(new SchemaTableName(baseTableName.getSchemaName(), baseTableName.getObjectName()), Arrays.asList(orderKeyColumnMetadata, orderDateColumnMetadata, custKeyColumnMetadata, totalPriceColumnMetadata)));
    columnCountAll = symbolAllocator.newSymbol("count_all", BIGINT);
    columnSumTotalPrice = symbolAllocator.newSymbol("sum_totalprice", DOUBLE);
    columnCountOrderKey = symbolAllocator.newSymbol("count_orderkey", BIGINT);
    columnGroupingBitSet = symbolAllocator.newSymbol("grouping_bit_set", BIGINT);
    cubeColumnCustKey = symbolAllocator.newSymbol("custkey", BIGINT);
    cubeColumnOrderDate = symbolAllocator.newSymbol("orderdate", DATE);
    countAllHandle = new TpchColumnHandle("count_all", BIGINT);
    sumTotalPriceHandle = new TpchColumnHandle("sum_totalprice", DOUBLE);
    countOrderKeyHandle = new TpchColumnHandle("count_orderkey", BIGINT);
    groupingBitSetHandle = new TpchColumnHandle("grouping_bit_set", BIGINT);
    custKeyCubeColumnHandle = new TpchColumnHandle("custkey", BIGINT);
    orderDateCubeColumnHandle = new TpchColumnHandle("orderdate", DATE);
    ordersCubeColumnHandles.put(countAllHandle.getColumnName(), countAllHandle);
    ordersCubeColumnHandles.put(sumTotalPriceHandle.getColumnName(), sumTotalPriceHandle);
    ordersCubeColumnHandles.put(countOrderKeyHandle.getColumnName(), countOrderKeyHandle);
    ordersCubeColumnHandles.put(groupingBitSetHandle.getColumnName(), groupingBitSetHandle);
    ordersCubeColumnHandles.put(custKeyCubeColumnHandle.getColumnName(), custKeyCubeColumnHandle);
    ordersCubeColumnHandles.put(orderDateCubeColumnHandle.getColumnName(), orderDateCubeColumnHandle);
    TpchTableHandle ordersCube = new TpchTableHandle("orders_cube", 1.0);
    ordersCubeHandle = new TableHandle(tester().getCurrentConnectorId(), ordersCube, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(ordersCube, TupleDomain.all())));
    countAllColumnMetadata = new ColumnMetadata(countAllHandle.getColumnName(), countAllHandle.getType());
    sumTotalPriceColumnMetadata = new ColumnMetadata(sumTotalPriceHandle.getColumnName(), sumTotalPriceHandle.getType());
    countOrderKeyColumnMetadata = new ColumnMetadata(countOrderKeyHandle.getColumnName(), countOrderKeyHandle.getType());
    groupingBitSetColumnMetadata = new ColumnMetadata(groupingBitSetHandle.getColumnName(), groupingBitSetHandle.getType());
    custKeyCubeColumnMetadata = new ColumnMetadata(custKeyCubeColumnHandle.getColumnName(), custKeyCubeColumnHandle.getType());
    orderDateCubeColumnMetadata = new ColumnMetadata(orderDateCubeColumnHandle.getColumnName(), orderDateCubeColumnHandle.getType());
    config = new FeaturesConfig();
    config.setEnableStarTreeIndex(true);
    cubeManager = Mockito.mock(CubeManager.class);
    provider = Mockito.mock(CubeProvider.class);
    cubeMetaStore = Mockito.mock(CubeMetaStore.class);
    cubeMetadata = Mockito.mock(CubeMetadata.class);
    ordersTableHandleMatcher = new BaseMatcher<TableHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof TableHandle)) {
                return false;
            }
            TableHandle th = (TableHandle) o;
            return th.getFullyQualifiedName().equals(ordersTableHandle.getFullyQualifiedName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
    ordersCubeHandleMatcher = new BaseMatcher<TableHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof TableHandle)) {
                return false;
            }
            TableHandle th = (TableHandle) o;
            return th.getFullyQualifiedName().equals(ordersCubeHandle.getFullyQualifiedName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
    countAllColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public void describeTo(Description description) {
        }

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(countAllHandle.getColumnName());
        }
    };
    sumTotalPriceColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public void describeTo(Description description) {
        }

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(sumTotalPriceHandle.getColumnName());
        }
    };
    countOrderKeyColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(countOrderKeyHandle.getColumnName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
    groupingBitSetColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(groupingBitSetHandle.getColumnName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
    orderDateCubeColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(orderDateCubeColumnHandle.getColumnName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
    custKeyCubeColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {

        @Override
        public boolean matches(Object o) {
            if (!(o instanceof ColumnHandle)) {
                return false;
            }
            ColumnHandle ch = (ColumnHandle) o;
            return ch.getColumnName().equalsIgnoreCase(custKeyCubeColumnHandle.getColumnName());
        }

        @Override
        public void describeTo(Description description) {
        }
    };
}