Examples with CubeMetadata io.hetu.core.spi.cube.CubeMetadata used on opensource projects

Example 1 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class TestStarTreeAggregationRule method testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated.

@Test
public void testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated() {
    Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
    Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
    Metadata metadata = Mockito.mock(Metadata.class);
    TableMetadata ordersTableMetadata = Mockito.mock(TableMetadata.class);
    QualifiedObjectName objectName = new QualifiedObjectName("local", "sf1.0", "orders");
    Mockito.when(metadata.getTableHandle(any(Session.class), eq(objectName))).thenReturn(Optional.of(ordersTableHandle));
    Mockito.when(metadata.getTableLastModifiedTimeSupplier(any(Session.class), any(TableHandle.class))).thenReturn(() -> DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-02 12:00:00"));
    Mockito.when(metadata.getTableMetadata(any(Session.class), eq(ordersTableHandle))).thenReturn(ordersTableMetadata);
    Mockito.when(ordersTableMetadata.getQualifiedName()).thenReturn(objectName);
    List<CubeMetadata> metadataList = ImmutableList.of(cubeMetadata);
    Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(metadataList));
    Mockito.when(cubeMetadata.matches(any(CubeStatement.class))).thenReturn(true);
    Mockito.when(cubeMetadata.getLastUpdatedTime()).thenReturn(DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-01 12:00:00"));
    StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, metadata);
    tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "true").on(p -> p.aggregation(builder -> builder.step(SINGLE).addAggregation(new Symbol("count_orderkey"), PlanBuilder.expression("count(orderkey)"), ImmutableList.of(BIGINT)).singleGroupingSet(new Symbol("orderdate")).source(p.project(Assignments.builder().put(p.symbol("orderdate", DATE), p.variable("orderdate", DATE)).put(p.symbol("orderkey", BIGINT), p.variable("orderkey", BIGINT)).build(), p.tableScan(ordersTableHandle, ImmutableList.of(p.symbol("orderdate", DATE), p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT), p.symbol("orderdate", DATE), new TpchColumnHandle("orderdate", DATE))))))).doesNotFire();
    Mockito.verify(cubeMetaStore, Mockito.atLeastOnce()).getMetadataList(eq("local.sf1.0.orders"));
    Mockito.verify(cubeMetadata, Mockito.atLeastOnce()).matches(any(CubeStatement.class));
}

Example 2 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class CubeFinishOperator method getOutput.

@Override
public Page getOutput() {
    if (state != State.HAS_OUTPUT) {
        return null;
    }
    CubeMetadata cubeMetadata = cubeMetastore.getMetadataFromCubeName(updateMetadata.getCubeName()).get();
    CubeMetadataBuilder builder = cubeMetastore.getBuilder(cubeMetadata);
    builder.withCubeFilter(mergePredicates(cubeMetadata.getCubeFilter(), updateMetadata.getDataPredicateString()));
    builder.setTableLastUpdatedTime(updateMetadata.getTableLastUpdatedTime());
    builder.setCubeLastUpdatedTime(System.currentTimeMillis());
    builder.setCubeStatus(READY);
    cubeMetastore.persist(builder.build());
    state = State.FINISHED;
    return page;
}

Example 3 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createInsertCubePlan.

private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
    Analysis.CubeInsert insert = analysis.getCubeInsert().get();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
    List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
    List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
    Assignments.Builder assignments = Assignments.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        int index = insert.getColumns().indexOf(columns.get(column.getName()));
        if (index < 0) {
            Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        } else {
            Symbol input = plan.getSymbol(index);
            Type tableType = column.getType();
            Type queryType = planSymbolAllocator.getTypes().get(input);
            if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, castToRowExpression(toSymbolReference(input)));
            } else {
                Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
                assignments.put(output, castToRowExpression(cast));
            }
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
    Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
    String catalogName = insert.getTarget().getCatalogName().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
    Expression rewritten = null;
    Set<Identifier> predicateColumns = new HashSet<>();
    if (insertCubeStatement.getWhere().isPresent()) {
        rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
        predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
    }
    CubeMetadata cubeMetadata = insert.getMetadata();
    if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
        // Means data some data was inserted before, but trying to insert entire dataset
        throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
    } else if (insertCubeStatement.getWhere().isPresent()) {
        if (!canSupportPredicate(rewritten)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
        }
        if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
        }
    }
    TableHandle sourceTableHandle = insert.getSourceTable();
    // At this point it has been verified that source table has not been updated
    // so insert into cube should be allowed
    LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
    checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
    Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
    CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
    return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}

Example 4 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class StarTreeMetaStore method convertTablesToMetadata.

private List<CubeMetadata> convertTablesToMetadata(List<TableEntity> tableEntities) {
    List<CubeMetadata> cubeMetadataList = new ArrayList<>();
    tableEntities.forEach(table -> {
        List<ColumnEntity> cols = table.getColumns();
        StarTreeMetadataBuilder builder = new StarTreeMetadataBuilder(table.getParameters().get(STAR_TABLE_NAME), table.getParameters().get(SOURCE_TABLE_NAME));
        cols.forEach(col -> {
            if (col.getType().equals("aggregate")) {
                builder.addAggregationColumn(col.getName(), col.getParameters().get("aggregateFunction"), col.getParameters().get(ORIGINAL_COLUMN), Boolean.parseBoolean(col.getParameters().get("distinct")));
            } else if (col.getType().equals("dimension")) {
                builder.addDimensionColumn(col.getName(), col.getParameters().get(ORIGINAL_COLUMN));
            }
        });
        String groupingString = table.getParameters().get(GROUPING_STRING);
        // Create empty set to support Empty Group
        builder.addGroup((groupingString == null || groupingString.isEmpty()) ? new HashSet<>() : Sets.newHashSet(groupingString.split(COLUMN_DELIMITER)));
        String sourceTablePredicate = table.getParameters().get(SOURCE_FILTER_STRING);
        String cubePredicate = table.getParameters().get(PREDICATE_STRING);
        if (sourceTablePredicate != null || cubePredicate != null) {
            builder.withCubeFilter(new CubeFilter(sourceTablePredicate, cubePredicate));
        }
        builder.setCubeStatus(CubeStatus.forValue(Integer.parseInt(table.getParameters().get(CUBE_STATUS))));
        builder.setTableLastUpdatedTime(Long.parseLong(table.getParameters().get(SOURCE_TABLE_LAST_UPDATED_TIME)));
        builder.setCubeLastUpdatedTime(Long.parseLong(table.getParameters().get(CUBE_LAST_UPDATED_TIME)));
        cubeMetadataList.add(builder.build());
    });
    return cubeMetadataList;
}

Example 5 with CubeMetadata

use of io.hetu.core.spi.cube.CubeMetadata in project hetu-core by openlookeng.

the class DropTableTask method execute.

@Override
public ListenableFuture<?> execute(DropTable statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters, HeuristicIndexerManager heuristicIndexerManager) {
    Session session = stateMachine.getSession();
    QualifiedObjectName fullObjectName = createQualifiedObjectName(session, statement, statement.getTableName());
    QualifiedName tableName = QualifiedName.of(fullObjectName.getCatalogName(), fullObjectName.getSchemaName(), fullObjectName.getObjectName());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, fullObjectName);
    if (!tableHandle.isPresent()) {
        if (!statement.isExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<CubeMetaStore> optionalCubeMetaStore = this.cubeManager.getMetaStore(STAR_TREE);
    if (optionalCubeMetaStore.isPresent() && optionalCubeMetaStore.get().getMetadataFromCubeName(tableName.toString()).isPresent()) {
        throw new SemanticException(DROP_TABLE_ON_CUBE, statement, "%s is a star-tree cube, drop using DROP CUBE", tableName);
    }
    accessControl.checkCanDropTable(session.getRequiredTransactionId(), session.getIdentity(), fullObjectName);
    if (PropertyService.getBooleanProperty(HetuConstant.SPLIT_CACHE_MAP_ENABLED)) {
        // Check if SplitCacheMap is enabled
        SplitCacheMap splitCacheMap = SplitCacheMap.getInstance();
        if (splitCacheMap.cacheExists(tableName)) {
            splitCacheMap.dropCache(tableName, Optional.empty());
        }
    }
    metadata.dropTable(session, tableHandle.get());
    if (optionalCubeMetaStore.isPresent()) {
        List<CubeMetadata> cubes = optionalCubeMetaStore.get().getMetadataList(tableName.toString());
        for (CubeMetadata cube : cubes) {
            String[] parts = cube.getCubeName().split("\\.");
            Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, createQualifiedObjectName(session, null, QualifiedName.of(parts[0], parts[1], parts[2])));
            try {
                cubeHandle.ifPresent(cubeTable -> metadata.dropTable(session, cubeTable));
                optionalCubeMetaStore.get().removeCube(cube);
            } catch (TableNotFoundException e) {
                // Can happen in concurrent drop table and drop cube calls
                LOG.debug("Tried dropping cube table but it is already dropped", e);
            }
        }
    }
    dropIndices(heuristicIndexerManager, tableName);
    return immediateFuture(null);
}