Search in sources :

Example 1 with CubeMetaStore

use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.

the class TestStarTreeAggregationRule method testDoNotFireWhenWithoutCube.

@Test
public void testDoNotFireWhenWithoutCube() {
    Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
    Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
    TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
    TableHandle tmpOrdersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
    Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(ImmutableList.of()));
    StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, tester().getMetadata());
    tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "true").on(p -> p.aggregation(builder -> builder.globalGrouping().addAggregation(p.symbol("count", BIGINT), new FunctionCallBuilder(tester().getMetadata()).setName(QualifiedName.of("count")).build(), ImmutableList.of(BIGINT)).step(SINGLE).source(p.project(Assignments.builder().put(p.symbol("custkey"), OriginalExpressionUtils.castToRowExpression(SymbolUtils.toSymbolReference(p.symbol("custkey")))).build(), p.tableScan(tmpOrdersTableHandle, ImmutableList.of(p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT))))))).doesNotFire();
    Mockito.verify(cubeMetaStore, Mockito.atLeastOnce()).getMetadataList(eq("local.sf1.0.orders"));
}
Also used : BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Arrays(java.util.Arrays) ListMultimap(com.google.common.collect.ListMultimap) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) QualifiedName(io.prestosql.sql.tree.QualifiedName) FunctionCallBuilder(io.prestosql.sql.planner.FunctionCallBuilder) Test(org.testng.annotations.Test) AggregationNode(io.prestosql.spi.plan.AggregationNode) Cast(io.prestosql.sql.tree.Cast) FilterNode(io.prestosql.spi.plan.FilterNode) ENABLE_STAR_TREE_INDEX(io.prestosql.SystemSessionProperties.ENABLE_STAR_TREE_INDEX) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) PlanBuilder.expression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.expression) Assert.assertFalse(org.testng.Assert.assertFalse) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) SymbolUtils(io.prestosql.sql.planner.SymbolUtils) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) TableScanNode(io.prestosql.spi.plan.TableScanNode) PlanNode(io.prestosql.spi.plan.PlanNode) UUID(java.util.UUID) CubeStatement(io.hetu.core.spi.cube.CubeStatement) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) FunctionHandle(io.prestosql.spi.function.FunctionHandle) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Matchers.any(org.mockito.Matchers.any) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) DoubleLiteral(io.prestosql.sql.tree.DoubleLiteral) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Returns(org.mockito.internal.stubbing.answers.Returns) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableMetadata(io.prestosql.metadata.TableMetadata) StandardTypes(io.prestosql.spi.type.StandardTypes) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) AbstractMockMetadata.dummyMetadata(io.prestosql.metadata.AbstractMockMetadata.dummyMetadata) TableHandle(io.prestosql.spi.metadata.TableHandle) Matchers.anyString(org.mockito.Matchers.anyString) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) BaseMatcher(org.hamcrest.BaseMatcher) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) SINGLE(io.prestosql.spi.plan.AggregationNode.Step.SINGLE) DateTimeUtils(io.prestosql.spi.util.DateTimeUtils) Session(io.prestosql.Session) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) TpchTransactionHandle(io.prestosql.plugin.tpch.TpchTransactionHandle) DATE(io.prestosql.spi.type.DateType.DATE) CubeProvider(io.prestosql.spi.cube.CubeProvider) Symbol(io.prestosql.spi.plan.Symbol) Description(org.hamcrest.Description) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) Assignments(io.prestosql.spi.plan.Assignments) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Expressions(io.prestosql.sql.relational.Expressions) CubeManager(io.prestosql.cube.CubeManager) Mockito(org.mockito.Mockito) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Matcher(org.hamcrest.Matcher) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) RowExpression(io.prestosql.spi.relation.RowExpression) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) Assert.assertTrue(org.testng.Assert.assertTrue) Returns(org.mockito.internal.stubbing.answers.Returns) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) FunctionCallBuilder(io.prestosql.sql.planner.FunctionCallBuilder) BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Test(org.testng.annotations.Test)

Example 2 with CubeMetaStore

use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.

the class TestStarTreeAggregationRule method testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated.

@Test
public void testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated() {
    Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
    Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
    Metadata metadata = Mockito.mock(Metadata.class);
    TableMetadata ordersTableMetadata = Mockito.mock(TableMetadata.class);
    QualifiedObjectName objectName = new QualifiedObjectName("local", "sf1.0", "orders");
    Mockito.when(metadata.getTableHandle(any(Session.class), eq(objectName))).thenReturn(Optional.of(ordersTableHandle));
    Mockito.when(metadata.getTableLastModifiedTimeSupplier(any(Session.class), any(TableHandle.class))).thenReturn(() -> DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-02 12:00:00"));
    Mockito.when(metadata.getTableMetadata(any(Session.class), eq(ordersTableHandle))).thenReturn(ordersTableMetadata);
    Mockito.when(ordersTableMetadata.getQualifiedName()).thenReturn(objectName);
    List<CubeMetadata> metadataList = ImmutableList.of(cubeMetadata);
    Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(metadataList));
    Mockito.when(cubeMetadata.matches(any(CubeStatement.class))).thenReturn(true);
    Mockito.when(cubeMetadata.getLastUpdatedTime()).thenReturn(DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-01 12:00:00"));
    StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, metadata);
    tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "true").on(p -> p.aggregation(builder -> builder.step(SINGLE).addAggregation(new Symbol("count_orderkey"), PlanBuilder.expression("count(orderkey)"), ImmutableList.of(BIGINT)).singleGroupingSet(new Symbol("orderdate")).source(p.project(Assignments.builder().put(p.symbol("orderdate", DATE), p.variable("orderdate", DATE)).put(p.symbol("orderkey", BIGINT), p.variable("orderkey", BIGINT)).build(), p.tableScan(ordersTableHandle, ImmutableList.of(p.symbol("orderdate", DATE), p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT), p.symbol("orderdate", DATE), new TpchColumnHandle("orderdate", DATE))))))).doesNotFire();
    Mockito.verify(cubeMetaStore, Mockito.atLeastOnce()).getMetadataList(eq("local.sf1.0.orders"));
    Mockito.verify(cubeMetadata, Mockito.atLeastOnce()).matches(any(CubeStatement.class));
}
Also used : TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Arrays(java.util.Arrays) ListMultimap(com.google.common.collect.ListMultimap) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) QualifiedName(io.prestosql.sql.tree.QualifiedName) FunctionCallBuilder(io.prestosql.sql.planner.FunctionCallBuilder) Test(org.testng.annotations.Test) AggregationNode(io.prestosql.spi.plan.AggregationNode) Cast(io.prestosql.sql.tree.Cast) FilterNode(io.prestosql.spi.plan.FilterNode) ENABLE_STAR_TREE_INDEX(io.prestosql.SystemSessionProperties.ENABLE_STAR_TREE_INDEX) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) PlanBuilder.expression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.expression) Assert.assertFalse(org.testng.Assert.assertFalse) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) SymbolUtils(io.prestosql.sql.planner.SymbolUtils) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) TableScanNode(io.prestosql.spi.plan.TableScanNode) PlanNode(io.prestosql.spi.plan.PlanNode) UUID(java.util.UUID) CubeStatement(io.hetu.core.spi.cube.CubeStatement) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) FunctionHandle(io.prestosql.spi.function.FunctionHandle) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Matchers.any(org.mockito.Matchers.any) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) DoubleLiteral(io.prestosql.sql.tree.DoubleLiteral) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Returns(org.mockito.internal.stubbing.answers.Returns) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableMetadata(io.prestosql.metadata.TableMetadata) StandardTypes(io.prestosql.spi.type.StandardTypes) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) AbstractMockMetadata.dummyMetadata(io.prestosql.metadata.AbstractMockMetadata.dummyMetadata) TableHandle(io.prestosql.spi.metadata.TableHandle) Matchers.anyString(org.mockito.Matchers.anyString) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) BaseMatcher(org.hamcrest.BaseMatcher) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) SINGLE(io.prestosql.spi.plan.AggregationNode.Step.SINGLE) DateTimeUtils(io.prestosql.spi.util.DateTimeUtils) Session(io.prestosql.Session) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) TpchTransactionHandle(io.prestosql.plugin.tpch.TpchTransactionHandle) DATE(io.prestosql.spi.type.DateType.DATE) CubeProvider(io.prestosql.spi.cube.CubeProvider) Symbol(io.prestosql.spi.plan.Symbol) Description(org.hamcrest.Description) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) Assignments(io.prestosql.spi.plan.Assignments) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Expressions(io.prestosql.sql.relational.Expressions) CubeManager(io.prestosql.cube.CubeManager) Mockito(org.mockito.Mockito) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Matcher(org.hamcrest.Matcher) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) RowExpression(io.prestosql.spi.relation.RowExpression) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) Assert.assertTrue(org.testng.Assert.assertTrue) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) Symbol(io.prestosql.spi.plan.Symbol) Metadata(io.prestosql.metadata.Metadata) TableMetadata(io.prestosql.metadata.TableMetadata) AbstractMockMetadata.dummyMetadata(io.prestosql.metadata.AbstractMockMetadata.dummyMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) CubeStatement(io.hetu.core.spi.cube.CubeStatement) Returns(org.mockito.internal.stubbing.answers.Returns) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) Session(io.prestosql.Session) BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Test(org.testng.annotations.Test)

Example 3 with CubeMetaStore

use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.

the class TestStarTreeAggregationRule method testDoNotFireWhenFeatureIsDisabled.

@Test
public void testDoNotFireWhenFeatureIsDisabled() {
    Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
    Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
    TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
    TableHandle tmpOrdersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
    StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, tester().getMetadata());
    tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "false").on(p -> p.aggregation(builder -> builder.globalGrouping().addAggregation(p.symbol("count", BIGINT), new FunctionCallBuilder(tester().getMetadata()).setName(QualifiedName.of("count")).build(), ImmutableList.of(BIGINT)).step(SINGLE).source(p.project(Assignments.builder().put(p.symbol("custkey"), p.variable("custkey", custkeyHandle.getType())).build(), p.tableScan(tmpOrdersTableHandle, ImmutableList.of(p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT))))))).doesNotFire();
    Mockito.verify(cubeMetaStore, Mockito.never()).getMetadataList(eq("local.sf1.0.orders"));
}
Also used : BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Arrays(java.util.Arrays) ListMultimap(com.google.common.collect.ListMultimap) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) QualifiedName(io.prestosql.sql.tree.QualifiedName) FunctionCallBuilder(io.prestosql.sql.planner.FunctionCallBuilder) Test(org.testng.annotations.Test) AggregationNode(io.prestosql.spi.plan.AggregationNode) Cast(io.prestosql.sql.tree.Cast) FilterNode(io.prestosql.spi.plan.FilterNode) ENABLE_STAR_TREE_INDEX(io.prestosql.SystemSessionProperties.ENABLE_STAR_TREE_INDEX) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) PlanBuilder.expression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.expression) Assert.assertFalse(org.testng.Assert.assertFalse) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) SymbolUtils(io.prestosql.sql.planner.SymbolUtils) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) TableScanNode(io.prestosql.spi.plan.TableScanNode) PlanNode(io.prestosql.spi.plan.PlanNode) UUID(java.util.UUID) CubeStatement(io.hetu.core.spi.cube.CubeStatement) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) FunctionHandle(io.prestosql.spi.function.FunctionHandle) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Matchers.any(org.mockito.Matchers.any) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) DoubleLiteral(io.prestosql.sql.tree.DoubleLiteral) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Returns(org.mockito.internal.stubbing.answers.Returns) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableMetadata(io.prestosql.metadata.TableMetadata) StandardTypes(io.prestosql.spi.type.StandardTypes) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) AbstractMockMetadata.dummyMetadata(io.prestosql.metadata.AbstractMockMetadata.dummyMetadata) TableHandle(io.prestosql.spi.metadata.TableHandle) Matchers.anyString(org.mockito.Matchers.anyString) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) BaseMatcher(org.hamcrest.BaseMatcher) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) SINGLE(io.prestosql.spi.plan.AggregationNode.Step.SINGLE) DateTimeUtils(io.prestosql.spi.util.DateTimeUtils) Session(io.prestosql.Session) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) TpchTransactionHandle(io.prestosql.plugin.tpch.TpchTransactionHandle) DATE(io.prestosql.spi.type.DateType.DATE) CubeProvider(io.prestosql.spi.cube.CubeProvider) Symbol(io.prestosql.spi.plan.Symbol) Description(org.hamcrest.Description) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) Assignments(io.prestosql.spi.plan.Assignments) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Expressions(io.prestosql.sql.relational.Expressions) CubeManager(io.prestosql.cube.CubeManager) Mockito(org.mockito.Mockito) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Matcher(org.hamcrest.Matcher) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) RowExpression(io.prestosql.spi.relation.RowExpression) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) Assert.assertTrue(org.testng.Assert.assertTrue) Returns(org.mockito.internal.stubbing.answers.Returns) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) FunctionCallBuilder(io.prestosql.sql.planner.FunctionCallBuilder) BaseRuleTest(io.prestosql.sql.planner.iterative.rule.test.BaseRuleTest) Test(org.testng.annotations.Test)

Example 4 with CubeMetaStore

use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.

the class CreateCubeTask method internalExecute.

@VisibleForTesting
public ListenableFuture<?> internalExecute(CreateCube statement, Metadata metadata, AccessControl accessControl, Session session, QueryStateMachine stateMachine, List<Expression> parameters) {
    Optional<CubeMetaStore> optionalCubeMetaStore = cubeManager.getMetaStore(STAR_TREE);
    if (!optionalCubeMetaStore.isPresent()) {
        throw new RuntimeException("HetuMetaStore is not initialized");
    }
    QualifiedObjectName cubeName = createQualifiedObjectName(session, statement, statement.getCubeName());
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSourceTableName());
    Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, cubeName);
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (optionalCubeMetaStore.get().getMetadataFromCubeName(cubeName.toString()).isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_ALREADY_EXISTS, statement, "Cube '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    if (cubeHandle.isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_OR_TABLE_ALREADY_EXISTS, statement, "Cube or Table '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    CatalogName catalogName = metadata.getCatalogHandle(session, cubeName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + cubeName.getCatalogName()));
    if (!metadata.isPreAggregationSupported(session, catalogName)) {
        throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, String.format("Cube cannot be created on catalog '%s'", catalogName.toString()));
    }
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
    }
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
    List<String> groupingSet = statement.getGroupingSet().stream().map(s -> s.getValue().toLowerCase(ENGLISH)).collect(Collectors.toList());
    Map<String, ColumnMetadata> sourceTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
    List<ColumnMetadata> cubeColumns = new ArrayList<>();
    Map<String, AggregationSignature> aggregations = new HashMap<>();
    Analysis analysis = analyzeStatement(statement, session, metadata, accessControl, parameters, stateMachine.getWarningCollector());
    Map<String, Field> fields = analysis.getOutputDescriptor().getAllFields().stream().collect(Collectors.toMap(col -> col.getName().map(String::toLowerCase).get(), col -> col));
    for (FunctionCall aggFunction : statement.getAggregations()) {
        String aggFunctionName = aggFunction.getName().toString().toLowerCase(ENGLISH);
        String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue().toLowerCase(ENGLISH);
        boolean distinct = aggFunction.isDistinct();
        String cubeColumnName = aggFunctionName + "_" + (argument == null ? "all" : argument) + (aggFunction.isDistinct() ? "_distinct" : "");
        CubeAggregateFunction cubeAggregateFunction = CubeAggregateFunction.valueOf(aggFunctionName.toUpperCase(ENGLISH));
        switch(cubeAggregateFunction) {
            case SUM:
                aggregations.put(cubeColumnName, AggregationSignature.sum(argument, distinct));
                break;
            case COUNT:
                AggregationSignature aggregationSignature = argument == null ? AggregationSignature.count() : AggregationSignature.count(argument, distinct);
                aggregations.put(cubeColumnName, aggregationSignature);
                break;
            case AVG:
                aggregations.put(cubeColumnName, AggregationSignature.avg(argument, distinct));
                break;
            case MAX:
                aggregations.put(cubeColumnName, AggregationSignature.max(argument, distinct));
                break;
            case MIN:
                aggregations.put(cubeColumnName, AggregationSignature.min(argument, distinct));
                break;
            default:
                throw new PrestoException(NOT_SUPPORTED, format("Unsupported aggregation function : %s", aggFunctionName));
        }
        Field tableField = fields.get(cubeColumnName);
        ColumnMetadata cubeCol = new ColumnMetadata(cubeColumnName, tableField.getType(), true, null, null, false, Collections.emptyMap());
        cubeColumns.add(cubeCol);
    }
    accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, Expression> sqlProperties = mapFromProperties(statement.getProperties());
    Map<String, Object> properties = metadata.getTablePropertyManager().getProperties(catalogName, cubeName.getCatalogName(), sqlProperties, session, metadata, parameters);
    if (properties.containsKey("partitioned_by")) {
        List<String> partitionCols = new ArrayList<>(((List<String>) properties.get("partitioned_by")));
        // put all partition columns at the end of the list
        groupingSet.removeAll(partitionCols);
        groupingSet.addAll(partitionCols);
    }
    for (String dimension : groupingSet) {
        if (!sourceTableColumns.containsKey(dimension)) {
            throw new SemanticException(MISSING_COLUMN, statement, "Column %s does not exist", dimension);
        }
        ColumnMetadata tableCol = sourceTableColumns.get(dimension);
        ColumnMetadata cubeCol = new ColumnMetadata(dimension, tableCol.getType(), tableCol.isNullable(), null, null, false, tableCol.getProperties());
        cubeColumns.add(cubeCol);
    }
    ConnectorTableMetadata cubeTableMetadata = new ConnectorTableMetadata(cubeName.asSchemaTableName(), ImmutableList.copyOf(cubeColumns), properties);
    try {
        metadata.createTable(session, cubeName.getCatalogName(), cubeTableMetadata, statement.isNotExists());
    } catch (PrestoException e) {
        // connectors are not required to handle the ignoreExisting flag
        if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) {
            throw e;
        }
    }
    CubeMetadataBuilder builder = optionalCubeMetaStore.get().getBuilder(cubeName.toString(), tableName.toString());
    groupingSet.forEach(dimension -> builder.addDimensionColumn(dimension, dimension));
    aggregations.forEach((column, aggregationSignature) -> builder.addAggregationColumn(column, aggregationSignature.getFunction(), aggregationSignature.getDimension(), aggregationSignature.isDistinct()));
    builder.addGroup(new HashSet<>(groupingSet));
    // Status and Table modified time will be updated on the first insert into the cube
    builder.setCubeStatus(CubeStatus.INACTIVE);
    builder.setTableLastUpdatedTime(-1L);
    statement.getSourceFilter().ifPresent(sourceTablePredicate -> {
        sourceTablePredicate = Coercer.addCoercions(sourceTablePredicate, analysis);
        builder.withCubeFilter(new CubeFilter(ExpressionFormatter.formatExpression(sourceTablePredicate, Optional.empty())));
    });
    builder.setCubeLastUpdatedTime(System.currentTimeMillis());
    optionalCubeMetaStore.get().persist(builder.build());
    return immediateFuture(null);
}
Also used : SqlParser(io.prestosql.sql.parser.SqlParser) CreateCube(io.prestosql.sql.tree.CreateCube) Statement(io.prestosql.sql.tree.Statement) CUBE_OR_TABLE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.CUBE_OR_TABLE_ALREADY_EXISTS) WarningCollector(io.prestosql.execution.warnings.WarningCollector) ExpressionFormatter(io.prestosql.sql.ExpressionFormatter) Map(java.util.Map) CubeFilter(io.hetu.core.spi.cube.CubeFilter) ENGLISH(java.util.Locale.ENGLISH) Coercer(io.prestosql.sql.planner.Coercer) Identifier(io.prestosql.sql.tree.Identifier) VisibleForTesting(org.assertj.core.util.VisibleForTesting) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PrestoException(io.prestosql.spi.PrestoException) AccessControl(io.prestosql.security.AccessControl) CatalogName(io.prestosql.spi.connector.CatalogName) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) String.format(java.lang.String.format) List(java.util.List) LongLiteral(io.prestosql.sql.tree.LongLiteral) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NOT_SUPPORTED(io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED) StandardErrorCode(io.prestosql.spi.StandardErrorCode) STAR_TREE(io.prestosql.cube.CubeManager.STAR_TREE) CubeMetadataBuilder(io.hetu.core.spi.cube.CubeMetadataBuilder) Field(io.prestosql.sql.analyzer.Field) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) ALREADY_EXISTS(io.prestosql.spi.StandardErrorCode.ALREADY_EXISTS) CUBE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.CUBE_ALREADY_EXISTS) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) TableMetadata(io.prestosql.metadata.TableMetadata) TransactionManager(io.prestosql.transaction.TransactionManager) HashMap(java.util.HashMap) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) SemanticException(io.prestosql.sql.analyzer.SemanticException) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) Session(io.prestosql.Session) MISSING_COLUMN(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_COLUMN) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) MISSING_TABLE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Analyzer(io.prestosql.sql.analyzer.Analyzer) CubeManager(io.prestosql.cube.CubeManager) Collections(java.util.Collections) Expression(io.prestosql.sql.tree.Expression) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) HashMap(java.util.HashMap) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) ArrayList(java.util.ArrayList) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) PrestoException(io.prestosql.spi.PrestoException) Field(io.prestosql.sql.analyzer.Field) Identifier(io.prestosql.sql.tree.Identifier) CubeMetadataBuilder(io.hetu.core.spi.cube.CubeMetadataBuilder) CubeFilter(io.hetu.core.spi.cube.CubeFilter) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) SemanticException(io.prestosql.sql.analyzer.SemanticException) TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) LongLiteral(io.prestosql.sql.tree.LongLiteral) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) Expression(io.prestosql.sql.tree.Expression) Analysis(io.prestosql.sql.analyzer.Analysis) TableHandle(io.prestosql.spi.metadata.TableHandle) CatalogName(io.prestosql.spi.connector.CatalogName) VisibleForTesting(org.assertj.core.util.VisibleForTesting)

Example 5 with CubeMetaStore

use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.

the class AddColumnTask method execute.

@Override
public ListenableFuture<?> execute(AddColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters, HeuristicIndexerManager heuristicIndexerManager) {
    Session session = stateMachine.getSession();
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName());
    Optional<CubeMetaStore> optionalCubeMetaStore = this.cubeManager.getMetaStore(STAR_TREE);
    if (optionalCubeMetaStore.isPresent() && optionalCubeMetaStore.get().getMetadataFromCubeName(tableName.toString()).isPresent()) {
        throw new SemanticException(ALTER_TABLE_ON_CUBE, statement, "Operation not permitted. %s is a Cube table, use CUBE statements instead.", tableName);
    }
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
    }
    CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName()));
    accessControl.checkCanAddColumns(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
    ColumnDefinition element = statement.getColumn();
    Type type;
    try {
        type = metadata.getType(parseTypeSignature(element.getType()));
    } catch (TypeNotFoundException e) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (type.equals(UNKNOWN)) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (columnHandles.containsKey(element.getName().getValue().toLowerCase(ENGLISH))) {
        throw new SemanticException(COLUMN_ALREADY_EXISTS, statement, "Column '%s' already exists", element.getName());
    }
    if (!element.isNullable() && !metadata.getConnectorCapabilities(session, catalogName).contains(NOT_NULL_COLUMN_CONSTRAINT)) {
        throw new SemanticException(NOT_SUPPORTED, element, "Catalog '%s' does not support NOT NULL for column '%s'", catalogName.getCatalogName(), element.getName());
    }
    Map<String, Expression> sqlProperties = mapFromProperties(element.getProperties());
    Map<String, Object> columnProperties = metadata.getColumnPropertyManager().getProperties(catalogName, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
    ColumnMetadata column = new ColumnMetadata(element.getName().getValue(), type, element.isNullable(), element.getComment().orElse(null), null, false, columnProperties);
    metadata.addColumn(session, tableHandle.get(), column);
    return immediateFuture(null);
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) PrestoException(io.prestosql.spi.PrestoException) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) ColumnDefinition(io.prestosql.sql.tree.ColumnDefinition) Type(io.prestosql.spi.type.Type) Expression(io.prestosql.sql.tree.Expression) TypeNotFoundException(io.prestosql.spi.type.TypeNotFoundException) TableHandle(io.prestosql.spi.metadata.TableHandle) CatalogName(io.prestosql.spi.connector.CatalogName) Session(io.prestosql.Session) SemanticException(io.prestosql.sql.analyzer.SemanticException)

Aggregations

CubeMetaStore (io.hetu.core.spi.cube.io.CubeMetaStore)13 QualifiedObjectName (io.prestosql.spi.connector.QualifiedObjectName)12 TableHandle (io.prestosql.spi.metadata.TableHandle)12 Session (io.prestosql.Session)11 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)8 CubeMetadata (io.hetu.core.spi.cube.CubeMetadata)7 MetadataUtil.createQualifiedObjectName (io.prestosql.metadata.MetadataUtil.createQualifiedObjectName)7 ColumnMetadata (io.prestosql.spi.connector.ColumnMetadata)7 SemanticException (io.prestosql.sql.analyzer.SemanticException)7 CubeManager (io.prestosql.cube.CubeManager)6 TableMetadata (io.prestosql.metadata.TableMetadata)6 ConnectorTableMetadata (io.prestosql.spi.connector.ConnectorTableMetadata)6 CubeProvider (io.prestosql.spi.cube.CubeProvider)6 QualifiedName (io.prestosql.sql.tree.QualifiedName)6 ImmutableList (com.google.common.collect.ImmutableList)5 Metadata (io.prestosql.metadata.Metadata)5 TpchColumnHandle (io.prestosql.plugin.tpch.TpchColumnHandle)5 TpchTableHandle (io.prestosql.plugin.tpch.TpchTableHandle)5 TpchTableLayoutHandle (io.prestosql.plugin.tpch.TpchTableLayoutHandle)5 SchemaTableName (io.prestosql.spi.connector.SchemaTableName)5