use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.
the class TestStarTreeAggregationRule method testDoNotFireWhenWithoutCube.
@Test
public void testDoNotFireWhenWithoutCube() {
Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
TableHandle tmpOrdersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(ImmutableList.of()));
StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, tester().getMetadata());
tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "true").on(p -> p.aggregation(builder -> builder.globalGrouping().addAggregation(p.symbol("count", BIGINT), new FunctionCallBuilder(tester().getMetadata()).setName(QualifiedName.of("count")).build(), ImmutableList.of(BIGINT)).step(SINGLE).source(p.project(Assignments.builder().put(p.symbol("custkey"), OriginalExpressionUtils.castToRowExpression(SymbolUtils.toSymbolReference(p.symbol("custkey")))).build(), p.tableScan(tmpOrdersTableHandle, ImmutableList.of(p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT))))))).doesNotFire();
Mockito.verify(cubeMetaStore, Mockito.atLeastOnce()).getMetadataList(eq("local.sf1.0.orders"));
}
use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.
the class TestStarTreeAggregationRule method testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated.
@Test
public void testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated() {
Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
Metadata metadata = Mockito.mock(Metadata.class);
TableMetadata ordersTableMetadata = Mockito.mock(TableMetadata.class);
QualifiedObjectName objectName = new QualifiedObjectName("local", "sf1.0", "orders");
Mockito.when(metadata.getTableHandle(any(Session.class), eq(objectName))).thenReturn(Optional.of(ordersTableHandle));
Mockito.when(metadata.getTableLastModifiedTimeSupplier(any(Session.class), any(TableHandle.class))).thenReturn(() -> DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-02 12:00:00"));
Mockito.when(metadata.getTableMetadata(any(Session.class), eq(ordersTableHandle))).thenReturn(ordersTableMetadata);
Mockito.when(ordersTableMetadata.getQualifiedName()).thenReturn(objectName);
List<CubeMetadata> metadataList = ImmutableList.of(cubeMetadata);
Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(metadataList));
Mockito.when(cubeMetadata.matches(any(CubeStatement.class))).thenReturn(true);
Mockito.when(cubeMetadata.getLastUpdatedTime()).thenReturn(DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-01 12:00:00"));
StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, metadata);
tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "true").on(p -> p.aggregation(builder -> builder.step(SINGLE).addAggregation(new Symbol("count_orderkey"), PlanBuilder.expression("count(orderkey)"), ImmutableList.of(BIGINT)).singleGroupingSet(new Symbol("orderdate")).source(p.project(Assignments.builder().put(p.symbol("orderdate", DATE), p.variable("orderdate", DATE)).put(p.symbol("orderkey", BIGINT), p.variable("orderkey", BIGINT)).build(), p.tableScan(ordersTableHandle, ImmutableList.of(p.symbol("orderdate", DATE), p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT), p.symbol("orderdate", DATE), new TpchColumnHandle("orderdate", DATE))))))).doesNotFire();
Mockito.verify(cubeMetaStore, Mockito.atLeastOnce()).getMetadataList(eq("local.sf1.0.orders"));
Mockito.verify(cubeMetadata, Mockito.atLeastOnce()).matches(any(CubeStatement.class));
}
use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.
the class TestStarTreeAggregationRule method testDoNotFireWhenFeatureIsDisabled.
@Test
public void testDoNotFireWhenFeatureIsDisabled() {
Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
TableHandle tmpOrdersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, tester().getMetadata());
tester().assertThat(starTreeAggregationRule).setSystemProperty(ENABLE_STAR_TREE_INDEX, "false").on(p -> p.aggregation(builder -> builder.globalGrouping().addAggregation(p.symbol("count", BIGINT), new FunctionCallBuilder(tester().getMetadata()).setName(QualifiedName.of("count")).build(), ImmutableList.of(BIGINT)).step(SINGLE).source(p.project(Assignments.builder().put(p.symbol("custkey"), p.variable("custkey", custkeyHandle.getType())).build(), p.tableScan(tmpOrdersTableHandle, ImmutableList.of(p.symbol("orderkey", BIGINT)), ImmutableMap.of(p.symbol("orderkey", BIGINT), new TpchColumnHandle("orderkey", BIGINT))))))).doesNotFire();
Mockito.verify(cubeMetaStore, Mockito.never()).getMetadataList(eq("local.sf1.0.orders"));
}
use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.
the class CreateCubeTask method internalExecute.
@VisibleForTesting
public ListenableFuture<?> internalExecute(CreateCube statement, Metadata metadata, AccessControl accessControl, Session session, QueryStateMachine stateMachine, List<Expression> parameters) {
Optional<CubeMetaStore> optionalCubeMetaStore = cubeManager.getMetaStore(STAR_TREE);
if (!optionalCubeMetaStore.isPresent()) {
throw new RuntimeException("HetuMetaStore is not initialized");
}
QualifiedObjectName cubeName = createQualifiedObjectName(session, statement, statement.getCubeName());
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSourceTableName());
Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, cubeName);
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
if (optionalCubeMetaStore.get().getMetadataFromCubeName(cubeName.toString()).isPresent()) {
if (!statement.isNotExists()) {
throw new SemanticException(CUBE_ALREADY_EXISTS, statement, "Cube '%s' already exists", cubeName);
}
return immediateFuture(null);
}
if (cubeHandle.isPresent()) {
if (!statement.isNotExists()) {
throw new SemanticException(CUBE_OR_TABLE_ALREADY_EXISTS, statement, "Cube or Table '%s' already exists", cubeName);
}
return immediateFuture(null);
}
CatalogName catalogName = metadata.getCatalogHandle(session, cubeName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + cubeName.getCatalogName()));
if (!metadata.isPreAggregationSupported(session, catalogName)) {
throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, String.format("Cube cannot be created on catalog '%s'", catalogName.toString()));
}
if (!tableHandle.isPresent()) {
throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
}
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
List<String> groupingSet = statement.getGroupingSet().stream().map(s -> s.getValue().toLowerCase(ENGLISH)).collect(Collectors.toList());
Map<String, ColumnMetadata> sourceTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
List<ColumnMetadata> cubeColumns = new ArrayList<>();
Map<String, AggregationSignature> aggregations = new HashMap<>();
Analysis analysis = analyzeStatement(statement, session, metadata, accessControl, parameters, stateMachine.getWarningCollector());
Map<String, Field> fields = analysis.getOutputDescriptor().getAllFields().stream().collect(Collectors.toMap(col -> col.getName().map(String::toLowerCase).get(), col -> col));
for (FunctionCall aggFunction : statement.getAggregations()) {
String aggFunctionName = aggFunction.getName().toString().toLowerCase(ENGLISH);
String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue().toLowerCase(ENGLISH);
boolean distinct = aggFunction.isDistinct();
String cubeColumnName = aggFunctionName + "_" + (argument == null ? "all" : argument) + (aggFunction.isDistinct() ? "_distinct" : "");
CubeAggregateFunction cubeAggregateFunction = CubeAggregateFunction.valueOf(aggFunctionName.toUpperCase(ENGLISH));
switch(cubeAggregateFunction) {
case SUM:
aggregations.put(cubeColumnName, AggregationSignature.sum(argument, distinct));
break;
case COUNT:
AggregationSignature aggregationSignature = argument == null ? AggregationSignature.count() : AggregationSignature.count(argument, distinct);
aggregations.put(cubeColumnName, aggregationSignature);
break;
case AVG:
aggregations.put(cubeColumnName, AggregationSignature.avg(argument, distinct));
break;
case MAX:
aggregations.put(cubeColumnName, AggregationSignature.max(argument, distinct));
break;
case MIN:
aggregations.put(cubeColumnName, AggregationSignature.min(argument, distinct));
break;
default:
throw new PrestoException(NOT_SUPPORTED, format("Unsupported aggregation function : %s", aggFunctionName));
}
Field tableField = fields.get(cubeColumnName);
ColumnMetadata cubeCol = new ColumnMetadata(cubeColumnName, tableField.getType(), true, null, null, false, Collections.emptyMap());
cubeColumns.add(cubeCol);
}
accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
Map<String, Expression> sqlProperties = mapFromProperties(statement.getProperties());
Map<String, Object> properties = metadata.getTablePropertyManager().getProperties(catalogName, cubeName.getCatalogName(), sqlProperties, session, metadata, parameters);
if (properties.containsKey("partitioned_by")) {
List<String> partitionCols = new ArrayList<>(((List<String>) properties.get("partitioned_by")));
// put all partition columns at the end of the list
groupingSet.removeAll(partitionCols);
groupingSet.addAll(partitionCols);
}
for (String dimension : groupingSet) {
if (!sourceTableColumns.containsKey(dimension)) {
throw new SemanticException(MISSING_COLUMN, statement, "Column %s does not exist", dimension);
}
ColumnMetadata tableCol = sourceTableColumns.get(dimension);
ColumnMetadata cubeCol = new ColumnMetadata(dimension, tableCol.getType(), tableCol.isNullable(), null, null, false, tableCol.getProperties());
cubeColumns.add(cubeCol);
}
ConnectorTableMetadata cubeTableMetadata = new ConnectorTableMetadata(cubeName.asSchemaTableName(), ImmutableList.copyOf(cubeColumns), properties);
try {
metadata.createTable(session, cubeName.getCatalogName(), cubeTableMetadata, statement.isNotExists());
} catch (PrestoException e) {
// connectors are not required to handle the ignoreExisting flag
if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) {
throw e;
}
}
CubeMetadataBuilder builder = optionalCubeMetaStore.get().getBuilder(cubeName.toString(), tableName.toString());
groupingSet.forEach(dimension -> builder.addDimensionColumn(dimension, dimension));
aggregations.forEach((column, aggregationSignature) -> builder.addAggregationColumn(column, aggregationSignature.getFunction(), aggregationSignature.getDimension(), aggregationSignature.isDistinct()));
builder.addGroup(new HashSet<>(groupingSet));
// Status and Table modified time will be updated on the first insert into the cube
builder.setCubeStatus(CubeStatus.INACTIVE);
builder.setTableLastUpdatedTime(-1L);
statement.getSourceFilter().ifPresent(sourceTablePredicate -> {
sourceTablePredicate = Coercer.addCoercions(sourceTablePredicate, analysis);
builder.withCubeFilter(new CubeFilter(ExpressionFormatter.formatExpression(sourceTablePredicate, Optional.empty())));
});
builder.setCubeLastUpdatedTime(System.currentTimeMillis());
optionalCubeMetaStore.get().persist(builder.build());
return immediateFuture(null);
}
use of io.hetu.core.spi.cube.io.CubeMetaStore in project hetu-core by openlookeng.
the class AddColumnTask method execute.
@Override
public ListenableFuture<?> execute(AddColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters, HeuristicIndexerManager heuristicIndexerManager) {
Session session = stateMachine.getSession();
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName());
Optional<CubeMetaStore> optionalCubeMetaStore = this.cubeManager.getMetaStore(STAR_TREE);
if (optionalCubeMetaStore.isPresent() && optionalCubeMetaStore.get().getMetadataFromCubeName(tableName.toString()).isPresent()) {
throw new SemanticException(ALTER_TABLE_ON_CUBE, statement, "Operation not permitted. %s is a Cube table, use CUBE statements instead.", tableName);
}
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
if (!tableHandle.isPresent()) {
throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
}
CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName()));
accessControl.checkCanAddColumns(session.getRequiredTransactionId(), session.getIdentity(), tableName);
Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
ColumnDefinition element = statement.getColumn();
Type type;
try {
type = metadata.getType(parseTypeSignature(element.getType()));
} catch (TypeNotFoundException e) {
throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
}
if (type.equals(UNKNOWN)) {
throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
}
if (columnHandles.containsKey(element.getName().getValue().toLowerCase(ENGLISH))) {
throw new SemanticException(COLUMN_ALREADY_EXISTS, statement, "Column '%s' already exists", element.getName());
}
if (!element.isNullable() && !metadata.getConnectorCapabilities(session, catalogName).contains(NOT_NULL_COLUMN_CONSTRAINT)) {
throw new SemanticException(NOT_SUPPORTED, element, "Catalog '%s' does not support NOT NULL for column '%s'", catalogName.getCatalogName(), element.getName());
}
Map<String, Expression> sqlProperties = mapFromProperties(element.getProperties());
Map<String, Object> columnProperties = metadata.getColumnPropertyManager().getProperties(catalogName, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
ColumnMetadata column = new ColumnMetadata(element.getName().getValue(), type, element.isNullable(), element.getComment().orElse(null), null, false, columnProperties);
metadata.addColumn(session, tableHandle.get(), column);
return immediateFuture(null);
}
Aggregations