use of io.prestosql.sql.tree.CreateIndex in project hetu-core by openlookeng.
the class QueryPlanner method createIndex.
/**
* CREATE INDEX statements are rewritten as SELECT statements,
* if the original statement was CREATE INDEX, create the necessary plan nodes
* to create the index
*/
private PlanBuilder createIndex(PlanBuilder subPlan, Statement originalStatement) {
if (!(originalStatement instanceof CreateIndex)) {
return subPlan;
}
// rewrite sub queries
CreateIndex createIndex = (CreateIndex) originalStatement;
String tableName = MetadataUtil.createQualifiedObjectName(session, originalStatement, createIndex.getTableName()).toString();
List<String> partitions = new ArrayList<>();
if (createIndex.getExpression().isPresent()) {
partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
}
Map<String, Type> columnTypes = new HashMap<>();
for (Field field : analysis.getRootScope().getRelationType().getAllFields()) {
if (INDEX_SUPPORTED_TYPES.get(createIndex.getIndexType().toLowerCase(Locale.ENGLISH)).stream().noneMatch(supportType -> field.getType().getDisplayName().contains(supportType))) {
throw new UnsupportedOperationException("Index creation on " + field.getType().getDisplayName() + " column is not supported");
}
columnTypes.put(field.getOriginColumnName().get(), field.getType());
}
Properties indexProperties = new Properties();
CreateIndexMetadata.Level indexCreationLevel = CreateIndexMetadata.Level.UNDEFINED;
indexProperties.setProperty(LEVEL_PROP_KEY, indexCreationLevel.toString());
boolean autoLoadFound = false;
for (Property property : createIndex.getProperties()) {
String key = extractPropertyValue(property.getName());
String val = extractPropertyValue(property.getValue()).toUpperCase(Locale.ENGLISH);
if (key.equals(LEVEL_PROP_KEY)) {
indexCreationLevel = CreateIndexMetadata.Level.valueOf(val);
continue;
}
if (key.equals(AUTOLOAD_PROP_KEY)) {
autoLoadFound = true;
String valInLowerCase = val.toLowerCase(Locale.ROOT);
if (valInLowerCase.equals("true") || valInLowerCase.equals("false")) {
indexProperties.setProperty(key, valInLowerCase);
} else {
throw new IllegalArgumentException("Unrecognized value for key '" + AUTOLOAD_PROP_KEY + "', only 'true' or 'false' are allowed");
}
continue;
}
indexProperties.setProperty(key, val);
}
if (!autoLoadFound) {
boolean defaultAutoloadProp = PropertyService.getBooleanProperty(HetuConstant.FILTER_CACHE_AUTOLOAD_DEFAULT);
indexProperties.setProperty(AUTOLOAD_PROP_KEY, String.valueOf(defaultAutoloadProp));
}
return subPlan.withNewRoot(new CreateIndexNode(idAllocator.getNextId(), ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.REMOTE, subPlan.getRoot()), new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, createIndex.getColumnAliases().stream().map(identifier -> new Pair<>(identifier.toString(), columnTypes.get(identifier.toString().toLowerCase(Locale.ROOT)))).collect(Collectors.toList()), partitions, indexProperties, session.getUser(), indexCreationLevel)));
}
use of io.prestosql.sql.tree.CreateIndex in project hetu-core by openlookeng.
the class StatementAnalyzer method validateCreateIndex.
private void validateCreateIndex(Table table, Optional<Scope> scope) {
CreateIndex createIndex = (CreateIndex) analysis.getOriginalStatement();
QualifiedObjectName tableFullName = createQualifiedObjectName(session, createIndex, createIndex.getTableName());
accessControl.checkCanCreateIndex(session.getRequiredTransactionId(), session.getIdentity(), tableFullName);
String tableName = tableFullName.toString();
// check whether catalog support create index
if (!metadata.isHeuristicIndexSupported(session, tableFullName)) {
throw new SemanticException(NOT_SUPPORTED, createIndex, "CREATE INDEX is not supported in catalog '%s'", tableFullName.getCatalogName());
}
List<String> partitions = new ArrayList<>();
String partitionColumn = null;
if (createIndex.getExpression().isPresent()) {
partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
// check partition name validate, create index …… where pt_d = xxx;
// pt_d must be partition column
Set<String> partitionColumns = partitions.stream().map(k -> k.substring(0, k.indexOf("="))).collect(Collectors.toSet());
if (partitionColumns.size() > 1) {
// currently only support one partition column
throw new IllegalArgumentException("Heuristic index only supports predicates on one column");
}
// The only entry in set should be the only partition column name
partitionColumn = partitionColumns.iterator().next();
}
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableFullName);
if (tableHandle.isPresent()) {
if (!tableHandle.get().getConnectorHandle().isHeuristicIndexSupported()) {
throw new SemanticException(NOT_SUPPORTED, table, "Catalog supported, but table storage format is not supported by heuristic index");
}
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
List<String> availableColumns = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
for (Identifier column : createIndex.getColumnAliases()) {
if (!availableColumns.contains(column.getValue().toLowerCase(Locale.ROOT))) {
throw new SemanticException(MISSING_ATTRIBUTE, table, "Column '%s' cannot be resolved", column.getValue());
}
}
if (partitionColumn != null && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) {
throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns");
}
} else {
throw new SemanticException(MISSING_ATTRIBUTE, table, "Table '%s' is invalid", tableFullName);
}
List<Pair<String, Type>> indexColumns = new LinkedList<>();
for (Identifier i : createIndex.getColumnAliases()) {
indexColumns.add(new Pair<>(i.toString(), UNKNOWN));
}
// For now, creating index for multiple columns is not supported
if (indexColumns.size() > 1) {
throw new SemanticException(NOT_SUPPORTED, table, "Multi-column indexes are currently not supported");
}
try {
// Use this place holder to check the existence of index and lock the place
Properties properties = new Properties();
properties.setProperty(INPROGRESS_PROPERTY_KEY, "TRUE");
CreateIndexMetadata placeHolder = new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, indexColumns, partitions, properties, session.getUser(), UNDEFINED);
synchronized (StatementAnalyzer.class) {
IndexClient.RecordStatus recordStatus = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(placeHolder);
switch(recordStatus) {
case SAME_NAME:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' already exists", createIndex.getIndexName().toString());
case SAME_CONTENT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists");
case SAME_INDEX_PART_CONFLICT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists and partition(s) contain conflicts");
case IN_PROGRESS_SAME_NAME:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' is being created by another user. Check running queries for details. If there is no running query for this index, " + "the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString(), createIndex.getIndexName().toString());
case IN_PROGRESS_SAME_CONTENT:
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX'");
case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
if (partitions.isEmpty()) {
throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString());
}
// allow different queries to run with explicitly same partitions
case SAME_INDEX_PART_CAN_MERGE:
case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
break;
case NOT_FOUND:
heuristicIndexerManager.getIndexClient().addIndexRecord(placeHolder);
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
use of io.prestosql.sql.tree.CreateIndex in project hetu-core by openlookeng.
the class CachedSqlQueryExecution method createPlan.
@Override
protected Plan createPlan(Analysis analysis, Session session, List<PlanOptimizer> planOptimizers, PlanNodeIdAllocator idAllocator, Metadata metadata, TypeAnalyzer typeAnalyzer, StatsCalculator statsCalculator, CostCalculator costCalculator, WarningCollector warningCollector) {
Statement statement = analysis.getStatement();
// Get relevant Session properties which may affect the resulting execution plan
// Property to property value mapping
Map<String, Object> systemSessionProperties = new HashMap<>();
SystemSessionProperties sessionProperties = new SystemSessionProperties();
for (PropertyMetadata<?> property : sessionProperties.getSessionProperties()) {
systemSessionProperties.put(property.getName(), session.getSystemProperty(property.getName(), property.getJavaType()));
}
// if the original statement before rewriting is CreateIndex, set session to let connector know that pageMetadata should be enabled
if (analysis.getOriginalStatement() instanceof CreateIndex || analysis.getOriginalStatement() instanceof UpdateIndex) {
session.setPageMetadataEnabled(true);
}
// build list of fully qualified table names
List<String> tableNames = new ArrayList<>();
Map<String, TableStatistics> tableStatistics = new HashMap<>();
// Get column name to column type to detect column type changes between queries more easily
Map<String, Type> columnTypes = new HashMap<>();
// Cacheable conditions:
// 1. Caching must be enabled globally
// 2. Caching must be enabled in the session
// 3. There must not be any parameters in the query
// TODO: remove requirement for empty params and implement parameter rewrite
// 4. Methods in ConnectorTableHandle and ConnectorMetadata must be
// overwritten to allow access to fully qualified table names and column names
// 5. Statement must be an instance of Query and not contain CurrentX functions
boolean cacheable = this.cache.isPresent() && isExecutionPlanCacheEnabled(session) && analysis.getParameters().isEmpty() && validateAndExtractTableAndColumns(analysis, metadata, session, tableNames, tableStatistics, columnTypes) && isCacheable(statement) && // create index and update index should not be cached
(!(analysis.getOriginalStatement() instanceof CreateIndex || analysis.getOriginalStatement() instanceof UpdateIndex));
cacheable = cacheable && !tableNames.isEmpty();
if (!cacheable) {
return super.createPlan(analysis, session, planOptimizers, idAllocator, metadata, typeAnalyzer, statsCalculator, costCalculator, warningCollector);
}
List<String> optimizers = new ArrayList<>();
// build list of enabled optimizers and rules for cache key
for (PlanOptimizer planOptimizer : planOptimizers) {
if (planOptimizer instanceof IterativeOptimizer) {
IterativeOptimizer iterativeOptimizer = (IterativeOptimizer) planOptimizer;
Set<Rule<?>> rules = iterativeOptimizer.getRules();
for (Rule rule : rules) {
if (OptimizerUtils.isEnabledRule(rule, session)) {
optimizers.add(rule.getClass().getSimpleName());
}
}
} else {
if (OptimizerUtils.isEnabledLegacy(planOptimizer, session)) {
optimizers.add(planOptimizer.getClass().getSimpleName());
}
}
}
Set<String> connectors = tableNames.stream().map(table -> table.substring(0, table.indexOf("."))).collect(Collectors.toSet());
connectors.stream().forEach(connector -> {
for (Map.Entry<String, String> property : session.getConnectorProperties(new CatalogName(connector)).entrySet()) {
systemSessionProperties.put(connector + "." + property.getKey(), property.getValue());
}
});
Plan plan;
// TODO: Traverse the statement to build the key then combine tables/optimizers.. etc
int key = SqlQueryExecutionCacheKeyGenerator.buildKey((Query) statement, tableNames, optimizers, columnTypes, session.getTimeZoneKey(), systemSessionProperties);
CachedSqlQueryExecutionPlan cachedPlan = this.cache.get().getIfPresent(key);
HetuLogicalPlanner logicalPlanner = new HetuLogicalPlanner(session, planOptimizers, idAllocator, metadata, typeAnalyzer, statsCalculator, costCalculator, warningCollector);
PlanNode root;
plan = cachedPlan != null ? cachedPlan.getPlan() : null;
// that rely on system time
if (plan != null && cachedPlan.getTimeZoneKey().equals(session.getTimeZoneKey()) && cachedPlan.getStatement().equals(statement) && session.getTransactionId().isPresent() && cachedPlan.getIdentity().getUser().equals(session.getIdentity().getUser())) {
// TODO: traverse the statement and accept partial match
root = plan.getRoot();
boolean isValidCachePlan = tablesMatch(root, analysis.getTables());
try {
if (!isEqualBasicStatistics(cachedPlan.getTableStatistics(), tableStatistics, tableNames) || !isValidCachePlan) {
for (TableHandle tableHandle : analysis.getTables()) {
tableStatistics.replace(tableHandle.getFullyQualifiedName(), metadata.getTableStatistics(session, tableHandle, Constraint.alwaysTrue(), true));
}
if (!cachedPlan.getTableStatistics().equals(tableStatistics) || !isValidCachePlan) {
// Table have changed, therfore the cached plan may no longer be applicable
throw new NoSuchElementException();
}
}
// TableScanNode may contain the old transaction id.
// The following logic rewrites the logical plan by replacing the TableScanNode with a new TableScanNode which
// contains the new transaction id from session.
root = SimplePlanRewriter.rewriteWith(new TableHandleRewriter(session, analysis, metadata), root);
} catch (NoSuchElementException e) {
// Cached plan is outdated
// invalidate cache
this.cache.get().invalidateAll();
// Build a new plan
plan = createAndCachePlan(key, logicalPlanner, statement, tableNames, tableStatistics, optimizers, analysis, columnTypes, systemSessionProperties);
root = plan.getRoot();
}
} else {
// Build a new plan
for (TableHandle tableHandle : analysis.getTables()) {
tableStatistics.replace(tableHandle.getFullyQualifiedName(), metadata.getTableStatistics(session, tableHandle, Constraint.alwaysTrue(), true));
}
plan = createAndCachePlan(key, logicalPlanner, statement, tableNames, tableStatistics, optimizers, analysis, columnTypes, systemSessionProperties);
root = plan.getRoot();
}
// BeginTableWrite optimizer must be run at the end as the last optimization
// due to a hack Hetu community added which also serves to updates
// metadata in the nodes
root = this.beginTableWrite.optimize(root, session, null, null, null, null);
plan = update(plan, root);
return plan;
}
Aggregations