use of org.voltdb.planner.ParsedSelectStmt in project voltdb by VoltDB.
the class DDLCompiler method addIndexToCatalog.
private static void addIndexToCatalog(Database db, Table table, VoltXMLElement node, Map<String, String> indexReplacementMap, HashMap<String, Index> indexMap, HashMap<String, Column> columnMap, VoltCompiler compiler) throws VoltCompilerException {
assert node.name.equals("index");
String name = node.attributes.get("name");
boolean unique = Boolean.parseBoolean(node.attributes.get("unique"));
boolean assumeUnique = Boolean.parseBoolean(node.attributes.get("assumeunique"));
AbstractParsedStmt dummy = new ParsedSelectStmt(null, db);
dummy.setDDLIndexedTable(table);
StringBuffer msg = new StringBuffer(String.format("Index \"%s\" ", name));
// "parse" the expression trees for an expression-based index (vs. a simple column value index)
List<AbstractExpression> exprs = null;
// "parse" the WHERE expression for partial index if any
AbstractExpression predicate = null;
// Some expressions have special validation in indices. Not all the expression
// can be indexed. We scan for result type at first here and block those which
// can't be indexed like boolean, geo ... We gather rest of expression into
// checkExpressions list. We will check on them all at once.
List<AbstractExpression> checkExpressions = new ArrayList<>();
for (VoltXMLElement subNode : node.children) {
if (subNode.name.equals("exprs")) {
exprs = new ArrayList<>();
for (VoltXMLElement exprNode : subNode.children) {
AbstractExpression expr = dummy.parseExpressionTree(exprNode);
expr.resolveForTable(table);
expr.finalizeValueTypes();
// string will be populated with an expression's details when
// its value type is not indexable
StringBuffer exprMsg = new StringBuffer();
if (!expr.isValueTypeIndexable(exprMsg)) {
// indexing on expression with boolean result is not supported.
throw compiler.new VoltCompilerException("Cannot create index \"" + name + "\" because it contains " + exprMsg + ", which is not supported.");
}
if ((unique || assumeUnique) && !expr.isValueTypeUniqueIndexable(exprMsg)) {
// indexing on expression with boolean result is not supported.
throw compiler.new VoltCompilerException("Cannot create unique index \"" + name + "\" because it contains " + exprMsg + ", which is not supported.");
}
// rest of the validity guards will be evaluated after collecting all the expressions.
checkExpressions.add(expr);
exprs.add(expr);
}
} else if (subNode.name.equals("predicate")) {
assert (subNode.children.size() == 1);
VoltXMLElement predicateXML = subNode.children.get(0);
assert (predicateXML != null);
predicate = buildPartialIndexPredicate(dummy, name, predicateXML, table, compiler);
}
}
// Check all the subexpressions we gathered up.
if (!AbstractExpression.validateExprsForIndexesAndMVs(checkExpressions, msg)) {
// The error message will be in the StringBuffer msg.
throw compiler.new VoltCompilerException(msg.toString());
}
String colList = node.attributes.get("columns");
String[] colNames = colList.split(",");
Column[] columns = new Column[colNames.length];
boolean has_nonint_col = false;
boolean has_geo_col = false;
String nonint_col_name = null;
for (int i = 0; i < colNames.length; i++) {
columns[i] = columnMap.get(colNames[i]);
if (columns[i] == null) {
return;
}
}
UnsafeOperatorsForDDL unsafeOps = new UnsafeOperatorsForDDL();
if (exprs == null) {
for (int i = 0; i < colNames.length; i++) {
VoltType colType = VoltType.get((byte) columns[i].getType());
if (!colType.isIndexable()) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " values are not currently supported as index keys: \"" + colNames[i] + "\"";
throw compiler.new VoltCompilerException(emsg);
}
if ((unique || assumeUnique) && !colType.isUniqueIndexable()) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " values are not currently supported as unique index keys: \"" + colNames[i] + "\"";
throw compiler.new VoltCompilerException(emsg);
}
if (!colType.isBackendIntegerType()) {
has_nonint_col = true;
nonint_col_name = colNames[i];
has_geo_col = colType.equals(VoltType.GEOGRAPHY);
if (has_geo_col && colNames.length > 1) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " values must be the only component of an index key: \"" + nonint_col_name + "\"";
throw compiler.new VoltCompilerException(emsg);
}
}
}
} else {
for (AbstractExpression expression : exprs) {
VoltType colType = expression.getValueType();
if (!colType.isIndexable()) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " valued expressions are not currently supported as index keys.";
throw compiler.new VoltCompilerException(emsg);
}
if ((unique || assumeUnique) && !colType.isUniqueIndexable()) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " valued expressions are not currently supported as unique index keys.";
throw compiler.new VoltCompilerException(emsg);
}
if (!colType.isBackendIntegerType()) {
has_nonint_col = true;
nonint_col_name = "<expression>";
has_geo_col = colType.equals(VoltType.GEOGRAPHY);
if (has_geo_col) {
if (exprs.size() > 1) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " values must be the only component of an index key.";
throw compiler.new VoltCompilerException(emsg);
} else if (!(expression instanceof TupleValueExpression)) {
String emsg = "Cannot create index \"" + name + "\" because " + colType.getName() + " expressions must be simple column expressions.";
throw compiler.new VoltCompilerException(emsg);
}
}
}
expression.findUnsafeOperatorsForDDL(unsafeOps);
}
}
Index index = table.getIndexes().add(name);
index.setCountable(false);
index.setIssafewithnonemptysources(!unsafeOps.isUnsafe());
// Set the index type. It will be one of:
// - Covering cell index (geo index for CONTAINS predicates)
// - HASH index (set in HSQL because "hash" is in the name of the
// constraint or the index
// - TREE index, which is the default
boolean isHashIndex = node.attributes.get("ishashindex").equals("true");
if (has_geo_col) {
index.setType(IndexType.COVERING_CELL_INDEX.getValue());
} else if (isHashIndex) {
// warn user that hash index will be deprecated
compiler.addWarn("Hash indexes are deprecated. In a future release, VoltDB will only support tree indexes, even if the index name contains the string \"hash\"");
// make the index a hash.
if (has_nonint_col) {
String emsg = "Index " + name + " in table " + table.getTypeName() + " uses a non-hashable column " + nonint_col_name;
throw compiler.new VoltCompilerException(emsg);
}
index.setType(IndexType.HASH_TABLE.getValue());
} else {
index.setType(IndexType.BALANCED_TREE.getValue());
index.setCountable(true);
}
// but they still represent the columns that will trigger an index update when their values change.
for (int i = 0; i < columns.length; i++) {
ColumnRef cref = index.getColumns().add(columns[i].getTypeName());
cref.setColumn(columns[i]);
cref.setIndex(i);
}
if (exprs != null) {
try {
index.setExpressionsjson(convertToJSONArray(exprs));
} catch (JSONException e) {
throw compiler.new VoltCompilerException("Unexpected error serializing non-column expressions for index '" + name + "' on type '" + table.getTypeName() + "': " + e.toString());
}
}
index.setUnique(unique);
if (assumeUnique) {
index.setUnique(true);
}
index.setAssumeunique(assumeUnique);
if (predicate != null) {
try {
index.setPredicatejson(convertToJSONObject(predicate));
} catch (JSONException e) {
throw compiler.new VoltCompilerException("Unexpected error serializing predicate for partial index '" + name + "' on type '" + table.getTypeName() + "': " + e.toString());
}
}
// will make two indexes different
for (Index existingIndex : table.getIndexes()) {
// skip thineself
if (existingIndex == index) {
continue;
}
if (indexesAreDups(existingIndex, index)) {
// replace any constraints using one index with the other
//for () TODO
// get ready for replacements from constraints created later
indexReplacementMap.put(index.getTypeName(), existingIndex.getTypeName());
// if the index is a user-named index...
if (index.getTypeName().startsWith(HSQLInterface.AUTO_GEN_PREFIX) == false) {
// on dup-detection, add a warning but don't fail
String emsg = String.format("Dropping index %s on table %s because it duplicates index %s.", index.getTypeName(), table.getTypeName(), existingIndex.getTypeName());
compiler.addWarn(emsg);
}
// drop the index and GTFO
table.getIndexes().delete(index.getTypeName());
return;
}
}
String smsg = "Created index: " + name + " on table: " + table.getTypeName() + " of type: " + IndexType.get(index.getType()).name();
compiler.addInfo(smsg);
indexMap.put(name, index);
}
use of org.voltdb.planner.ParsedSelectStmt in project voltdb by VoltDB.
the class MaterializedViewProcessor method startProcessing.
/**
* Add materialized view info to the catalog for the tables that are
* materialized views.
* @throws VoltCompilerException
*/
public void startProcessing(Database db, HashMap<Table, String> matViewMap, TreeSet<String> exportTableNames) throws VoltCompilerException {
HashSet<String> viewTableNames = new HashSet<>();
for (Entry<Table, String> entry : matViewMap.entrySet()) {
viewTableNames.add(entry.getKey().getTypeName());
}
for (Entry<Table, String> entry : matViewMap.entrySet()) {
Table destTable = entry.getKey();
String query = entry.getValue();
// get the xml for the query
VoltXMLElement xmlquery = null;
try {
xmlquery = m_hsql.getXMLCompiledStatement(query);
} catch (HSQLParseException e) {
e.printStackTrace();
}
assert (xmlquery != null);
// parse the xml like any other sql statement
ParsedSelectStmt stmt = null;
try {
stmt = (ParsedSelectStmt) AbstractParsedStmt.parse(query, xmlquery, null, db, null);
} catch (Exception e) {
throw m_compiler.new VoltCompilerException(e.getMessage());
}
assert (stmt != null);
String viewName = destTable.getTypeName();
// throw an error if the view isn't within voltdb's limited world view
checkViewMeetsSpec(viewName, stmt);
// The primary key index is yet to be defined (below).
for (Index destIndex : destTable.getIndexes()) {
if (destIndex.getUnique() || destIndex.getAssumeunique()) {
String msg = "A UNIQUE or ASSUMEUNIQUE index is not allowed on a materialized view. " + "Remove the qualifier from the index " + destIndex.getTypeName() + "defined on the materialized view \"" + viewName + "\".";
throw m_compiler.new VoltCompilerException(msg);
}
}
// A Materialized view cannot depend on another view.
for (Table srcTable : stmt.m_tableList) {
if (viewTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) can not be defined on another view (%s).", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
}
// The existing code base still need this materializer field to tell if a table
// is a materialized view table. Leaving this for future refactoring.
destTable.setMaterializer(stmt.m_tableList.get(0));
List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(destTable.getColumns(), "index");
List<AbstractExpression> groupbyExprs = null;
if (stmt.hasComplexGroupby()) {
groupbyExprs = new ArrayList<>();
for (ParsedColInfo col : stmt.groupByColumns()) {
groupbyExprs.add(col.expression);
}
}
// Generate query XMLs for min/max recalculation (ENG-8641)
boolean isMultiTableView = stmt.m_tableList.size() > 1;
MatViewFallbackQueryXMLGenerator xmlGen = new MatViewFallbackQueryXMLGenerator(xmlquery, stmt.groupByColumns(), stmt.m_displayColumns, isMultiTableView);
List<VoltXMLElement> fallbackQueryXMLs = xmlGen.getFallbackQueryXMLs();
// index or constraint in order to avoid error and crash.
if (stmt.groupByColumns().size() != 0) {
Index pkIndex = destTable.getIndexes().add(HSQLInterface.AUTO_GEN_MATVIEW_IDX);
pkIndex.setType(IndexType.BALANCED_TREE.getValue());
pkIndex.setUnique(true);
// assume index 1 throuh #grpByCols + 1 are the cols
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ColumnRef c = pkIndex.getColumns().add(String.valueOf(i));
c.setColumn(destColumnArray.get(i));
c.setIndex(i);
}
Constraint pkConstraint = destTable.getConstraints().add(HSQLInterface.AUTO_GEN_MATVIEW_CONST);
pkConstraint.setType(ConstraintType.PRIMARY_KEY.getValue());
pkConstraint.setIndex(pkIndex);
}
// If we have an unsafe MV message, then
// remember it here. We don't really know how
// to transfer the message through the catalog, but
// we can transmit the existence of the message.
boolean isSafeForDDL = (stmt.getUnsafeMVMessage() == null);
// Here the code path diverges for different kinds of views (single table view and joined table view)
if (isMultiTableView) {
// Materialized view on joined tables
// Add mvHandlerInfo to the destTable:
MaterializedViewHandlerInfo mvHandlerInfo = destTable.getMvhandlerinfo().add("mvHandlerInfo");
mvHandlerInfo.setDesttable(destTable);
for (Table srcTable : stmt.m_tableList) {
// Now we do not support having a view on persistent tables joining streamed tables.
if (exportTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) on joined tables cannot have streamed table (%s) as its source.", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
// The view table will need to keep a list of its source tables.
// The list is used to install / uninstall the view reference on the source tables when the
// view handler is constructed / destroyed.
TableRef tableRef = mvHandlerInfo.getSourcetables().add(srcTable.getTypeName());
tableRef.setTable(srcTable);
// There could be more than one partition column candidate, but we will only use the first one we found.
if (destTable.getPartitioncolumn() == null && srcTable.getPartitioncolumn() != null) {
Column partitionColumn = srcTable.getPartitioncolumn();
String partitionColName = partitionColumn.getTypeName();
String srcTableName = srcTable.getTypeName();
destTable.setIsreplicated(false);
if (stmt.hasComplexGroupby()) {
for (int i = 0; i < groupbyExprs.size(); i++) {
AbstractExpression groupbyExpr = groupbyExprs.get(i);
if (groupbyExpr instanceof TupleValueExpression) {
TupleValueExpression tve = (TupleValueExpression) groupbyExpr;
if (tve.getTableName().equals(srcTableName) && tve.getColumnName().equals(partitionColName)) {
// The partition column is set to destColumnArray.get(i), because we have the restriction
// that the non-aggregate columns must come at the very begining, and must exactly match
// the group-by columns.
// If we are going to remove this restriction in the future, then we need to do more work
// in order to find a proper partition column.
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
} else {
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
if (gbcol.tableName.equals(srcTableName) && gbcol.columnName.equals(partitionColName)) {
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
}
// end find partition column
}
// end for each source table
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, mvHandlerInfo);
compileCreateQueryAndUpdateCatalog(db, query, xmlquery, mvHandlerInfo);
mvHandlerInfo.setGroupbycolumncount(stmt.groupByColumns().size());
for (int i = 0; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
// Set the expression type here to determine the behavior of the merge function.
destColumn.setAggregatetype(col.expression.getExpressionType().getValue());
}
mvHandlerInfo.setIssafewithnonemptysources(isSafeForDDL);
} else {
// =======================================================================================
// Materialized view on single table
// create the materializedviewinfo catalog node for the source table
Table srcTable = stmt.m_tableList.get(0);
MaterializedViewInfo matviewinfo = srcTable.getViews().add(viewName);
matviewinfo.setDest(destTable);
AbstractExpression where = stmt.getSingleTableFilterExpression();
if (where != null) {
String hex = Encoder.hexEncode(where.toJSONString());
matviewinfo.setPredicate(hex);
} else {
matviewinfo.setPredicate("");
}
List<Column> srcColumnArray = CatalogUtil.getSortedCatalogItems(srcTable.getColumns(), "index");
if (stmt.hasComplexGroupby()) {
// Parse group by expressions to json string
String groupbyExprsJson = null;
try {
groupbyExprsJson = DDLCompiler.convertToJSONArray(groupbyExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for group by expressions: " + e.toString());
}
matviewinfo.setGroupbyexpressionsjson(groupbyExprsJson);
} else {
// add the group by columns from the src table
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
Column srcCol = srcColumnArray.get(gbcol.index);
ColumnRef cref = matviewinfo.getGroupbycols().add(srcCol.getTypeName());
// groupByColumns is iterating in order of groups. Store that grouping order
// in the column ref index. When the catalog is serialized, it will, naturally,
// scramble this order like a two year playing dominos, presenting the data
// in a meaningless sequence.
// the column offset in the view's grouping order
cref.setIndex(i);
// the source column from the base (non-view) table
cref.setColumn(srcCol);
// parse out the group by columns into the dest table
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
processMaterializedViewColumn(srcTable, destColumn, ExpressionType.VALUE_TUPLE, (TupleValueExpression) col.expression);
}
}
// Set up COUNT(*) column
ParsedColInfo countCol = stmt.m_displayColumns.get(stmt.groupByColumns().size());
assert (countCol.expression.getExpressionType() == ExpressionType.AGGREGATE_COUNT_STAR);
assert (countCol.expression.getLeft() == null);
processMaterializedViewColumn(srcTable, destColumnArray.get(stmt.groupByColumns().size()), ExpressionType.AGGREGATE_COUNT_STAR, null);
// prepare info for aggregation columns.
List<AbstractExpression> aggregationExprs = new ArrayList<>();
boolean hasAggregationExprs = false;
ArrayList<AbstractExpression> minMaxAggs = new ArrayList<>();
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
AbstractExpression aggExpr = col.expression.getLeft();
if (aggExpr.getExpressionType() != ExpressionType.VALUE_TUPLE) {
hasAggregationExprs = true;
}
aggregationExprs.add(aggExpr);
if (col.expression.getExpressionType() == ExpressionType.AGGREGATE_MIN || col.expression.getExpressionType() == ExpressionType.AGGREGATE_MAX) {
minMaxAggs.add(aggExpr);
}
}
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, matviewinfo);
// set Aggregation Expressions.
if (hasAggregationExprs) {
String aggregationExprsJson = null;
try {
aggregationExprsJson = DDLCompiler.convertToJSONArray(aggregationExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for aggregation expressions: " + e.toString());
}
matviewinfo.setAggregationexpressionsjson(aggregationExprsJson);
}
// Find index for each min/max aggCol/aggExpr (ENG-6511 and ENG-8512)
for (Integer i = 0; i < minMaxAggs.size(); ++i) {
Index found = findBestMatchIndexForMatviewMinOrMax(matviewinfo, srcTable, groupbyExprs, minMaxAggs.get(i));
IndexRef refFound = matviewinfo.getIndexforminmax().add(i.toString());
if (found != null) {
refFound.setName(found.getTypeName());
} else {
refFound.setName("");
}
}
// The COUNT(*) should return a BIGINT column, whereas we found here the COUNT(*) was assigned a INTEGER column.
for (int i = 0; i <= stmt.groupByColumns().size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
}
// parse out the aggregation columns into the dest table
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
AbstractExpression colExpr = col.expression.getLeft();
TupleValueExpression tve = null;
if (colExpr.getExpressionType() == ExpressionType.VALUE_TUPLE) {
tve = (TupleValueExpression) colExpr;
}
processMaterializedViewColumn(srcTable, destColumn, col.expression.getExpressionType(), tve);
setTypeAttributesForColumn(destColumn, col.expression);
}
if (srcTable.getPartitioncolumn() != null) {
// Set the partitioning of destination tables of associated views.
// If a view's source table is replicated, then a full scan of the
// associated view is single-sited. If the source is partitioned,
// a full scan of the view must be distributed, unless it is filtered
// by the original table's partitioning key, which, to be filtered,
// must also be a GROUP BY key.
destTable.setIsreplicated(false);
setGroupedTablePartitionColumn(matviewinfo, srcTable.getPartitioncolumn());
}
matviewinfo.setIssafewithnonemptysources(isSafeForDDL);
}
// end if single table view materialized view.
}
}
use of org.voltdb.planner.ParsedSelectStmt in project voltdb by VoltDB.
the class StmtSubqueryScan method canRunInOneFragment.
/**
* Some subquery results can only be joined with a partitioned table after
* it finishes some work on the coordinator. With the 2 fragment plan limit,
* those queries can not be supported.
* Other than that case, the planner will typically have added a
* send/receive pair to the subquery plan that is actually only suitable to
* a stand-alone plan. This function distinguishes subqueries that should NOT
* have a send/receive pair.
* @param root
* @return true if there is no aspect to the plan that requires execution on the coordinator.
*/
public boolean canRunInOneFragment() {
assert (m_subqueriesPartitioning != null);
assert (m_subqueryStmt != null);
if (m_subqueriesPartitioning.getCountOfPartitionedTables() == 0) {
return true;
}
// of their results.
if (failsSingleFragmentTest()) {
return false;
}
// Tentative assignment in case of early return.
// This gets immediately reset if it passes all the tests.
m_failedSingleFragmentTest = true;
if (m_subqueryStmt instanceof ParsedUnionStmt) {
// Union are just returned
return false;
}
if (!(m_subqueryStmt instanceof ParsedSelectStmt)) {
throw new PlanningErrorException("Unsupported subquery found in FROM clause:" + m_subqueryStmt);
}
ParsedSelectStmt selectStmt = (ParsedSelectStmt) m_subqueryStmt;
// we should get rid of the receive node. I (--paul) don't know what this means.
if (selectStmt.hasLimitOrOffset() || selectStmt.hasDistinctWithGroupBy()) {
return false;
}
// because it contains a partitioned view that does not have a partition column.
if (selectStmt.m_mvFixInfo.needed()) {
return false;
}
// Table aggregate cases should not get rid of the receive node
if (selectStmt.hasAggregateOrGroupby()) {
if (!selectStmt.isGrouped()) {
m_tableAggregateSubquery = true;
return false;
}
// Detect case (1) to mark receive node.
if (!selectStmt.hasPartitionColumnInGroupby()) {
return false;
}
}
if (!selectStmt.hasPartitionColumnInWindowFunctionExpression()) {
return false;
}
// Now. If this sub-query joins with a partitioned table in the parent statement,
// push the join down by removing the send/receive plan node pair.
m_failedSingleFragmentTest = false;
return true;
}
Aggregations