use of org.voltdb.catalog.MaterializedViewHandlerInfo in project voltdb by VoltDB.
the class SwapTablesPlanNode method viewsDependOn.
/**
* @param theTable
* @return
*/
private static boolean viewsDependOn(Table aTable, StringBuilder viewNames) {
String separator = "(";
for (MaterializedViewInfo anyView : aTable.getViews()) {
viewNames.append(separator).append(anyView.getTypeName());
separator = ", ";
}
for (Table anyTable : ((Database) aTable.getParent()).getTables()) {
for (MaterializedViewHandlerInfo anyView : anyTable.getMvhandlerinfo()) {
if (anyView.getSourcetables().getIgnoreCase(aTable.getTypeName()) != null) {
viewNames.append(separator).append(anyView.getDesttable().getTypeName());
separator = ", ";
}
}
}
if (", ".equals(separator)) {
viewNames.append(")");
return true;
}
return false;
}
use of org.voltdb.catalog.MaterializedViewHandlerInfo in project voltdb by VoltDB.
the class MaterializedViewFixInfo method processMVBasedQueryFix.
/**
* Check whether the results from a materialized view need to be
* re-aggregated on the coordinator by the view's GROUP BY columns
* prior to any of the processing specified by the query.
* This is normally the case when a mat view's source table is partitioned
* and the view's GROUP BY does not include the partition key.
* There is a special edge case where the query already contains the exact
* reaggregations that the added-cost fix would introduce, so the fix can
* be skipped as an optimization.
* Set the m_needed flag to true, only if the reaggregation fix is needed.
* @return The value of m_needed
*/
public boolean processMVBasedQueryFix(StmtTableScan mvTableScan, Set<SchemaColumn> scanColumns, JoinNode joinTree, List<ParsedColInfo> displayColumns, List<ParsedColInfo> groupByColumns) {
//@TODO
if (!(mvTableScan instanceof StmtTargetTableScan)) {
return false;
}
Table table = ((StmtTargetTableScan) mvTableScan).getTargetTable();
assert (table != null);
String mvTableName = table.getTypeName();
Table srcTable = table.getMaterializer();
if (srcTable == null) {
return false;
}
if (table.getIsreplicated()) {
return false;
}
// Justify whether partition column is in group by column list or not
if (table.getPartitioncolumn() != null) {
return false;
}
m_mvTableScan = mvTableScan;
Set<String> mvDDLGroupbyColumnNames = new HashSet<>();
List<Column> mvColumnArray = CatalogUtil.getSortedCatalogItems(table.getColumns(), "index");
String mvTableAlias = getMVTableAlias();
// Get the number of group-by columns.
int numOfGroupByColumns;
MaterializedViewInfo mvInfo = srcTable.getViews().get(mvTableName);
if (mvInfo != null) {
// single table view
String complexGroupbyJson = mvInfo.getGroupbyexpressionsjson();
if (complexGroupbyJson.length() > 0) {
List<AbstractExpression> mvComplexGroupbyCols = null;
try {
mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
} catch (JSONException e) {
e.printStackTrace();
}
numOfGroupByColumns = mvComplexGroupbyCols.size();
} else {
numOfGroupByColumns = mvInfo.getGroupbycols().size();
}
} else {
// joined table view
MaterializedViewHandlerInfo mvHandlerInfo = table.getMvhandlerinfo().get("mvHandlerInfo");
numOfGroupByColumns = mvHandlerInfo.getGroupbycolumncount();
}
if (scanColumns.isEmpty() && numOfGroupByColumns == 0) {
// This is an edge case that can happen if the view
// has no group by keys, and we are just
// doing a count(*) on the output of the view.
//
// Having no GB keys or scan columns would cause us to
// produce plan nodes that have a 0-column output schema.
// We can't handle this in several places, so add the
// count(*) column from the view to the scan columns.
// this is the "count(*)" column.
Column mvCol = mvColumnArray.get(0);
TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, 0);
tve.setOrigStmtId(mvTableScan.getStatementId());
String colName = mvCol.getName();
SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
scanColumns.add(scol);
}
// Start to do real materialized view processing to fix the duplicates problem.
// (1) construct new projection columns for scan plan node.
Set<SchemaColumn> mvDDLGroupbyColumns = new HashSet<>();
NodeSchema inlineProjSchema = new NodeSchema();
for (SchemaColumn scol : scanColumns) {
inlineProjSchema.addColumn(scol);
}
for (int i = 0; i < numOfGroupByColumns; i++) {
Column mvCol = mvColumnArray.get(i);
String colName = mvCol.getName();
TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, i);
tve.setOrigStmtId(mvTableScan.getStatementId());
mvDDLGroupbyColumnNames.add(colName);
SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
mvDDLGroupbyColumns.add(scol);
if (!scanColumns.contains(scol)) {
scanColumns.add(scol);
// construct new projection columns for scan plan node.
inlineProjSchema.addColumn(scol);
}
}
// Record the re-aggregation type for each scan columns.
Map<String, ExpressionType> mvColumnReAggType = new HashMap<>();
for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
Column mvCol = mvColumnArray.get(i);
ExpressionType reAggType = ExpressionType.get(mvCol.getAggregatetype());
if (reAggType == ExpressionType.AGGREGATE_COUNT_STAR || reAggType == ExpressionType.AGGREGATE_COUNT) {
reAggType = ExpressionType.AGGREGATE_SUM;
}
mvColumnReAggType.put(mvCol.getName(), reAggType);
}
assert (inlineProjSchema.size() > 0);
m_scanInlinedProjectionNode = new ProjectionPlanNode(inlineProjSchema);
// (2) Construct the reAggregation Node.
// Construct the reAggregation plan node's aggSchema
m_reAggNode = new HashAggregatePlanNode();
int outputColumnIndex = 0;
// inlineProjSchema contains the group by columns, while aggSchema may do not.
NodeSchema aggSchema = new NodeSchema();
// Construct reAggregation node's aggregation and group by list.
for (SchemaColumn scol : inlineProjSchema.getColumns()) {
if (mvDDLGroupbyColumns.contains(scol)) {
// Add group by expression.
m_reAggNode.addGroupByExpression(scol.getExpression());
} else {
ExpressionType reAggType = mvColumnReAggType.get(scol.getColumnName());
assert (reAggType != null);
AbstractExpression agg_input_expr = scol.getExpression();
assert (agg_input_expr instanceof TupleValueExpression);
// Add aggregation information.
m_reAggNode.addAggregate(reAggType, false, outputColumnIndex, agg_input_expr);
}
aggSchema.addColumn(scol);
outputColumnIndex++;
}
assert (aggSchema.size() > 0);
m_reAggNode.setOutputSchema(aggSchema);
// Collect all TVEs that need to be do re-aggregation in coordinator.
List<TupleValueExpression> needReAggTVEs = new ArrayList<>();
List<AbstractExpression> aggPostExprs = new ArrayList<>();
for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
Column mvCol = mvColumnArray.get(i);
TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, -1);
tve.setOrigStmtId(mvTableScan.getStatementId());
needReAggTVEs.add(tve);
}
collectReAggNodePostExpressions(joinTree, needReAggTVEs, aggPostExprs);
AbstractExpression aggPostExpr = ExpressionUtil.combinePredicates(aggPostExprs);
// Add post filters for the reAggregation node.
m_reAggNode.setPostPredicate(aggPostExpr);
// ENG-5386
if (m_edgeCaseQueryNoFixNeeded && edgeCaseQueryNoFixNeeded(mvDDLGroupbyColumnNames, mvColumnReAggType, displayColumns, groupByColumns)) {
return false;
}
m_needed = true;
return true;
}
use of org.voltdb.catalog.MaterializedViewHandlerInfo in project voltdb by VoltDB.
the class ViewExplainer method explain.
public static ArrayList<String[]> explain(Table viewTable) throws Exception {
String viewName = viewTable.getTypeName();
MaterializedViewHandlerInfo mvHandlerInfo = viewTable.getMvhandlerinfo().get("mvHandlerInfo");
MaterializedViewInfo mvInfo = null;
CatalogMap<Statement> fallBackQueryStmts;
List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(viewTable.getColumns(), "index");
ArrayList<String[]> retval = new ArrayList<String[]>();
// Is this view single-table?
if (mvHandlerInfo == null) {
// (Legacy code for single table view uses a different model and code path)
if (viewTable.getMaterializer() == null) {
// If we cannot find view metadata from both locations, this table is not a materialized view.
throw new Exception("Table " + viewName + " is not a view.");
}
mvInfo = viewTable.getMaterializer().getViews().get(viewName);
fallBackQueryStmts = mvInfo.getFallbackquerystmts();
} else {
// For multi-table views we need to show the query plan for evaluating joins.
Statement createQuery = mvHandlerInfo.getCreatequery().get("createQuery");
retval.add(new String[] { "Join Evaluation", Encoder.hexDecodeToString(createQuery.getExplainplan()) });
fallBackQueryStmts = mvHandlerInfo.getFallbackquerystmts();
}
// For each min/max column find out if an execution plan is used.
int minMaxAggIdx = 0;
for (int j = 0; j < destColumnArray.size(); j++) {
Column destColumn = destColumnArray.get(j);
ExpressionType aggType = ExpressionType.get(destColumn.getAggregatetype());
if (aggType == ExpressionType.AGGREGATE_MIN || aggType == ExpressionType.AGGREGATE_MAX) {
Statement fallBackQueryStmt = fallBackQueryStmts.get(String.valueOf(minMaxAggIdx));
// How this min/max will be refreshed?
String plan = "";
// * execution plan
if (mvHandlerInfo == null) {
CatalogMap<IndexRef> hardCodedIndicesForSingleTableView = mvInfo.getIndexforminmax();
String hardCodedIndexName = hardCodedIndicesForSingleTableView.get(String.valueOf(minMaxAggIdx)).getName();
String indexNameUsedInStatement = getIndexNameUsedInStatement(fallBackQueryStmt);
if (!indexNameUsedInStatement.equalsIgnoreCase(hardCodedIndexName)) {
plan = Encoder.hexDecodeToString(fallBackQueryStmt.getExplainplan());
}
// If we do not use execution plan, see which built-in method is used.
if (plan.equals("")) {
if (hardCodedIndexName.equals("")) {
plan = "Built-in sequential scan.";
} else {
plan = "Built-in index scan \"" + hardCodedIndexName + "\".";
}
}
} else {
plan = Encoder.hexDecodeToString(fallBackQueryStmt.getExplainplan());
}
retval.add(new String[] { "Refresh " + (aggType == ExpressionType.AGGREGATE_MIN ? "MIN" : "MAX") + " column \"" + destColumn.getName() + "\"", plan });
minMaxAggIdx++;
}
}
return retval;
}
use of org.voltdb.catalog.MaterializedViewHandlerInfo in project voltdb by VoltDB.
the class MaterializedViewProcessor method startProcessing.
/**
* Add materialized view info to the catalog for the tables that are
* materialized views.
* @throws VoltCompilerException
*/
public void startProcessing(Database db, HashMap<Table, String> matViewMap, TreeSet<String> exportTableNames) throws VoltCompilerException {
HashSet<String> viewTableNames = new HashSet<>();
for (Entry<Table, String> entry : matViewMap.entrySet()) {
viewTableNames.add(entry.getKey().getTypeName());
}
for (Entry<Table, String> entry : matViewMap.entrySet()) {
Table destTable = entry.getKey();
String query = entry.getValue();
// get the xml for the query
VoltXMLElement xmlquery = null;
try {
xmlquery = m_hsql.getXMLCompiledStatement(query);
} catch (HSQLParseException e) {
e.printStackTrace();
}
assert (xmlquery != null);
// parse the xml like any other sql statement
ParsedSelectStmt stmt = null;
try {
stmt = (ParsedSelectStmt) AbstractParsedStmt.parse(query, xmlquery, null, db, null);
} catch (Exception e) {
throw m_compiler.new VoltCompilerException(e.getMessage());
}
assert (stmt != null);
String viewName = destTable.getTypeName();
// throw an error if the view isn't within voltdb's limited world view
checkViewMeetsSpec(viewName, stmt);
// The primary key index is yet to be defined (below).
for (Index destIndex : destTable.getIndexes()) {
if (destIndex.getUnique() || destIndex.getAssumeunique()) {
String msg = "A UNIQUE or ASSUMEUNIQUE index is not allowed on a materialized view. " + "Remove the qualifier from the index " + destIndex.getTypeName() + "defined on the materialized view \"" + viewName + "\".";
throw m_compiler.new VoltCompilerException(msg);
}
}
// A Materialized view cannot depend on another view.
for (Table srcTable : stmt.m_tableList) {
if (viewTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) can not be defined on another view (%s).", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
}
// The existing code base still need this materializer field to tell if a table
// is a materialized view table. Leaving this for future refactoring.
destTable.setMaterializer(stmt.m_tableList.get(0));
List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(destTable.getColumns(), "index");
List<AbstractExpression> groupbyExprs = null;
if (stmt.hasComplexGroupby()) {
groupbyExprs = new ArrayList<>();
for (ParsedColInfo col : stmt.groupByColumns()) {
groupbyExprs.add(col.expression);
}
}
// Generate query XMLs for min/max recalculation (ENG-8641)
boolean isMultiTableView = stmt.m_tableList.size() > 1;
MatViewFallbackQueryXMLGenerator xmlGen = new MatViewFallbackQueryXMLGenerator(xmlquery, stmt.groupByColumns(), stmt.m_displayColumns, isMultiTableView);
List<VoltXMLElement> fallbackQueryXMLs = xmlGen.getFallbackQueryXMLs();
// index or constraint in order to avoid error and crash.
if (stmt.groupByColumns().size() != 0) {
Index pkIndex = destTable.getIndexes().add(HSQLInterface.AUTO_GEN_MATVIEW_IDX);
pkIndex.setType(IndexType.BALANCED_TREE.getValue());
pkIndex.setUnique(true);
// assume index 1 throuh #grpByCols + 1 are the cols
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ColumnRef c = pkIndex.getColumns().add(String.valueOf(i));
c.setColumn(destColumnArray.get(i));
c.setIndex(i);
}
Constraint pkConstraint = destTable.getConstraints().add(HSQLInterface.AUTO_GEN_MATVIEW_CONST);
pkConstraint.setType(ConstraintType.PRIMARY_KEY.getValue());
pkConstraint.setIndex(pkIndex);
}
// If we have an unsafe MV message, then
// remember it here. We don't really know how
// to transfer the message through the catalog, but
// we can transmit the existence of the message.
boolean isSafeForDDL = (stmt.getUnsafeMVMessage() == null);
// Here the code path diverges for different kinds of views (single table view and joined table view)
if (isMultiTableView) {
// Materialized view on joined tables
// Add mvHandlerInfo to the destTable:
MaterializedViewHandlerInfo mvHandlerInfo = destTable.getMvhandlerinfo().add("mvHandlerInfo");
mvHandlerInfo.setDesttable(destTable);
for (Table srcTable : stmt.m_tableList) {
// Now we do not support having a view on persistent tables joining streamed tables.
if (exportTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) on joined tables cannot have streamed table (%s) as its source.", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
// The view table will need to keep a list of its source tables.
// The list is used to install / uninstall the view reference on the source tables when the
// view handler is constructed / destroyed.
TableRef tableRef = mvHandlerInfo.getSourcetables().add(srcTable.getTypeName());
tableRef.setTable(srcTable);
// There could be more than one partition column candidate, but we will only use the first one we found.
if (destTable.getPartitioncolumn() == null && srcTable.getPartitioncolumn() != null) {
Column partitionColumn = srcTable.getPartitioncolumn();
String partitionColName = partitionColumn.getTypeName();
String srcTableName = srcTable.getTypeName();
destTable.setIsreplicated(false);
if (stmt.hasComplexGroupby()) {
for (int i = 0; i < groupbyExprs.size(); i++) {
AbstractExpression groupbyExpr = groupbyExprs.get(i);
if (groupbyExpr instanceof TupleValueExpression) {
TupleValueExpression tve = (TupleValueExpression) groupbyExpr;
if (tve.getTableName().equals(srcTableName) && tve.getColumnName().equals(partitionColName)) {
// The partition column is set to destColumnArray.get(i), because we have the restriction
// that the non-aggregate columns must come at the very begining, and must exactly match
// the group-by columns.
// If we are going to remove this restriction in the future, then we need to do more work
// in order to find a proper partition column.
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
} else {
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
if (gbcol.tableName.equals(srcTableName) && gbcol.columnName.equals(partitionColName)) {
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
}
// end find partition column
}
// end for each source table
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, mvHandlerInfo);
compileCreateQueryAndUpdateCatalog(db, query, xmlquery, mvHandlerInfo);
mvHandlerInfo.setGroupbycolumncount(stmt.groupByColumns().size());
for (int i = 0; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
// Set the expression type here to determine the behavior of the merge function.
destColumn.setAggregatetype(col.expression.getExpressionType().getValue());
}
mvHandlerInfo.setIssafewithnonemptysources(isSafeForDDL);
} else {
// =======================================================================================
// Materialized view on single table
// create the materializedviewinfo catalog node for the source table
Table srcTable = stmt.m_tableList.get(0);
MaterializedViewInfo matviewinfo = srcTable.getViews().add(viewName);
matviewinfo.setDest(destTable);
AbstractExpression where = stmt.getSingleTableFilterExpression();
if (where != null) {
String hex = Encoder.hexEncode(where.toJSONString());
matviewinfo.setPredicate(hex);
} else {
matviewinfo.setPredicate("");
}
List<Column> srcColumnArray = CatalogUtil.getSortedCatalogItems(srcTable.getColumns(), "index");
if (stmt.hasComplexGroupby()) {
// Parse group by expressions to json string
String groupbyExprsJson = null;
try {
groupbyExprsJson = DDLCompiler.convertToJSONArray(groupbyExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for group by expressions: " + e.toString());
}
matviewinfo.setGroupbyexpressionsjson(groupbyExprsJson);
} else {
// add the group by columns from the src table
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
Column srcCol = srcColumnArray.get(gbcol.index);
ColumnRef cref = matviewinfo.getGroupbycols().add(srcCol.getTypeName());
// groupByColumns is iterating in order of groups. Store that grouping order
// in the column ref index. When the catalog is serialized, it will, naturally,
// scramble this order like a two year playing dominos, presenting the data
// in a meaningless sequence.
// the column offset in the view's grouping order
cref.setIndex(i);
// the source column from the base (non-view) table
cref.setColumn(srcCol);
// parse out the group by columns into the dest table
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
processMaterializedViewColumn(srcTable, destColumn, ExpressionType.VALUE_TUPLE, (TupleValueExpression) col.expression);
}
}
// Set up COUNT(*) column
ParsedColInfo countCol = stmt.m_displayColumns.get(stmt.groupByColumns().size());
assert (countCol.expression.getExpressionType() == ExpressionType.AGGREGATE_COUNT_STAR);
assert (countCol.expression.getLeft() == null);
processMaterializedViewColumn(srcTable, destColumnArray.get(stmt.groupByColumns().size()), ExpressionType.AGGREGATE_COUNT_STAR, null);
// prepare info for aggregation columns.
List<AbstractExpression> aggregationExprs = new ArrayList<>();
boolean hasAggregationExprs = false;
ArrayList<AbstractExpression> minMaxAggs = new ArrayList<>();
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
AbstractExpression aggExpr = col.expression.getLeft();
if (aggExpr.getExpressionType() != ExpressionType.VALUE_TUPLE) {
hasAggregationExprs = true;
}
aggregationExprs.add(aggExpr);
if (col.expression.getExpressionType() == ExpressionType.AGGREGATE_MIN || col.expression.getExpressionType() == ExpressionType.AGGREGATE_MAX) {
minMaxAggs.add(aggExpr);
}
}
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, matviewinfo);
// set Aggregation Expressions.
if (hasAggregationExprs) {
String aggregationExprsJson = null;
try {
aggregationExprsJson = DDLCompiler.convertToJSONArray(aggregationExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for aggregation expressions: " + e.toString());
}
matviewinfo.setAggregationexpressionsjson(aggregationExprsJson);
}
// Find index for each min/max aggCol/aggExpr (ENG-6511 and ENG-8512)
for (Integer i = 0; i < minMaxAggs.size(); ++i) {
Index found = findBestMatchIndexForMatviewMinOrMax(matviewinfo, srcTable, groupbyExprs, minMaxAggs.get(i));
IndexRef refFound = matviewinfo.getIndexforminmax().add(i.toString());
if (found != null) {
refFound.setName(found.getTypeName());
} else {
refFound.setName("");
}
}
// The COUNT(*) should return a BIGINT column, whereas we found here the COUNT(*) was assigned a INTEGER column.
for (int i = 0; i <= stmt.groupByColumns().size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
}
// parse out the aggregation columns into the dest table
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
AbstractExpression colExpr = col.expression.getLeft();
TupleValueExpression tve = null;
if (colExpr.getExpressionType() == ExpressionType.VALUE_TUPLE) {
tve = (TupleValueExpression) colExpr;
}
processMaterializedViewColumn(srcTable, destColumn, col.expression.getExpressionType(), tve);
setTypeAttributesForColumn(destColumn, col.expression);
}
if (srcTable.getPartitioncolumn() != null) {
// Set the partitioning of destination tables of associated views.
// If a view's source table is replicated, then a full scan of the
// associated view is single-sited. If the source is partitioned,
// a full scan of the view must be distributed, unless it is filtered
// by the original table's partitioning key, which, to be filtered,
// must also be a GROUP BY key.
destTable.setIsreplicated(false);
setGroupedTablePartitionColumn(matviewinfo, srcTable.getPartitioncolumn());
}
matviewinfo.setIssafewithnonemptysources(isSafeForDDL);
}
// end if single table view materialized view.
}
}
Aggregations