use of org.voltdb.catalog.ColumnRef in project voltdb by VoltDB.
the class IndexScanPlanNode method isOutputOrdered.
@Override
public boolean isOutputOrdered(List<AbstractExpression> sortExpressions, List<SortDirectionType> sortDirections) {
assert (sortExpressions.size() == sortDirections.size());
// The output is unordered if there is an inline hash aggregate
AbstractPlanNode agg = AggregatePlanNode.getInlineAggregationNode(this);
if (agg != null && agg.getPlanNodeType() == PlanNodeType.HASHAGGREGATE) {
return false;
}
// Verify that all sortDirections match
for (SortDirectionType sortDirection : sortDirections) {
if (sortDirection != getSortDirection()) {
return false;
}
}
// Verify that all sort expressions are covered by the consecutive index expressions
// starting from the first one
List<AbstractExpression> indexedExprs = new ArrayList<>();
List<ColumnRef> indexedColRefs = new ArrayList<>();
boolean columnIndex = CatalogUtil.getCatalogIndexExpressions(getCatalogIndex(), getTableScan(), indexedExprs, indexedColRefs);
int indexExprCount = (columnIndex) ? indexedColRefs.size() : indexedExprs.size();
if (indexExprCount < sortExpressions.size()) {
// Not enough index expressions to cover all of the sort expressions
return false;
}
if (columnIndex) {
for (int idxToCover = 0; idxToCover < sortExpressions.size(); ++idxToCover) {
AbstractExpression sortExpression = sortExpressions.get(idxToCover);
if (!isSortExpressionCovered(sortExpression, indexedColRefs, idxToCover, getTableScan())) {
return false;
}
}
} else {
for (int idxToCover = 0; idxToCover < sortExpressions.size(); ++idxToCover) {
AbstractExpression sortExpression = sortExpressions.get(idxToCover);
if (!isSortExpressionCovered(sortExpression, indexedExprs, idxToCover)) {
return false;
}
}
}
return true;
}
use of org.voltdb.catalog.ColumnRef in project voltdb by VoltDB.
the class IndexScanPlanNode method explainPlanForNode.
@Override
protected String explainPlanForNode(String indent) {
assert (m_catalogIndex != null);
int keySize = m_searchkeyExpressions.size();
// When there is no start key, count a range scan key for each ANDed end condition.
if (keySize == 0 && m_endExpression != null) {
keySize = ExpressionUtil.uncombineAny(m_endExpression).size();
}
String usageInfo;
String predicatePrefix;
if (keySize == 0) {
// -- either for determinism or for an explicit ORDER BY requirement.
if (m_purpose == FOR_DETERMINISM) {
usageInfo = " (for deterministic order only)";
} else if (m_purpose == FOR_GROUPING) {
usageInfo = " (for optimized grouping only)";
} else {
usageInfo = " (for sort order only)";
}
// Introduce on its own indented line, any unrelated post-filter applied to the result.
// e.g. " filter by OTHER_COL = 1"
predicatePrefix = "\n" + indent + " filter by ";
} else {
int indexSize = CatalogUtil.getCatalogIndexSize(m_catalogIndex);
String[] asIndexed = new String[indexSize];
// they beat an NPE.
for (int ii = 0; ii < keySize; ++ii) {
asIndexed[ii] = "(index key " + ii + ")";
}
String jsonExpr = m_catalogIndex.getExpressionsjson();
// if this is a pure-column index...
if (jsonExpr.isEmpty()) {
// grab the short names of the indexed columns in use.
for (ColumnRef cref : m_catalogIndex.getColumns()) {
Column col = cref.getColumn();
asIndexed[cref.getIndex()] = col.getName();
}
} else {
try {
List<AbstractExpression> indexExpressions = AbstractExpression.fromJSONArrayString(jsonExpr, m_tableScan);
int ii = 0;
for (AbstractExpression ae : indexExpressions) {
asIndexed[ii++] = ae.explain(getTableNameForExplain());
}
} catch (JSONException e) {
// If something unexpected went wrong,
// just fall back on the positional key labels.
}
}
// Explain the search criteria that describe the start of the index scan, like
// "(event_type = 1 AND event_start > x.start_time)"
String start = explainSearchKeys(asIndexed, keySize);
if (m_lookupType == IndexLookupType.EQ) {
// " scan matches for (event_type = 1) AND (event_location = x.region)"
if (m_catalogIndex.getUnique()) {
usageInfo = "\n" + indent + " uniquely match " + start;
} else {
usageInfo = "\n" + indent + " scan matches for " + start;
}
} else if (m_lookupType == IndexLookupType.GEO_CONTAINS) {
usageInfo = "\n" + indent + " scan for " + start;
} else {
usageInfo = "\n" + indent;
if (isReverseScan()) {
usageInfo += "reverse ";
}
// " " range-scan on 1 of 2 cols from event_type = 1"
if (indexSize == keySize) {
usageInfo += "range-scan covering from " + start;
} else {
usageInfo += String.format("range-scan on %d of %d cols from %s", keySize, indexSize, start);
}
// Explain the criteria for continuinuing the scan such as
// "while (event_type = 1 AND event_start < x.start_time+30)"
// or label it as a scan "to the end"
usageInfo += explainEndKeys();
}
// Introduce any additional filters not related to the index
// that could cause rows to be skipped.
// e.g. "... scan ... from ... while ..., filter by OTHER_COL = 1"
predicatePrefix = ", filter by ";
}
// Describe any additional filters not related to the index
// e.g. "...filter by OTHER_COL = 1".
String predicate = explainPredicate(predicatePrefix);
// Describe the table name and either a user-provided name of the index or
// its user-specified role ("primary key").
String tableName = m_targetTableName;
if (m_targetTableAlias != null && !m_targetTableAlias.equals(m_targetTableName)) {
tableName += " (" + m_targetTableAlias + ")";
}
String retval = "INDEX SCAN of \"" + tableName + "\"";
String indexDescription = " using \"" + m_targetIndexName + "\"";
// Replace ugly system-generated index name with a description of its user-specified role.
if (m_targetIndexName.startsWith(HSQLInterface.AUTO_GEN_PRIMARY_KEY_PREFIX) || m_targetIndexName.startsWith(HSQLInterface.AUTO_GEN_NAMED_CONSTRAINT_IDX) || m_targetIndexName.equals(HSQLInterface.AUTO_GEN_MATVIEW_IDX)) {
indexDescription = " using its primary key index";
}
// Bring all the pieces together describing the index, how it is scanned,
// and whatever extra filter processing is done to the result.
retval += indexDescription;
retval += usageInfo + predicate;
return retval;
}
use of org.voltdb.catalog.ColumnRef in project voltdb by VoltDB.
the class CatalogSizing method getIndexSize.
private static CatalogItemSizeBase getIndexSize(Index index) {
// this is sizeof(CompactingMap::TreeNode), not counting template parameter KeyValuePair.
final long TREE_MAP_ENTRY_OVERHEAD = 32;
final long TUPLE_PTR_SIZE = 8;
// All index types consume the space taken by the column data,
// except that 8 byte pointers references replace large var... data.
// Additional overhead is determined by the index type.
CatalogMap<ColumnRef> columnRefsMap = index.getColumns();
List<Column> indexColumns = new ArrayList<Column>(columnRefsMap.size());
for (ColumnRef columnRef : columnRefsMap) {
indexColumns.add(columnRef.getColumn());
}
//For index Size dont count the DR AA conflict column.
CatalogItemSizeBase isize = getColumnsSize(indexColumns, true, false);
if (index.getType() == IndexType.HASH_TABLE.getValue()) {
// Hash index overhead follows this documented formula:
// w=column width, r=row count
// (((2 * r) + 1) * 8) + ((w + 32) * r)
// This can be reduced to the following:
// (w + 48) * r + 8
// For approximation purposes the constant +8 is ignorable.
isize.widthMin += 48;
isize.widthMax += 48;
} else if (index.getType() == IndexType.COVERING_CELL_INDEX.getValue()) {
// Covering cell indexes are implemented in the EE with two maps:
//
// [1 entry per table row] tuple address -> fixed-size array of 8 cell ids
// [1-8 entries per table row] cell id -> tuple address
//
// The polygon value is not referenced at all in the index, just the tuple address.
// The call to getColumnsSize above purposely omits the size of the pointer to
// the geography value for this reason.
//
// Other columns in the index are included, so if in the future we decide to support
// multi-component geospatial indexes to optimize predicates like
// "WHERE id = 10 and contains(geog, ?)", then this code would not need to change.
final long MIN_CELLS = 1;
final long MAX_CELLS = 8;
final long CELL_SIZE = 8;
final long TUPLE_MAP_ENTRY = TREE_MAP_ENTRY_OVERHEAD + TUPLE_PTR_SIZE + MAX_CELLS * CELL_SIZE;
final long CELL_MAP_ENTRY = TREE_MAP_ENTRY_OVERHEAD + CELL_SIZE + TUPLE_PTR_SIZE;
isize.widthMin += TUPLE_MAP_ENTRY + MIN_CELLS * CELL_MAP_ENTRY;
isize.widthMax += TUPLE_MAP_ENTRY + MAX_CELLS * CELL_MAP_ENTRY;
} else {
// Tree indexes have a 40 byte overhead per row.
isize.widthMin += TREE_MAP_ENTRY_OVERHEAD + TUPLE_PTR_SIZE;
isize.widthMax += TREE_MAP_ENTRY_OVERHEAD + TUPLE_PTR_SIZE;
}
return isize;
}
use of org.voltdb.catalog.ColumnRef in project voltdb by VoltDB.
the class DefaultProcedureManager method addShimProcedure.
private void addShimProcedure(String name, Table table, Constraint pkey, boolean tableCols, int partitionParamIndex, Column partitionColumn, boolean readOnly) {
Procedure proc = m_fakeDb.getProcedures().add(name);
proc.setClassname(name);
proc.setDefaultproc(true);
proc.setHasjava(false);
proc.setHasseqscans(false);
proc.setSinglepartition(partitionParamIndex >= 0);
proc.setPartitioncolumn(partitionColumn);
proc.setPartitionparameter(partitionParamIndex);
proc.setReadonly(readOnly);
proc.setEverysite(false);
proc.setSystemproc(false);
proc.setPartitiontable(table);
if (partitionParamIndex >= 0) {
proc.setAttachment(new ProcedurePartitionInfo(VoltType.get((byte) partitionColumn.getType()), partitionParamIndex));
}
int paramCount = 0;
if (tableCols) {
for (Column col : table.getColumns()) {
// name each parameter "param1", "param2", etc...
ProcParameter procParam = proc.getParameters().add("param" + String.valueOf(paramCount));
procParam.setIndex(col.getIndex());
procParam.setIsarray(false);
procParam.setType(col.getType());
paramCount++;
}
}
if (pkey != null) {
CatalogMap<ColumnRef> pkeycols = pkey.getIndex().getColumns();
int paramCount2 = paramCount;
for (ColumnRef cref : pkeycols) {
// name each parameter "param1", "param2", etc...
ProcParameter procParam = proc.getParameters().add("param" + String.valueOf(paramCount2));
procParam.setIndex(cref.getIndex() + paramCount);
procParam.setIsarray(false);
procParam.setType(cref.getColumn().getType());
paramCount2++;
}
}
m_defaultProcMap.put(name.toLowerCase(), proc);
}
use of org.voltdb.catalog.ColumnRef in project voltdb by VoltDB.
the class MaterializedViewProcessor method startProcessing.
/**
* Add materialized view info to the catalog for the tables that are
* materialized views.
* @throws VoltCompilerException
*/
public void startProcessing(Database db, HashMap<Table, String> matViewMap, TreeSet<String> exportTableNames) throws VoltCompilerException {
HashSet<String> viewTableNames = new HashSet<>();
for (Entry<Table, String> entry : matViewMap.entrySet()) {
viewTableNames.add(entry.getKey().getTypeName());
}
for (Entry<Table, String> entry : matViewMap.entrySet()) {
Table destTable = entry.getKey();
String query = entry.getValue();
// get the xml for the query
VoltXMLElement xmlquery = null;
try {
xmlquery = m_hsql.getXMLCompiledStatement(query);
} catch (HSQLParseException e) {
e.printStackTrace();
}
assert (xmlquery != null);
// parse the xml like any other sql statement
ParsedSelectStmt stmt = null;
try {
stmt = (ParsedSelectStmt) AbstractParsedStmt.parse(query, xmlquery, null, db, null);
} catch (Exception e) {
throw m_compiler.new VoltCompilerException(e.getMessage());
}
assert (stmt != null);
String viewName = destTable.getTypeName();
// throw an error if the view isn't within voltdb's limited world view
checkViewMeetsSpec(viewName, stmt);
// The primary key index is yet to be defined (below).
for (Index destIndex : destTable.getIndexes()) {
if (destIndex.getUnique() || destIndex.getAssumeunique()) {
String msg = "A UNIQUE or ASSUMEUNIQUE index is not allowed on a materialized view. " + "Remove the qualifier from the index " + destIndex.getTypeName() + "defined on the materialized view \"" + viewName + "\".";
throw m_compiler.new VoltCompilerException(msg);
}
}
// A Materialized view cannot depend on another view.
for (Table srcTable : stmt.m_tableList) {
if (viewTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) can not be defined on another view (%s).", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
}
// The existing code base still need this materializer field to tell if a table
// is a materialized view table. Leaving this for future refactoring.
destTable.setMaterializer(stmt.m_tableList.get(0));
List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(destTable.getColumns(), "index");
List<AbstractExpression> groupbyExprs = null;
if (stmt.hasComplexGroupby()) {
groupbyExprs = new ArrayList<>();
for (ParsedColInfo col : stmt.groupByColumns()) {
groupbyExprs.add(col.expression);
}
}
// Generate query XMLs for min/max recalculation (ENG-8641)
boolean isMultiTableView = stmt.m_tableList.size() > 1;
MatViewFallbackQueryXMLGenerator xmlGen = new MatViewFallbackQueryXMLGenerator(xmlquery, stmt.groupByColumns(), stmt.m_displayColumns, isMultiTableView);
List<VoltXMLElement> fallbackQueryXMLs = xmlGen.getFallbackQueryXMLs();
// index or constraint in order to avoid error and crash.
if (stmt.groupByColumns().size() != 0) {
Index pkIndex = destTable.getIndexes().add(HSQLInterface.AUTO_GEN_MATVIEW_IDX);
pkIndex.setType(IndexType.BALANCED_TREE.getValue());
pkIndex.setUnique(true);
// assume index 1 throuh #grpByCols + 1 are the cols
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ColumnRef c = pkIndex.getColumns().add(String.valueOf(i));
c.setColumn(destColumnArray.get(i));
c.setIndex(i);
}
Constraint pkConstraint = destTable.getConstraints().add(HSQLInterface.AUTO_GEN_MATVIEW_CONST);
pkConstraint.setType(ConstraintType.PRIMARY_KEY.getValue());
pkConstraint.setIndex(pkIndex);
}
// If we have an unsafe MV message, then
// remember it here. We don't really know how
// to transfer the message through the catalog, but
// we can transmit the existence of the message.
boolean isSafeForDDL = (stmt.getUnsafeMVMessage() == null);
// Here the code path diverges for different kinds of views (single table view and joined table view)
if (isMultiTableView) {
// Materialized view on joined tables
// Add mvHandlerInfo to the destTable:
MaterializedViewHandlerInfo mvHandlerInfo = destTable.getMvhandlerinfo().add("mvHandlerInfo");
mvHandlerInfo.setDesttable(destTable);
for (Table srcTable : stmt.m_tableList) {
// Now we do not support having a view on persistent tables joining streamed tables.
if (exportTableNames.contains(srcTable.getTypeName())) {
String msg = String.format("A materialized view (%s) on joined tables cannot have streamed table (%s) as its source.", viewName, srcTable.getTypeName());
throw m_compiler.new VoltCompilerException(msg);
}
// The view table will need to keep a list of its source tables.
// The list is used to install / uninstall the view reference on the source tables when the
// view handler is constructed / destroyed.
TableRef tableRef = mvHandlerInfo.getSourcetables().add(srcTable.getTypeName());
tableRef.setTable(srcTable);
// There could be more than one partition column candidate, but we will only use the first one we found.
if (destTable.getPartitioncolumn() == null && srcTable.getPartitioncolumn() != null) {
Column partitionColumn = srcTable.getPartitioncolumn();
String partitionColName = partitionColumn.getTypeName();
String srcTableName = srcTable.getTypeName();
destTable.setIsreplicated(false);
if (stmt.hasComplexGroupby()) {
for (int i = 0; i < groupbyExprs.size(); i++) {
AbstractExpression groupbyExpr = groupbyExprs.get(i);
if (groupbyExpr instanceof TupleValueExpression) {
TupleValueExpression tve = (TupleValueExpression) groupbyExpr;
if (tve.getTableName().equals(srcTableName) && tve.getColumnName().equals(partitionColName)) {
// The partition column is set to destColumnArray.get(i), because we have the restriction
// that the non-aggregate columns must come at the very begining, and must exactly match
// the group-by columns.
// If we are going to remove this restriction in the future, then we need to do more work
// in order to find a proper partition column.
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
} else {
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
if (gbcol.tableName.equals(srcTableName) && gbcol.columnName.equals(partitionColName)) {
destTable.setPartitioncolumn(destColumnArray.get(i));
break;
}
}
}
}
// end find partition column
}
// end for each source table
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, mvHandlerInfo);
compileCreateQueryAndUpdateCatalog(db, query, xmlquery, mvHandlerInfo);
mvHandlerInfo.setGroupbycolumncount(stmt.groupByColumns().size());
for (int i = 0; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
// Set the expression type here to determine the behavior of the merge function.
destColumn.setAggregatetype(col.expression.getExpressionType().getValue());
}
mvHandlerInfo.setIssafewithnonemptysources(isSafeForDDL);
} else {
// =======================================================================================
// Materialized view on single table
// create the materializedviewinfo catalog node for the source table
Table srcTable = stmt.m_tableList.get(0);
MaterializedViewInfo matviewinfo = srcTable.getViews().add(viewName);
matviewinfo.setDest(destTable);
AbstractExpression where = stmt.getSingleTableFilterExpression();
if (where != null) {
String hex = Encoder.hexEncode(where.toJSONString());
matviewinfo.setPredicate(hex);
} else {
matviewinfo.setPredicate("");
}
List<Column> srcColumnArray = CatalogUtil.getSortedCatalogItems(srcTable.getColumns(), "index");
if (stmt.hasComplexGroupby()) {
// Parse group by expressions to json string
String groupbyExprsJson = null;
try {
groupbyExprsJson = DDLCompiler.convertToJSONArray(groupbyExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for group by expressions: " + e.toString());
}
matviewinfo.setGroupbyexpressionsjson(groupbyExprsJson);
} else {
// add the group by columns from the src table
for (int i = 0; i < stmt.groupByColumns().size(); i++) {
ParsedColInfo gbcol = stmt.groupByColumns().get(i);
Column srcCol = srcColumnArray.get(gbcol.index);
ColumnRef cref = matviewinfo.getGroupbycols().add(srcCol.getTypeName());
// groupByColumns is iterating in order of groups. Store that grouping order
// in the column ref index. When the catalog is serialized, it will, naturally,
// scramble this order like a two year playing dominos, presenting the data
// in a meaningless sequence.
// the column offset in the view's grouping order
cref.setIndex(i);
// the source column from the base (non-view) table
cref.setColumn(srcCol);
// parse out the group by columns into the dest table
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
processMaterializedViewColumn(srcTable, destColumn, ExpressionType.VALUE_TUPLE, (TupleValueExpression) col.expression);
}
}
// Set up COUNT(*) column
ParsedColInfo countCol = stmt.m_displayColumns.get(stmt.groupByColumns().size());
assert (countCol.expression.getExpressionType() == ExpressionType.AGGREGATE_COUNT_STAR);
assert (countCol.expression.getLeft() == null);
processMaterializedViewColumn(srcTable, destColumnArray.get(stmt.groupByColumns().size()), ExpressionType.AGGREGATE_COUNT_STAR, null);
// prepare info for aggregation columns.
List<AbstractExpression> aggregationExprs = new ArrayList<>();
boolean hasAggregationExprs = false;
ArrayList<AbstractExpression> minMaxAggs = new ArrayList<>();
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
AbstractExpression aggExpr = col.expression.getLeft();
if (aggExpr.getExpressionType() != ExpressionType.VALUE_TUPLE) {
hasAggregationExprs = true;
}
aggregationExprs.add(aggExpr);
if (col.expression.getExpressionType() == ExpressionType.AGGREGATE_MIN || col.expression.getExpressionType() == ExpressionType.AGGREGATE_MAX) {
minMaxAggs.add(aggExpr);
}
}
compileFallbackQueriesAndUpdateCatalog(db, query, fallbackQueryXMLs, matviewinfo);
// set Aggregation Expressions.
if (hasAggregationExprs) {
String aggregationExprsJson = null;
try {
aggregationExprsJson = DDLCompiler.convertToJSONArray(aggregationExprs);
} catch (JSONException e) {
throw m_compiler.new VoltCompilerException("Unexpected error serializing non-column " + "expressions for aggregation expressions: " + e.toString());
}
matviewinfo.setAggregationexpressionsjson(aggregationExprsJson);
}
// Find index for each min/max aggCol/aggExpr (ENG-6511 and ENG-8512)
for (Integer i = 0; i < minMaxAggs.size(); ++i) {
Index found = findBestMatchIndexForMatviewMinOrMax(matviewinfo, srcTable, groupbyExprs, minMaxAggs.get(i));
IndexRef refFound = matviewinfo.getIndexforminmax().add(i.toString());
if (found != null) {
refFound.setName(found.getTypeName());
} else {
refFound.setName("");
}
}
// The COUNT(*) should return a BIGINT column, whereas we found here the COUNT(*) was assigned a INTEGER column.
for (int i = 0; i <= stmt.groupByColumns().size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
setTypeAttributesForColumn(destColumn, col.expression);
}
// parse out the aggregation columns into the dest table
for (int i = stmt.groupByColumns().size() + 1; i < stmt.m_displayColumns.size(); i++) {
ParsedColInfo col = stmt.m_displayColumns.get(i);
Column destColumn = destColumnArray.get(i);
AbstractExpression colExpr = col.expression.getLeft();
TupleValueExpression tve = null;
if (colExpr.getExpressionType() == ExpressionType.VALUE_TUPLE) {
tve = (TupleValueExpression) colExpr;
}
processMaterializedViewColumn(srcTable, destColumn, col.expression.getExpressionType(), tve);
setTypeAttributesForColumn(destColumn, col.expression);
}
if (srcTable.getPartitioncolumn() != null) {
// Set the partitioning of destination tables of associated views.
// If a view's source table is replicated, then a full scan of the
// associated view is single-sited. If the source is partitioned,
// a full scan of the view must be distributed, unless it is filtered
// by the original table's partitioning key, which, to be filtered,
// must also be a GROUP BY key.
destTable.setIsreplicated(false);
setGroupedTablePartitionColumn(matviewinfo, srcTable.getPartitioncolumn());
}
matviewinfo.setIssafewithnonemptysources(isSafeForDDL);
}
// end if single table view materialized view.
}
}
Aggregations