use of herddb.model.planner.JoinOp in project herddb by diennea.
the class CalcitePlanner method planEnumerableJoin.
private PlannerOp planEnumerableJoin(EnumerableJoin op, RelDataType rowType) {
// please note that EnumerableJoin has a condition field which actually is not useful
PlannerOp left = convertRelNode(op.getLeft(), null, false);
PlannerOp right = convertRelNode(op.getRight(), null, false);
int[] leftKeys = op.getLeftKeys().toIntArray();
int[] rightKeys = op.getRightKeys().toIntArray();
boolean generateNullsOnLeft = op.getJoinType().generatesNullsOnLeft();
boolean generateNullsOnRight = op.getJoinType().generatesNullsOnRight();
final RelDataType _rowType = rowType == null ? op.getRowType() : rowType;
List<RelDataTypeField> fieldList = _rowType.getFieldList();
Column[] columns = new Column[fieldList.size()];
String[] fieldNames = new String[columns.length];
int i = 0;
for (RelDataTypeField field : fieldList) {
Column col = Column.column(field.getName().toLowerCase(), convertToHerdType(field.getType()));
fieldNames[i] = col.name;
columns[i++] = col;
}
return new JoinOp(fieldNames, columns, leftKeys, left, rightKeys, right, generateNullsOnLeft, generateNullsOnRight, false);
}
use of herddb.model.planner.JoinOp in project herddb by diennea.
the class SimpleJoinTest method testStartTransactionInJoinBranch.
@Test
public void testStartTransactionInJoinBranch() throws Exception {
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.waitForTablespace("tblspace1", 10000);
execute(manager, "CREATE TABLE tblspace1.table1 (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
execute(manager, "CREATE TABLE tblspace1.table2 (k2 string primary key,n2 int,s2 string)", Collections.emptyList());
execute(manager, "INSERT INTO tblspace1.table1 (k1,n1,s1) values('a',1,'A')", Collections.emptyList());
execute(manager, "INSERT INTO tblspace1.table1 (k1,n1,s1) values('b',2,'B')", Collections.emptyList());
execute(manager, "INSERT INTO tblspace1.table2 (k2,n2,s2) values('c',3,'A')", Collections.emptyList());
execute(manager, "INSERT INTO tblspace1.table2 (k2,n2,s2) values('d',4,'A')", Collections.emptyList());
{
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM" + " tblspace1.table1 t1" + " JOIN tblspace1.table2 t2" + " ON t1.n1 > 0" + " and t2.n2 >= 1", Collections.emptyList(), true, true, false, -1);
translated.context.setForceRetainReadLock(true);
if (manager.getPlanner() instanceof CalcitePlanner) {
assertThat(translated.plan.originalRoot, instanceOf(NestedLoopJoinOp.class));
NestedLoopJoinOp join = (NestedLoopJoinOp) translated.plan.originalRoot;
assertThat(join.getLeft(), instanceOf(SimpleScanOp.class));
assertThat(join.getRight(), instanceOf(SimpleScanOp.class));
}
// we want that the left branch of the join starts the transactoion
ScanResult scanResult = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.AUTOTRANSACTION_TRANSACTION));
List<DataAccessor> tuples = scanResult.dataScanner.consumeAndClose();
assertTrue(scanResult.transactionId > 0);
for (DataAccessor t : tuples) {
System.out.println("t:" + t);
assertEquals(6, t.getFieldNames().length);
assertEquals("k1", t.getFieldNames()[0]);
assertEquals("n1", t.getFieldNames()[1]);
assertEquals("s1", t.getFieldNames()[2]);
assertEquals("k2", t.getFieldNames()[3]);
assertEquals("n2", t.getFieldNames()[4]);
assertEquals("s2", t.getFieldNames()[5]);
}
assertEquals(4, tuples.size());
TestUtils.commitTransaction(manager, "tblspace1", scanResult.transactionId);
}
{
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM" + " tblspace1.table1 t1" + " JOIN tblspace1.table2 t2" + " ON t1.n1 > 0" + " and t2.n2 >= 1" + " order by t1.n1, t2.n2", Collections.emptyList(), true, true, false, // with order by
-1);
translated.context.setForceRetainReadLock(true);
assertThat(translated.plan.originalRoot, instanceOf(SortOp.class));
if (manager.getPlanner() instanceof CalcitePlanner) {
NestedLoopJoinOp join = (NestedLoopJoinOp) ((SortOp) translated.plan.originalRoot).getInput();
assertThat(join.getLeft(), instanceOf(SimpleScanOp.class));
assertThat(join.getRight(), instanceOf(SimpleScanOp.class));
} else {
JoinOp join = (JoinOp) ((SortOp) translated.plan.originalRoot).getInput();
assertThat(join.getLeft(), instanceOf(SimpleScanOp.class));
assertThat(join.getRight(), instanceOf(SimpleScanOp.class));
}
// we want that the left branch of the join starts the transactoion
ScanResult scanResult = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.AUTOTRANSACTION_TRANSACTION));
List<DataAccessor> tuples = scanResult.dataScanner.consumeAndClose();
assertTrue(scanResult.transactionId > 0);
for (DataAccessor t : tuples) {
System.out.println("t:" + t);
assertEquals(6, t.getFieldNames().length);
assertEquals("k1", t.getFieldNames()[0]);
assertEquals("n1", t.getFieldNames()[1]);
assertEquals("s1", t.getFieldNames()[2]);
assertEquals("k2", t.getFieldNames()[3]);
assertEquals("n2", t.getFieldNames()[4]);
assertEquals("s2", t.getFieldNames()[5]);
}
assertEquals(4, tuples.size());
TestUtils.commitTransaction(manager, "tblspace1", scanResult.transactionId);
}
{
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT t1.s1 FROM" + " tblspace1.table1 t1" + " JOIN tblspace1.table2 t2" + " ON t1.n1 > 0" + " and t2.n2 >= 1" + " order by t1.n1, t2.n2", Collections.emptyList(), true, true, false, // with order by on non selected fields
-1);
translated.context.setForceRetainReadLock(true);
assertThat(translated.plan.originalRoot, instanceOf(ProjectOp.class));
SortOp sort = (SortOp) ((ProjectOp) translated.plan.originalRoot).getInput();
assertThat(((ProjectOp) translated.plan.originalRoot).getProjection(), instanceOf(ProjectOp.ZeroCopyProjection.class));
if (manager.getPlanner() instanceof CalcitePlanner) {
NestedLoopJoinOp join = (NestedLoopJoinOp) sort.getInput();
assertThat(join.getLeft(), instanceOf(SimpleScanOp.class));
assertThat(join.getRight(), instanceOf(SimpleScanOp.class));
} else {
JoinOp join = (JoinOp) sort.getInput();
assertThat(join.getLeft(), instanceOf(SimpleScanOp.class));
assertThat(join.getRight(), instanceOf(SimpleScanOp.class));
}
// we want that the left branch of the join starts the transactoion
ScanResult scanResult = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.AUTOTRANSACTION_TRANSACTION));
List<DataAccessor> tuples = scanResult.dataScanner.consumeAndClose();
assertTrue(scanResult.transactionId > 0);
for (DataAccessor t : tuples) {
System.out.println("t:" + t);
assertEquals(1, t.getFieldNames().length);
assertEquals("s1", t.getFieldNames()[0]);
}
assertEquals(4, tuples.size());
TestUtils.commitTransaction(manager, "tblspace1", scanResult.transactionId);
}
}
}
use of herddb.model.planner.JoinOp in project herddb by diennea.
the class CalcitePlanner method planEnumerableMergeJoin.
private PlannerOp planEnumerableMergeJoin(EnumerableMergeJoin op, RelDataType rowType) {
// please note that EnumerableMergeJoin has a condition field which actually is not useful
PlannerOp left = convertRelNode(op.getLeft(), null, false, false);
PlannerOp right = convertRelNode(op.getRight(), null, false, false);
final JoinInfo analyzeCondition = op.analyzeCondition();
int[] leftKeys = analyzeCondition.leftKeys.toIntArray();
int[] rightKeys = analyzeCondition.rightKeys.toIntArray();
boolean generateNullsOnLeft = op.getJoinType().generatesNullsOnLeft();
boolean generateNullsOnRight = op.getJoinType().generatesNullsOnRight();
final RelDataType _rowType = rowType == null ? op.getRowType() : rowType;
List<CompiledSQLExpression> nonEquiConditions = convertJoinNonEquiConditions(analyzeCondition);
List<RelDataTypeField> fieldList = _rowType.getFieldList();
Column[] columns = new Column[fieldList.size()];
String[] fieldNames = new String[columns.length];
int i = 0;
for (RelDataTypeField field : fieldList) {
Column col = Column.column(field.getName().toLowerCase(), convertToHerdType(field.getType()));
fieldNames[i] = col.name;
columns[i++] = col;
}
return new JoinOp(fieldNames, columns, leftKeys, left, rightKeys, right, generateNullsOnLeft, generateNullsOnRight, true, nonEquiConditions);
}
use of herddb.model.planner.JoinOp in project herddb by diennea.
the class CalcitePlanner method planEnumerableHashJoin.
private PlannerOp planEnumerableHashJoin(EnumerableHashJoin op, RelDataType rowType) {
// please note that EnumerableSemiJoin has a condition field which actually is not useful
PlannerOp left = convertRelNode(op.getLeft(), null, false, false);
PlannerOp right = convertRelNode(op.getRight(), null, false, false);
final JoinInfo analyzeCondition = op.analyzeCondition();
int[] leftKeys = analyzeCondition.leftKeys.toIntArray();
int[] rightKeys = analyzeCondition.rightKeys.toIntArray();
boolean generateNullsOnLeft = op.getJoinType().generatesNullsOnLeft();
boolean generateNullsOnRight = op.getJoinType().generatesNullsOnRight();
List<CompiledSQLExpression> nonEquiConditions = convertJoinNonEquiConditions(analyzeCondition);
final RelDataType _rowType = rowType == null ? op.getRowType() : rowType;
List<RelDataTypeField> fieldList = _rowType.getFieldList();
Column[] columns = new Column[fieldList.size()];
String[] fieldNames = new String[columns.length];
int i = 0;
for (RelDataTypeField field : fieldList) {
Column col = Column.column(field.getName().toLowerCase(), convertToHerdType(field.getType()));
fieldNames[i] = col.name;
columns[i++] = col;
}
if (op.isSemiJoin()) {
return new SemiJoinOp(fieldNames, columns, leftKeys, left, rightKeys, right);
} else {
return new JoinOp(fieldNames, columns, leftKeys, left, rightKeys, right, generateNullsOnLeft, generateNullsOnRight, false, nonEquiConditions);
}
}
use of herddb.model.planner.JoinOp in project herddb by diennea.
the class JSQLParserPlanner method buildSelectBody.
private PlannerOp buildSelectBody(String defaultTableSpace, int maxRows, SelectBody selectBody, boolean forceScan) throws StatementExecutionException {
if (selectBody instanceof SetOperationList) {
SetOperationList list = (SetOperationList) selectBody;
return buildSetOperationList(defaultTableSpace, maxRows, list, forceScan);
}
checkSupported(selectBody instanceof PlainSelect, selectBody.getClass().getName());
PlainSelect plainSelect = (PlainSelect) selectBody;
checkSupported(!plainSelect.getMySqlHintStraightJoin());
checkSupported(!plainSelect.getMySqlSqlCalcFoundRows());
checkSupported(!plainSelect.getMySqlSqlNoCache());
checkSupported(plainSelect.getDistinct() == null);
checkSupported(plainSelect.getFetch() == null);
checkSupported(plainSelect.getFirst() == null);
checkSupported(plainSelect.getForUpdateTable() == null);
checkSupported(plainSelect.getForXmlPath() == null);
checkSupported(plainSelect.getHaving() == null);
checkSupported(plainSelect.getIntoTables() == null);
checkSupported(plainSelect.getOffset() == null);
checkSupported(plainSelect.getOptimizeFor() == null);
checkSupported(plainSelect.getOracleHierarchical() == null);
checkSupported(plainSelect.getOracleHint() == null);
checkSupported(plainSelect.getSkip() == null);
checkSupported(plainSelect.getWait() == null);
checkSupported(plainSelect.getKsqlWindow() == null);
FromItem fromItem = plainSelect.getFromItem();
checkSupported(fromItem instanceof net.sf.jsqlparser.schema.Table);
OpSchema primaryTableSchema = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) fromItem);
OpSchema[] joinedTables = new OpSchema[0];
int totalJoinOutputFieldsCount = primaryTableSchema.columns.length;
if (plainSelect.getJoins() != null) {
joinedTables = new OpSchema[plainSelect.getJoins().size()];
int i = 0;
for (Join join : plainSelect.getJoins()) {
checkSupported(!join.isApply());
checkSupported(!join.isCross());
checkSupported(!join.isFull() || (join.isFull() && join.isOuter()));
checkSupported(!join.isSemi());
checkSupported(!join.isStraight());
checkSupported(!join.isWindowJoin());
checkSupported(join.getJoinWindow() == null);
checkSupported(join.getUsingColumns() == null);
FromItem rightItem = join.getRightItem();
checkSupported(rightItem instanceof net.sf.jsqlparser.schema.Table);
OpSchema joinedTable = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) rightItem);
joinedTables[i++] = joinedTable;
totalJoinOutputFieldsCount += joinedTable.columns.length;
}
}
int pos = 0;
String[] joinOutputFieldnames = new String[totalJoinOutputFieldsCount];
ColumnRef[] joinOutputColumns = new ColumnRef[totalJoinOutputFieldsCount];
System.arraycopy(primaryTableSchema.columnNames, 0, joinOutputFieldnames, 0, primaryTableSchema.columnNames.length);
System.arraycopy(primaryTableSchema.columns, 0, joinOutputColumns, 0, primaryTableSchema.columns.length);
pos += primaryTableSchema.columnNames.length;
for (OpSchema joinedTable : joinedTables) {
System.arraycopy(joinedTable.columnNames, 0, joinOutputFieldnames, pos, joinedTable.columnNames.length);
System.arraycopy(joinedTable.columns, 0, joinOutputColumns, pos, joinedTable.columns.length);
pos += joinedTable.columnNames.length;
}
OpSchema currentSchema = primaryTableSchema;
// single JOIN only at the moment
checkSupported(joinedTables.length <= 1);
if (joinedTables.length > 0) {
currentSchema = new OpSchema(primaryTableSchema.tableSpace, null, null, joinOutputFieldnames, joinOutputColumns);
}
List<SelectItem> selectItems = plainSelect.getSelectItems();
checkSupported(!selectItems.isEmpty());
Predicate predicate = null;
TupleComparator comparator = null;
ScanLimits limits = null;
boolean identityProjection = false;
Projection projection;
List<SelectExpressionItem> selectedFields = new ArrayList<>(selectItems.size());
boolean containsAggregatedFunctions = false;
if (selectItems.size() == 1 && selectItems.get(0) instanceof AllColumns) {
projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
identityProjection = true;
} else {
checkSupported(!selectItems.isEmpty());
for (SelectItem item : selectItems) {
if (item instanceof SelectExpressionItem) {
SelectExpressionItem selectExpressionItem = (SelectExpressionItem) item;
selectedFields.add(selectExpressionItem);
if (SQLParserExpressionCompiler.detectAggregatedFunction(selectExpressionItem.getExpression()) != null) {
containsAggregatedFunctions = true;
}
} else if (item instanceof AllTableColumns) {
// expand alias.* to the full list of columns (according to pyhsical schema!)
AllTableColumns allTablesColumn = (AllTableColumns) item;
net.sf.jsqlparser.schema.Table table = allTablesColumn.getTable();
String tableName = fixMySqlBackTicks(table.getName());
boolean found = false;
if (primaryTableSchema.isTableOrAlias(tableName)) {
for (ColumnRef col : primaryTableSchema.columns) {
net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
selectedFields.add(selectExpressionItem);
found = true;
}
} else {
for (OpSchema joinedTableSchema : joinedTables) {
if (joinedTableSchema.isTableOrAlias(tableName)) {
for (ColumnRef col : joinedTableSchema.columns) {
net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
selectedFields.add(selectExpressionItem);
}
found = true;
break;
}
}
}
if (!found) {
checkSupported(false, "Bad table ref " + tableName + ".*");
}
} else {
checkSupported(false);
}
}
if (!containsAggregatedFunctions) {
// building the projection
// we have the current physical schema, we can create references by position (as Calcite does)
// in order to not need accessing columns by name and also making it easier to support
// ZeroCopyProjections
projection = buildProjection(selectedFields, true, currentSchema);
} else {
// start by full table scan, the AggregateOp operator will create the final projection
projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
identityProjection = true;
}
}
TableSpaceManager tableSpaceManager = this.manager.getTableSpaceManager(primaryTableSchema.tableSpace);
AbstractTableManager tableManager = tableSpaceManager.getTableManager(primaryTableSchema.name);
Table tableImpl = tableManager.getTable();
CompiledSQLExpression whereExpression = null;
if (plainSelect.getWhere() != null) {
whereExpression = SQLParserExpressionCompiler.compileExpression(plainSelect.getWhere(), currentSchema);
if (joinedTables.length == 0 && whereExpression != null) {
SQLRecordPredicate sqlWhere = new SQLRecordPredicate(tableImpl, null, whereExpression);
IndexUtils.discoverIndexOperations(primaryTableSchema.tableSpace, whereExpression, tableImpl, sqlWhere, selectBody, tableSpaceManager);
predicate = sqlWhere;
}
}
// start with a TableScan + filters
ScanStatement scan = new ScanStatement(primaryTableSchema.tableSpace, primaryTableSchema.name, Projection.IDENTITY(primaryTableSchema.columnNames, ColumnRef.toColumnsArray(primaryTableSchema.columns)), predicate, comparator, limits);
scan.setTableDef(tableImpl);
PlannerOp op = new BindableTableScanOp(scan);
PlannerOp[] scanJoinedTables = new PlannerOp[joinedTables.length];
int ji = 0;
for (OpSchema joinedTable : joinedTables) {
ScanStatement scanSecondaryTable = new ScanStatement(joinedTable.tableSpace, joinedTable.name, Projection.IDENTITY(joinedTable.columnNames, ColumnRef.toColumnsArray(joinedTable.columns)), null, null, null);
scan.setTableDef(tableImpl);
checkSupported(joinedTable.tableSpace.equalsIgnoreCase(primaryTableSchema.tableSpace));
PlannerOp opSecondaryTable = new BindableTableScanOp(scanSecondaryTable);
scanJoinedTables[ji++] = opSecondaryTable;
}
if (scanJoinedTables.length > 0) {
// assuming only one JOIN clause
Join joinClause = plainSelect.getJoins().get(0);
List<CompiledSQLExpression> joinConditions = new ArrayList<>();
if (joinClause.isNatural()) {
// NATURAL join adds a constraint on every column with the same name
List<CompiledSQLExpression> naturalJoinConstraints = new ArrayList<>();
int posInRowLeft = 0;
for (ColumnRef ref : primaryTableSchema.columns) {
int posInRowRight = primaryTableSchema.columns.length;
for (ColumnRef ref2 : joinedTables[0].columns) {
// assuming only one join
if (ref2.name.equalsIgnoreCase(ref.name)) {
CompiledSQLExpression equals = new CompiledEqualsExpression(new AccessCurrentRowExpression(posInRowLeft, ref.type), new AccessCurrentRowExpression(posInRowRight, ref2.type));
naturalJoinConstraints.add(equals);
}
posInRowRight++;
}
posInRowLeft++;
}
CompiledSQLExpression naturalJoin = new CompiledMultiAndExpression(naturalJoinConstraints.toArray(new CompiledSQLExpression[0]));
joinConditions.add(naturalJoin);
}
// handle "ON" clause
Expression onExpression = joinClause.getOnExpression();
if (onExpression != null) {
// TODO: this works for INNER join, but not for LEFT/RIGHT joins
CompiledSQLExpression onCondition = SQLParserExpressionCompiler.compileExpression(onExpression, currentSchema);
joinConditions.add(onCondition);
}
op = new JoinOp(joinOutputFieldnames, ColumnRef.toColumnsArray(joinOutputColumns), new int[0], op, new int[0], scanJoinedTables[0], // generateNullsOnLeft
joinClause.isRight() || (joinClause.isFull() && joinClause.isOuter()), // generateNullsOnRight
joinClause.isLeft() || (joinClause.isFull() && joinClause.isOuter()), // mergeJoin
false, // "ON" conditions
joinConditions);
// handle "WHERE" in case of JOIN
if (whereExpression != null) {
op = new FilterOp(op, whereExpression);
}
}
// add aggregations
if (containsAggregatedFunctions) {
checkSupported(scanJoinedTables.length == 0);
op = planAggregate(selectedFields, currentSchema, op, currentSchema, plainSelect.getGroupBy());
currentSchema = new OpSchema(currentSchema.tableSpace, currentSchema.name, currentSchema.alias, ColumnRef.toColumnsRefsArray(currentSchema.name, op.getOutputSchema()));
}
// add order by
if (plainSelect.getOrderByElements() != null) {
op = planSort(op, currentSchema, plainSelect.getOrderByElements());
}
// add limit
if (plainSelect.getLimit() != null) {
checkSupported(scanJoinedTables.length == 0);
// cannot mix LIMIT and TOP
checkSupported(plainSelect.getTop() == null);
Limit limit = plainSelect.getLimit();
CompiledSQLExpression offset;
if (limit.getOffset() != null) {
offset = SQLParserExpressionCompiler.compileExpression(limit.getOffset(), currentSchema);
} else {
offset = new ConstantExpression(0, ColumnTypes.NOTNULL_LONG);
}
CompiledSQLExpression rowCount = null;
if (limit.getRowCount() != null) {
rowCount = SQLParserExpressionCompiler.compileExpression(limit.getRowCount(), currentSchema);
}
op = new LimitOp(op, rowCount, offset);
}
if (plainSelect.getTop() != null) {
checkSupported(scanJoinedTables.length == 0);
// cannot mix LIMIT and TOP
checkSupported(plainSelect.getLimit() == null);
Top limit = plainSelect.getTop();
CompiledSQLExpression rowCount = null;
if (limit.getExpression() != null) {
rowCount = SQLParserExpressionCompiler.compileExpression(limit.getExpression(), currentSchema);
}
op = new LimitOp(op, rowCount, new ConstantExpression(0, ColumnTypes.NOTNULL_LONG));
}
if (!containsAggregatedFunctions && !identityProjection) {
// add projection
op = new ProjectOp(projection, op);
}
// additional maxrows from JDBC PreparedStatement
if (maxRows > 0) {
op = new LimitOp(op, new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), new ConstantExpression(0, ColumnTypes.NOTNULL_LONG)).optimize();
}
return op;
}
Aggregations