Search in sources :

Example 6 with Projection

use of herddb.model.Projection in project herddb by diennea.

the class CalcitePlannerTest method simplePlansTests.

@Test
public void simplePlansTests() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        execute(manager, "INSERT INTO tblspace1.tsql (k1,n1) values(?,?)", Arrays.asList("mykey", 1234), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
        try (DataScanner scan = scan(manager, "SELECT n1,k1 FROM tblspace1.tsql", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
        try (DataScanner scan = scan(manager, "SELECT k1,n1 FROM tblspace1.tsql", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
        try (DataScanner scan = scan(manager, "SELECT n1,k1 FROM tblspace1.tsql where k1='mykey'", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
        {
            // test ReduceExpressionsRule.FILTER_INSTANCE
            // "n1 = 1 and n1 is not null" -> this must be simplified to "n1 = 1"
            BindableTableScanOp plan = assertInstanceOf(plan(manager, "select * from tblspace1.tsql where n1 = 1 and n1 is not null"), BindableTableScanOp.class);
            ScanStatement statement = (ScanStatement) plan.getStatement();
            Projection projection = statement.getProjection();
            assertThat(projection, instanceOf(IdentityProjection.class));
            SQLRecordPredicate predicate = (SQLRecordPredicate) statement.getPredicate();
            assertThat(predicate.getWhere(), instanceOf(CompiledEqualsExpression.class));
            CompiledEqualsExpression equals = (CompiledEqualsExpression) predicate.getWhere();
            assertThat(equals.getLeft(), instanceOf(AccessCurrentRowExpression.class));
            assertThat(equals.getRight(), instanceOf(ConstantExpression.class));
        }
        if (manager.getPlanner() instanceof JSQLParserPlanner) {
            assertInstanceOf(plan(manager, "select * from tblspace1.tsql"), BindableTableScanOp.class);
        } else {
            assertInstanceOf(plan(manager, "select * from tblspace1.tsql"), TableScanOp.class);
        }
        assertInstanceOf(plan(manager, "select * from tblspace1.tsql where n1=1"), BindableTableScanOp.class);
        assertInstanceOf(plan(manager, "select n1 from tblspace1.tsql"), BindableTableScanOp.class);
        assertInstanceOf(plan(manager, "update tblspace1.tsql set n1=? where k1=?"), SimpleUpdateOp.class);
        assertInstanceOf(plan(manager, "update tblspace1.tsql set n1=? where k1=? and n1=?"), SimpleUpdateOp.class);
        if (manager.isFullSQLSupportEnabled()) {
            assertInstanceOf(plan(manager, "update tblspace1.tsql set n1=?" + " where n1 in (select b.n1*2 from tblspace1.tsql b)"), UpdateOp.class);
        }
        assertInstanceOf(plan(manager, "delete from tblspace1.tsql where k1=?"), SimpleDeleteOp.class);
        assertInstanceOf(plan(manager, "delete from tblspace1.tsql where k1=? and n1=?"), SimpleDeleteOp.class);
        if (manager.isFullSQLSupportEnabled()) {
            assertInstanceOf(plan(manager, "delete from tblspace1.tsql where n1 in (select b.n1*2 from tblspace1.tsql b)"), DeleteOp.class);
        }
        assertInstanceOf(plan(manager, "INSERT INTO tblspace1.tsql (k1,n1) values(?,?)"), SimpleInsertOp.class);
        assertInstanceOf(plan(manager, "INSERT INTO tblspace1.tsql (k1,n1) values(?,?),(?,?)"), InsertOp.class);
        if (manager.getPlanner() instanceof JSQLParserPlanner) {
            assertInstanceOf(plan(manager, "select * from tblspace1.tsql order by k1"), SortedBindableTableScanOp.class);
        } else {
            assertInstanceOf(plan(manager, "select * from tblspace1.tsql order by k1"), SortedTableScanOp.class);
        }
        assertInstanceOf(plan(manager, "select k1 from tblspace1.tsql order by k1"), SortedBindableTableScanOp.class);
        assertInstanceOf(plan(manager, "select k1 from tblspace1.tsql order by k1 limit 10"), LimitedSortedBindableTableScanOp.class);
        {
            BindableTableScanOp plan = assertInstanceOf(plan(manager, "select * from tblspace1.tsql where k1=?"), BindableTableScanOp.class);
            Projection projection = plan.getStatement().getProjection();
            System.out.println("projection:" + projection);
            assertThat(projection, instanceOf(IdentityProjection.class));
            assertNull(plan.getStatement().getComparator());
        }
        {
            SortedBindableTableScanOp plan = assertInstanceOf(plan(manager, "select n1,k1 from tblspace1.tsql order by n1"), SortedBindableTableScanOp.class);
            Projection projection = plan.getStatement().getProjection();
            System.out.println("projection:" + projection);
            assertThat(projection, instanceOf(ZeroCopyProjection.class));
            ZeroCopyProjection zero = (ZeroCopyProjection) projection;
            assertArrayEquals(new String[] { "n1", "k1" }, zero.getFieldNames());
            assertEquals("n1", zero.getColumns()[0].name);
            assertEquals("k1", zero.getColumns()[1].name);
            TupleComparator comparator = plan.getStatement().getComparator();
            System.out.println("comparator:" + comparator);
            assertThat(comparator, instanceOf(SortOp.class));
            SortOp sortOp = (SortOp) comparator;
            assertFalse(sortOp.isOnlyPrimaryKeyAndAscending());
        }
        {
            SortedBindableTableScanOp plan = assertInstanceOf(plan(manager, "select n1,k1 from tblspace1.tsql order by k1"), SortedBindableTableScanOp.class);
            Projection projection = plan.getStatement().getProjection();
            System.out.println("projection:" + projection);
            assertThat(projection, instanceOf(ZeroCopyProjection.class));
            ZeroCopyProjection zero = (ZeroCopyProjection) projection;
            assertArrayEquals(new String[] { "n1", "k1" }, zero.getFieldNames());
            assertEquals("n1", zero.getColumns()[0].name);
            assertEquals("k1", zero.getColumns()[1].name);
            TupleComparator comparator = plan.getStatement().getComparator();
            System.out.println("comparator:" + comparator);
            assertThat(comparator, instanceOf(SortOp.class));
            SortOp sortOp = (SortOp) comparator;
            assertTrue(sortOp.isOnlyPrimaryKeyAndAscending());
        }
        {
            SimpleScanOp plan = (SimpleScanOp) plan(manager, "select * from tblspace1.tsql order by n1");
            if (manager.getPlanner() instanceof JSQLParserPlanner) {
                assertThat(plan, instanceOf(SortedBindableTableScanOp.class));
            } else {
                assertThat(plan, instanceOf(SortedTableScanOp.class));
            }
            Projection projection = plan.getStatement().getProjection();
            System.out.println("projection:" + projection);
            assertThat(projection, instanceOf(IdentityProjection.class));
            IdentityProjection zero = (IdentityProjection) projection;
            assertArrayEquals(new String[] { "k1", "n1", "s1" }, zero.getFieldNames());
            assertEquals("k1", zero.getColumns()[0].name);
            assertEquals("n1", zero.getColumns()[1].name);
            assertEquals("s1", zero.getColumns()[2].name);
            TupleComparator comparator = plan.getStatement().getComparator();
            System.out.println("comparator:" + comparator);
            assertThat(comparator, instanceOf(SortOp.class));
            SortOp sortOp = (SortOp) comparator;
            assertFalse(sortOp.isOnlyPrimaryKeyAndAscending());
        }
        {
            SimpleScanOp plan = (SimpleScanOp) plan(manager, "select * from tblspace1.tsql order by k1");
            if (manager.getPlanner() instanceof JSQLParserPlanner) {
                assertThat(plan, instanceOf(SortedBindableTableScanOp.class));
            } else {
                assertThat(plan, instanceOf(SortedTableScanOp.class));
            }
            Projection projection = plan.getStatement().getProjection();
            System.out.println("projection:" + projection);
            assertThat(projection, instanceOf(IdentityProjection.class));
            IdentityProjection zero = (IdentityProjection) projection;
            assertArrayEquals(new String[] { "k1", "n1", "s1" }, zero.getFieldNames());
            assertEquals("k1", zero.getColumns()[0].name);
            assertEquals("n1", zero.getColumns()[1].name);
            assertEquals("s1", zero.getColumns()[2].name);
            TupleComparator comparator = plan.getStatement().getComparator();
            System.out.println("comparator:" + comparator);
            assertThat(comparator, instanceOf(SortOp.class));
            SortOp sortOp = (SortOp) comparator;
            assertTrue(sortOp.isOnlyPrimaryKeyAndAscending());
        }
    }
}
Also used : SimpleScanOp(herddb.model.planner.SimpleScanOp) MemoryDataStorageManager(herddb.mem.MemoryDataStorageManager) ZeroCopyProjection(herddb.model.planner.ProjectOp.ZeroCopyProjection) IdentityProjection(herddb.model.planner.ProjectOp.IdentityProjection) Projection(herddb.model.Projection) TupleComparator(herddb.model.TupleComparator) RawString(herddb.utils.RawString) SortOp(herddb.model.planner.SortOp) IdentityProjection(herddb.model.planner.ProjectOp.IdentityProjection) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) CompiledEqualsExpression(herddb.sql.expressions.CompiledEqualsExpression) DBManager(herddb.core.DBManager) DataScanner(herddb.model.DataScanner) ZeroCopyProjection(herddb.model.planner.ProjectOp.ZeroCopyProjection) MemoryCommitLogManager(herddb.mem.MemoryCommitLogManager) SortedBindableTableScanOp(herddb.model.planner.SortedBindableTableScanOp) LimitedSortedBindableTableScanOp(herddb.model.planner.LimitedSortedBindableTableScanOp) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) BindableTableScanOp(herddb.model.planner.BindableTableScanOp) SortedBindableTableScanOp(herddb.model.planner.SortedBindableTableScanOp) LimitedSortedBindableTableScanOp(herddb.model.planner.LimitedSortedBindableTableScanOp) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Example 7 with Projection

use of herddb.model.Projection in project herddb by diennea.

the class CalcitePlanner method planEnumerableTableScan.

private PlannerOp planEnumerableTableScan(EnumerableTableScan scan, RelDataType rowType) {
    final String tableSpace = scan.getTable().getQualifiedName().get(0);
    final TableImpl tableImpl = (TableImpl) scan.getTable().unwrap(org.apache.calcite.schema.Table.class);
    Table table = tableImpl.tableManager.getTable();
    Column[] columns = table.getColumns();
    int numColumns = columns.length;
    boolean usingAliases = false;
    if (rowType != null) {
        List<String> fieldNamesFromQuery = rowType.getFieldNames();
        for (int i = 0; i < numColumns; i++) {
            String fieldName = fieldNamesFromQuery.get(i);
            String alias = fieldName.toLowerCase();
            String colName = columns[i].name;
            if (!alias.equals(colName)) {
                usingAliases = true;
                break;
            }
        }
    }
    if (usingAliases) {
        List<String> fieldNamesFromQuery = rowType.getFieldNames();
        String[] fieldNames = new String[numColumns];
        int[] projections = new int[numColumns];
        for (int i = 0; i < numColumns; i++) {
            String alias = fieldNamesFromQuery.get(i).toLowerCase();
            fieldNames[i] = alias;
            projections[i] = i;
        }
        Projection zeroCopy = new ProjectOp.ZeroCopyProjection(fieldNames, columns, projections);
        ScanStatement scanStatement = new ScanStatement(tableSpace, table, zeroCopy, null);
        return new TableScanOp(scanStatement);
    } else {
        ScanStatement scanStatement = new ScanStatement(tableSpace, table, null);
        return new TableScanOp(scanStatement);
    }
}
Also used : Table(herddb.model.Table) ShowCreateTableCalculator.calculateShowCreateTable(herddb.sql.functions.ShowCreateTableCalculator.calculateShowCreateTable) SqlStdOperatorTable(org.apache.calcite.sql.fun.SqlStdOperatorTable) RelOptTable(org.apache.calcite.plan.RelOptTable) ProjectableFilterableTable(org.apache.calcite.schema.ProjectableFilterableTable) ScannableTable(org.apache.calcite.schema.ScannableTable) AbstractTable(org.apache.calcite.schema.impl.AbstractTable) ModifiableTable(org.apache.calcite.schema.ModifiableTable) FilteredTableScanOp(herddb.model.planner.FilteredTableScanOp) BindableTableScanOp(herddb.model.planner.BindableTableScanOp) TableScanOp(herddb.model.planner.TableScanOp) Projection(herddb.model.Projection) Column(herddb.model.Column) ScanStatement(herddb.model.commands.ScanStatement)

Example 8 with Projection

use of herddb.model.Projection in project herddb by diennea.

the class JSQLParserPlanner method buildSelectBody.

private PlannerOp buildSelectBody(String defaultTableSpace, int maxRows, SelectBody selectBody, boolean forceScan) throws StatementExecutionException {
    if (selectBody instanceof SetOperationList) {
        SetOperationList list = (SetOperationList) selectBody;
        return buildSetOperationList(defaultTableSpace, maxRows, list, forceScan);
    }
    checkSupported(selectBody instanceof PlainSelect, selectBody.getClass().getName());
    PlainSelect plainSelect = (PlainSelect) selectBody;
    checkSupported(!plainSelect.getMySqlHintStraightJoin());
    checkSupported(!plainSelect.getMySqlSqlCalcFoundRows());
    checkSupported(!plainSelect.getMySqlSqlNoCache());
    checkSupported(plainSelect.getDistinct() == null);
    checkSupported(plainSelect.getFetch() == null);
    checkSupported(plainSelect.getFirst() == null);
    checkSupported(plainSelect.getForUpdateTable() == null);
    checkSupported(plainSelect.getForXmlPath() == null);
    checkSupported(plainSelect.getHaving() == null);
    checkSupported(plainSelect.getIntoTables() == null);
    checkSupported(plainSelect.getOffset() == null);
    checkSupported(plainSelect.getOptimizeFor() == null);
    checkSupported(plainSelect.getOracleHierarchical() == null);
    checkSupported(plainSelect.getOracleHint() == null);
    checkSupported(plainSelect.getSkip() == null);
    checkSupported(plainSelect.getWait() == null);
    checkSupported(plainSelect.getKsqlWindow() == null);
    FromItem fromItem = plainSelect.getFromItem();
    checkSupported(fromItem instanceof net.sf.jsqlparser.schema.Table);
    OpSchema primaryTableSchema = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) fromItem);
    OpSchema[] joinedTables = new OpSchema[0];
    int totalJoinOutputFieldsCount = primaryTableSchema.columns.length;
    if (plainSelect.getJoins() != null) {
        joinedTables = new OpSchema[plainSelect.getJoins().size()];
        int i = 0;
        for (Join join : plainSelect.getJoins()) {
            checkSupported(!join.isApply());
            checkSupported(!join.isCross());
            checkSupported(!join.isFull() || (join.isFull() && join.isOuter()));
            checkSupported(!join.isSemi());
            checkSupported(!join.isStraight());
            checkSupported(!join.isWindowJoin());
            checkSupported(join.getJoinWindow() == null);
            checkSupported(join.getUsingColumns() == null);
            FromItem rightItem = join.getRightItem();
            checkSupported(rightItem instanceof net.sf.jsqlparser.schema.Table);
            OpSchema joinedTable = getTableSchema(defaultTableSpace, (net.sf.jsqlparser.schema.Table) rightItem);
            joinedTables[i++] = joinedTable;
            totalJoinOutputFieldsCount += joinedTable.columns.length;
        }
    }
    int pos = 0;
    String[] joinOutputFieldnames = new String[totalJoinOutputFieldsCount];
    ColumnRef[] joinOutputColumns = new ColumnRef[totalJoinOutputFieldsCount];
    System.arraycopy(primaryTableSchema.columnNames, 0, joinOutputFieldnames, 0, primaryTableSchema.columnNames.length);
    System.arraycopy(primaryTableSchema.columns, 0, joinOutputColumns, 0, primaryTableSchema.columns.length);
    pos += primaryTableSchema.columnNames.length;
    for (OpSchema joinedTable : joinedTables) {
        System.arraycopy(joinedTable.columnNames, 0, joinOutputFieldnames, pos, joinedTable.columnNames.length);
        System.arraycopy(joinedTable.columns, 0, joinOutputColumns, pos, joinedTable.columns.length);
        pos += joinedTable.columnNames.length;
    }
    OpSchema currentSchema = primaryTableSchema;
    // single JOIN only at the moment
    checkSupported(joinedTables.length <= 1);
    if (joinedTables.length > 0) {
        currentSchema = new OpSchema(primaryTableSchema.tableSpace, null, null, joinOutputFieldnames, joinOutputColumns);
    }
    List<SelectItem> selectItems = plainSelect.getSelectItems();
    checkSupported(!selectItems.isEmpty());
    Predicate predicate = null;
    TupleComparator comparator = null;
    ScanLimits limits = null;
    boolean identityProjection = false;
    Projection projection;
    List<SelectExpressionItem> selectedFields = new ArrayList<>(selectItems.size());
    boolean containsAggregatedFunctions = false;
    if (selectItems.size() == 1 && selectItems.get(0) instanceof AllColumns) {
        projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
        identityProjection = true;
    } else {
        checkSupported(!selectItems.isEmpty());
        for (SelectItem item : selectItems) {
            if (item instanceof SelectExpressionItem) {
                SelectExpressionItem selectExpressionItem = (SelectExpressionItem) item;
                selectedFields.add(selectExpressionItem);
                if (SQLParserExpressionCompiler.detectAggregatedFunction(selectExpressionItem.getExpression()) != null) {
                    containsAggregatedFunctions = true;
                }
            } else if (item instanceof AllTableColumns) {
                // expand  alias.* to the full list of columns (according to pyhsical schema!)
                AllTableColumns allTablesColumn = (AllTableColumns) item;
                net.sf.jsqlparser.schema.Table table = allTablesColumn.getTable();
                String tableName = fixMySqlBackTicks(table.getName());
                boolean found = false;
                if (primaryTableSchema.isTableOrAlias(tableName)) {
                    for (ColumnRef col : primaryTableSchema.columns) {
                        net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                        SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                        selectedFields.add(selectExpressionItem);
                        found = true;
                    }
                } else {
                    for (OpSchema joinedTableSchema : joinedTables) {
                        if (joinedTableSchema.isTableOrAlias(tableName)) {
                            for (ColumnRef col : joinedTableSchema.columns) {
                                net.sf.jsqlparser.schema.Column c = new net.sf.jsqlparser.schema.Column(table, col.name);
                                SelectExpressionItem selectExpressionItem = new SelectExpressionItem(c);
                                selectedFields.add(selectExpressionItem);
                            }
                            found = true;
                            break;
                        }
                    }
                }
                if (!found) {
                    checkSupported(false, "Bad table ref " + tableName + ".*");
                }
            } else {
                checkSupported(false);
            }
        }
        if (!containsAggregatedFunctions) {
            // building the projection
            // we have the current physical schema, we can create references by position (as Calcite does)
            // in order to not need accessing columns by name and also making it easier to support
            // ZeroCopyProjections
            projection = buildProjection(selectedFields, true, currentSchema);
        } else {
            // start by full table scan, the AggregateOp operator will create the final projection
            projection = Projection.IDENTITY(currentSchema.columnNames, ColumnRef.toColumnsArray(currentSchema.columns));
            identityProjection = true;
        }
    }
    TableSpaceManager tableSpaceManager = this.manager.getTableSpaceManager(primaryTableSchema.tableSpace);
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(primaryTableSchema.name);
    Table tableImpl = tableManager.getTable();
    CompiledSQLExpression whereExpression = null;
    if (plainSelect.getWhere() != null) {
        whereExpression = SQLParserExpressionCompiler.compileExpression(plainSelect.getWhere(), currentSchema);
        if (joinedTables.length == 0 && whereExpression != null) {
            SQLRecordPredicate sqlWhere = new SQLRecordPredicate(tableImpl, null, whereExpression);
            IndexUtils.discoverIndexOperations(primaryTableSchema.tableSpace, whereExpression, tableImpl, sqlWhere, selectBody, tableSpaceManager);
            predicate = sqlWhere;
        }
    }
    // start with a TableScan + filters
    ScanStatement scan = new ScanStatement(primaryTableSchema.tableSpace, primaryTableSchema.name, Projection.IDENTITY(primaryTableSchema.columnNames, ColumnRef.toColumnsArray(primaryTableSchema.columns)), predicate, comparator, limits);
    scan.setTableDef(tableImpl);
    PlannerOp op = new BindableTableScanOp(scan);
    PlannerOp[] scanJoinedTables = new PlannerOp[joinedTables.length];
    int ji = 0;
    for (OpSchema joinedTable : joinedTables) {
        ScanStatement scanSecondaryTable = new ScanStatement(joinedTable.tableSpace, joinedTable.name, Projection.IDENTITY(joinedTable.columnNames, ColumnRef.toColumnsArray(joinedTable.columns)), null, null, null);
        scan.setTableDef(tableImpl);
        checkSupported(joinedTable.tableSpace.equalsIgnoreCase(primaryTableSchema.tableSpace));
        PlannerOp opSecondaryTable = new BindableTableScanOp(scanSecondaryTable);
        scanJoinedTables[ji++] = opSecondaryTable;
    }
    if (scanJoinedTables.length > 0) {
        // assuming only one JOIN clause
        Join joinClause = plainSelect.getJoins().get(0);
        List<CompiledSQLExpression> joinConditions = new ArrayList<>();
        if (joinClause.isNatural()) {
            // NATURAL join adds a constraint on every column with the same name
            List<CompiledSQLExpression> naturalJoinConstraints = new ArrayList<>();
            int posInRowLeft = 0;
            for (ColumnRef ref : primaryTableSchema.columns) {
                int posInRowRight = primaryTableSchema.columns.length;
                for (ColumnRef ref2 : joinedTables[0].columns) {
                    // assuming only one join
                    if (ref2.name.equalsIgnoreCase(ref.name)) {
                        CompiledSQLExpression equals = new CompiledEqualsExpression(new AccessCurrentRowExpression(posInRowLeft, ref.type), new AccessCurrentRowExpression(posInRowRight, ref2.type));
                        naturalJoinConstraints.add(equals);
                    }
                    posInRowRight++;
                }
                posInRowLeft++;
            }
            CompiledSQLExpression naturalJoin = new CompiledMultiAndExpression(naturalJoinConstraints.toArray(new CompiledSQLExpression[0]));
            joinConditions.add(naturalJoin);
        }
        // handle "ON" clause
        Expression onExpression = joinClause.getOnExpression();
        if (onExpression != null) {
            // TODO: this works for INNER join, but not for LEFT/RIGHT joins
            CompiledSQLExpression onCondition = SQLParserExpressionCompiler.compileExpression(onExpression, currentSchema);
            joinConditions.add(onCondition);
        }
        op = new JoinOp(joinOutputFieldnames, ColumnRef.toColumnsArray(joinOutputColumns), new int[0], op, new int[0], scanJoinedTables[0], // generateNullsOnLeft
        joinClause.isRight() || (joinClause.isFull() && joinClause.isOuter()), // generateNullsOnRight
        joinClause.isLeft() || (joinClause.isFull() && joinClause.isOuter()), // mergeJoin
        false, // "ON" conditions
        joinConditions);
        // handle "WHERE" in case of JOIN
        if (whereExpression != null) {
            op = new FilterOp(op, whereExpression);
        }
    }
    // add aggregations
    if (containsAggregatedFunctions) {
        checkSupported(scanJoinedTables.length == 0);
        op = planAggregate(selectedFields, currentSchema, op, currentSchema, plainSelect.getGroupBy());
        currentSchema = new OpSchema(currentSchema.tableSpace, currentSchema.name, currentSchema.alias, ColumnRef.toColumnsRefsArray(currentSchema.name, op.getOutputSchema()));
    }
    // add order by
    if (plainSelect.getOrderByElements() != null) {
        op = planSort(op, currentSchema, plainSelect.getOrderByElements());
    }
    // add limit
    if (plainSelect.getLimit() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getTop() == null);
        Limit limit = plainSelect.getLimit();
        CompiledSQLExpression offset;
        if (limit.getOffset() != null) {
            offset = SQLParserExpressionCompiler.compileExpression(limit.getOffset(), currentSchema);
        } else {
            offset = new ConstantExpression(0, ColumnTypes.NOTNULL_LONG);
        }
        CompiledSQLExpression rowCount = null;
        if (limit.getRowCount() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getRowCount(), currentSchema);
        }
        op = new LimitOp(op, rowCount, offset);
    }
    if (plainSelect.getTop() != null) {
        checkSupported(scanJoinedTables.length == 0);
        // cannot mix LIMIT and TOP
        checkSupported(plainSelect.getLimit() == null);
        Top limit = plainSelect.getTop();
        CompiledSQLExpression rowCount = null;
        if (limit.getExpression() != null) {
            rowCount = SQLParserExpressionCompiler.compileExpression(limit.getExpression(), currentSchema);
        }
        op = new LimitOp(op, rowCount, new ConstantExpression(0, ColumnTypes.NOTNULL_LONG));
    }
    if (!containsAggregatedFunctions && !identityProjection) {
        // add projection
        op = new ProjectOp(projection, op);
    }
    // additional maxrows from JDBC PreparedStatement
    if (maxRows > 0) {
        op = new LimitOp(op, new ConstantExpression(maxRows, ColumnTypes.NOTNULL_LONG), new ConstantExpression(0, ColumnTypes.NOTNULL_LONG)).optimize();
    }
    return op;
}
Also used : CompiledMultiAndExpression(herddb.sql.expressions.CompiledMultiAndExpression) ConstantExpression(herddb.sql.expressions.ConstantExpression) ArrayList(java.util.ArrayList) Projection(herddb.model.Projection) CompiledSQLExpression(herddb.sql.expressions.CompiledSQLExpression) Predicate(herddb.model.Predicate) CompiledEqualsExpression(herddb.sql.expressions.CompiledEqualsExpression) TableSpaceManager(herddb.core.TableSpaceManager) PlannerOp(herddb.model.planner.PlannerOp) ScanLimits(herddb.model.ScanLimits) ProjectOp(herddb.model.planner.ProjectOp) AccessCurrentRowExpression(herddb.sql.expressions.AccessCurrentRowExpression) Top(net.sf.jsqlparser.statement.select.Top) AbstractTableManager(herddb.core.AbstractTableManager) ColumnRef(herddb.sql.expressions.ColumnRef) BindableTableScanOp(herddb.model.planner.BindableTableScanOp) SelectExpressionItem(net.sf.jsqlparser.statement.select.SelectExpressionItem) FilterOp(herddb.model.planner.FilterOp) PlainSelect(net.sf.jsqlparser.statement.select.PlainSelect) TupleComparator(herddb.model.TupleComparator) AllColumns(net.sf.jsqlparser.statement.select.AllColumns) Column(herddb.model.Column) SelectItem(net.sf.jsqlparser.statement.select.SelectItem) FromItem(net.sf.jsqlparser.statement.select.FromItem) ScanStatement(herddb.model.commands.ScanStatement) JoinOp(herddb.model.planner.JoinOp) Table(herddb.model.Table) ShowCreateTableCalculator.calculateShowCreateTable(herddb.sql.functions.ShowCreateTableCalculator.calculateShowCreateTable) CreateTable(net.sf.jsqlparser.statement.create.table.CreateTable) Join(net.sf.jsqlparser.statement.select.Join) LimitOp(herddb.model.planner.LimitOp) SetOperationList(net.sf.jsqlparser.statement.select.SetOperationList) AccessCurrentRowExpression(herddb.sql.expressions.AccessCurrentRowExpression) Expression(net.sf.jsqlparser.expression.Expression) CompiledMultiAndExpression(herddb.sql.expressions.CompiledMultiAndExpression) JdbcParameterExpression(herddb.sql.expressions.JdbcParameterExpression) AlterExpression(net.sf.jsqlparser.statement.alter.AlterExpression) TypedJdbcParameterExpression(herddb.sql.expressions.TypedJdbcParameterExpression) ConstantExpression(herddb.sql.expressions.ConstantExpression) SignedExpression(net.sf.jsqlparser.expression.SignedExpression) CompiledSQLExpression(herddb.sql.expressions.CompiledSQLExpression) CompiledEqualsExpression(herddb.sql.expressions.CompiledEqualsExpression) AllTableColumns(net.sf.jsqlparser.statement.select.AllTableColumns) Limit(net.sf.jsqlparser.statement.select.Limit) OpSchema(herddb.sql.expressions.OpSchema)

Example 9 with Projection

use of herddb.model.Projection in project herddb by diennea.

the class TableManager method scanNoStream.

private DataScanner scanNoStream(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
    if (transaction != null) {
        transaction.increaseRefcount();
    }
    try {
        boolean sorted = statement.getComparator() != null;
        boolean sortedByClusteredIndex = statement.getComparator() != null && statement.getComparator().isOnlyPrimaryKeyAndAscending() && keyToPageSortedAscending;
        final Projection projection = statement.getProjection();
        boolean applyProjectionDuringScan = !sorted && projection != null;
        MaterializedRecordSet recordSet;
        if (applyProjectionDuringScan) {
            recordSet = tableSpaceManager.getDbmanager().getRecordSetFactory().createRecordSet(projection.getFieldNames(), projection.getColumns());
        } else {
            recordSet = tableSpaceManager.getDbmanager().getRecordSetFactory().createRecordSet(table.columnNames, table.columns);
        }
        ScanLimits limits = statement.getLimits();
        int maxRows = limits == null ? 0 : limits.computeMaxRows(context);
        int offset = limits == null ? 0 : limits.computeOffset(context);
        boolean sortDone = false;
        if (maxRows > 0) {
            if (sortedByClusteredIndex) {
                // leverage the sorted nature of the clustered primary key index
                AtomicInteger remaining = new AtomicInteger(maxRows);
                if (offset > 0) {
                    remaining.getAndAdd(offset);
                }
                accessTableData(statement, context, new ScanResultOperation() {

                    private boolean inTransactionData;

                    @Override
                    public void beginNewRecordsInTransactionBlock() {
                        inTransactionData = true;
                    }

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                recordSet.add(tuple);
                            } else {
                                recordSet.add(record.getDataAccessor(table));
                            }
                            if (!inTransactionData) {
                                // in the same order as the clustered index
                                if (remaining.decrementAndGet() == 0) {
                                    // we want to receive transaction data uncommitted records too
                                    throw new ExitLoop(true);
                                }
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
                // we have to sort data any way, because accessTableData will return partially sorted data
                sortDone = transaction == null;
            } else if (sorted) {
                InStreamTupleSorter sorter = new InStreamTupleSorter(offset + maxRows, statement.getComparator());
                accessTableData(statement, context, new ScanResultOperation() {

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                sorter.collect(tuple);
                            } else {
                                sorter.collect(record.getDataAccessor(table));
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
                sorter.flushToRecordSet(recordSet);
                sortDone = true;
            } else {
                // if no sort is present the limits can be applying during the scan and perform an early exit
                AtomicInteger remaining = new AtomicInteger(maxRows);
                if (offset > 0) {
                    remaining.getAndAdd(offset);
                }
                accessTableData(statement, context, new ScanResultOperation() {

                    @Override
                    public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                        try {
                            if (applyProjectionDuringScan) {
                                DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                                recordSet.add(tuple);
                            } else {
                                recordSet.add(record.getDataAccessor(table));
                            }
                            if (remaining.decrementAndGet() == 0) {
                                throw new ExitLoop(false);
                            }
                        } finally {
                            locksManager.releaseLock(lockHandle);
                        }
                    }
                }, transaction, lockRequired, forWrite);
            }
        } else {
            accessTableData(statement, context, new ScanResultOperation() {

                @Override
                public void accept(Record record, LockHandle lockHandle) throws StatementExecutionException {
                    try {
                        if (applyProjectionDuringScan) {
                            DataAccessor tuple = projection.map(record.getDataAccessor(table), context);
                            recordSet.add(tuple);
                        } else {
                            recordSet.add(record.getDataAccessor(table));
                        }
                    } finally {
                        locksManager.releaseLock(lockHandle);
                    }
                }
            }, transaction, lockRequired, forWrite);
        }
        recordSet.writeFinished();
        if (!sortDone) {
            recordSet.sort(statement.getComparator());
        }
        recordSet.applyLimits(statement.getLimits(), context);
        if (!applyProjectionDuringScan) {
            recordSet.applyProjection(statement.getProjection(), context);
        }
        return new SimpleDataScanner(transaction, recordSet);
    } finally {
        if (transaction != null) {
            transaction.decreaseRefCount();
        }
    }
}
Also used : LockHandle(herddb.utils.LockHandle) ScanLimits(herddb.model.ScanLimits) DataAccessor(herddb.utils.DataAccessor) Projection(herddb.model.Projection) StatementExecutionException(herddb.model.StatementExecutionException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Record(herddb.model.Record)

Example 10 with Projection

use of herddb.model.Projection in project herddb by diennea.

the class RecordSetSuite method testApplyProjectionSwap.

@Test
public void testApplyProjectionSwap() throws Exception {
    RecordSetFactory factory = buildRecordSetFactory(1);
    Column[] columns = new Column[2];
    columns[0] = Column.column("s1", ColumnTypes.STRING);
    columns[1] = Column.column("n1", ColumnTypes.LONG);
    Set<String> expected_s2 = new HashSet<>();
    Set<Integer> expected_n2 = new HashSet<>();
    String[] fieldNames = Column.buildFieldNamesList(columns);
    try (MaterializedRecordSet rs = factory.createRecordSet(fieldNames, columns)) {
        for (int i = 0; i < 100; i++) {
            Map<String, Object> record = new HashMap<>();
            String s1 = "test_" + i;
            record.put("s1", s1);
            record.put("n1", i);
            expected_s2.add(s1);
            expected_n2.add(i);
            rs.add(new Tuple(record, fieldNames));
        }
        rs.writeFinished();
        Column[] columns_projected = new Column[2];
        columns_projected[0] = Column.column("n2", ColumnTypes.LONG);
        columns_projected[1] = Column.column("s2", ColumnTypes.STRING);
        String[] fieldNames_projected = new String[] { "n2", "s2" };
        rs.applyProjection(new Projection() {

            @Override
            public Column[] getColumns() {
                return columns_projected;
            }

            @Override
            public String[] getFieldNames() {
                return fieldNames_projected;
            }

            @Override
            public Tuple map(DataAccessor tuple, StatementEvaluationContext context) throws StatementExecutionException {
                Object[] projected_values = new Object[2];
                projected_values[0] = tuple.get("n1");
                projected_values[1] = tuple.get("s1");
                return new Tuple(fieldNames_projected, projected_values);
            }
        }, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT());
        for (DataAccessor t : rs) {
            System.out.println("t:" + t.toMap());
            expected_s2.remove(t.get("s2").toString());
            expected_n2.remove(t.get("n2"));
        }
        assertTrue(expected_s2.isEmpty());
        assertTrue(expected_n2.isEmpty());
        assertEquals("n2", rs.getColumns()[0].name);
        assertEquals("s2", rs.getColumns()[1].name);
    }
}
Also used : HashMap(java.util.HashMap) DataAccessor(herddb.utils.DataAccessor) Projection(herddb.model.Projection) StatementExecutionException(herddb.model.StatementExecutionException) Column(herddb.model.Column) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Tuple(herddb.model.Tuple) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

Projection (herddb.model.Projection)11 Column (herddb.model.Column)6 ScanStatement (herddb.model.commands.ScanStatement)6 StatementExecutionException (herddb.model.StatementExecutionException)5 Table (herddb.model.Table)5 BindableTableScanOp (herddb.model.planner.BindableTableScanOp)5 TupleComparator (herddb.model.TupleComparator)4 DataAccessor (herddb.utils.DataAccessor)4 Test (org.junit.Test)4 TableSpaceManager (herddb.core.TableSpaceManager)3 DataScanner (herddb.model.DataScanner)3 Predicate (herddb.model.Predicate)3 StatementEvaluationContext (herddb.model.StatementEvaluationContext)3 CompiledSQLExpression (herddb.sql.expressions.CompiledSQLExpression)3 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 AbstractTableManager (herddb.core.AbstractTableManager)2 DBManager (herddb.core.DBManager)2 MemoryCommitLogManager (herddb.mem.MemoryCommitLogManager)2 MemoryDataStorageManager (herddb.mem.MemoryDataStorageManager)2