Examples with VirtualColumn com.questdb.ql.ops.VirtualColumn used on opensource projects

Example 6 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class VirtualColumnTest method testPlusDouble.

@Test
public void testPlusDouble() throws Exception {
    try (final JournalWriter w = getFactory().writer(new JournalStructure("xyz") {

        {
            $str("ccy");
            $double("bid");
        }
    })) {
        Rnd rnd = new Rnd();
        for (int i = 0; i < 100; i++) {
            JournalEntryWriter ew = w.entryWriter();
            ew.putStr(0, rnd.nextString(10));
            ew.putDouble(1, rnd.nextDouble());
            ew.append();
        }
        w.commit();
        StringSink sink = new StringSink();
        RecordSourcePrinter p = new RecordSourcePrinter(sink);
        BootstrapEnv env = new BootstrapEnv();
        env.configuration = new ServerConfiguration();
        final AddDoubleOperator plus = (AddDoubleOperator) AddDoubleOperator.FACTORY.newInstance(0, env);
        plus.setName("plus");
        plus.setLhs(new DoubleRecordSourceColumn(w.getMetadata().getColumnIndex("bid"), 0));
        plus.setRhs(new DoubleConstant(12.5, 0));
        // select ccy, bid, bid+12.5 plus from xyz
        try (VirtualColumnRecordSource src = new VirtualColumnRecordSource(compile("xyz"), new ObjList<VirtualColumn>() {

            {
                add(plus);
            }
        })) {
            p.print(src, getFactory());
            final String expected = "VTJWCPSWHY\t-104.021850585938\t-91.521850585938\n" + "PEHNRXGZSX\t0.000020634160\t12.500020634160\n" + "IBBTGPGWFF\t0.000000567185\t12.500000567185\n" + "DEYYQEHBHF\t0.000000401164\t12.500000401164\n" + "LPDXYSBEOU\t384.072387695313\t396.572387695313\n" + "SHRUEDRQQU\t109.355468750000\t121.855468750000\n" + "FJGETJRSZS\t198.750000000000\t211.250000000000\n" + "RFBVTMHGOO\t172.796875000000\t185.296875000000\n" + "VDZJMYICCX\t-1024.000000000000\t-1011.500000000000\n" + "UICWEKGHVU\t0.000425009843\t12.500425009843\n" + "DOTSEDYYCT\t557.000000000000\t569.500000000000\n" + "OLYXWCKYLS\t0.000553289865\t12.500553289865\n" + "DSWUGSHOLN\t-1013.467773437500\t-1000.967773437500\n" + "IQBZXIOVIK\t512.000000000000\t524.500000000000\n" + "MSSUQSRLTK\t0.000000776007\t12.500000776007\n" + "SJOJIPHZEP\t174.774871826172\t187.274871826172\n" + "VLTOVLJUML\t-77.428833007813\t-64.928833007813\n" + "HMLLEOYPHR\t0.000327562877\t12.500327562877\n" + "ZIMNZZRMFM\t238.632812500000\t251.132812500000\n" + "ZGHWVDKFLO\t0.000102697388\t12.500102697388\n" + "OXPKRGIIHY\t128.000000000000\t140.500000000000\n" + "OQMYSSMPGL\t-144.421875000000\t-131.921875000000\n" + "HNZHZSQLDG\t832.000000000000\t844.500000000000\n" + "GIFOUSZMZV\t-200.000000000000\t-187.500000000000\n" + "BNDCQCEHNO\t2.602588653564\t15.102588653564\n" + "ELLKKHTWNW\t0.000969694171\t12.500969694171\n" + "FLRBROMNXK\t0.000000548919\t12.500000548919\n" + "ZULIGYVFZF\t-327.250000000000\t-314.750000000000\n" + "ZLUOGXHFVW\t0.000002116648\t12.500002116648\n" + "SRGOONFCLT\t310.101058959961\t322.601058959961\n" + "KFMQNTOGMX\t0.000012478828\t12.500012478828\n" + "LGMXSLUQDY\t0.000013214448\t12.500013214448\n" + "HNIMYFFDTN\t0.000000001910\t12.500000001910\n" + "FLPBNHGZWW\t695.173828125000\t707.673828125000\n" + "NGTNLEGPUH\t0.205350898206\t12.705350898206\n" + "UGGLNYRZLC\t638.000000000000\t650.500000000000\n" + "MIGQZVKHTL\t0.000000040298\t12.500000040298\n" + "SLQVFGPPRG\t-1024.000000000000\t-1011.500000000000\n" + "BHYSBQYMIZ\t5.336447119713\t17.836447119713\n" + "VTNPIWZNFK\t0.000000012570\t12.500000012570\n" + "VMCGFNWGRM\t-128.000000000000\t-115.500000000000\n" + "GIJYDVRVNG\t0.000000052204\t12.500000052204\n" + "EQODRZEIWF\t-31.000000000000\t-18.500000000000\n" + "KYHQQUWQOE\t880.000000000000\t892.500000000000\n" + "NEBQQEMXDK\t25.839271545410\t38.339271545410\n" + "JCTIZKYFLU\t768.000000000000\t780.500000000000\n" + "QSNPXMKJSM\t0.004184104619\t12.504184104619\n" + "XEYVTUPDHH\t845.823730468750\t858.323730468750\n" + "IWHPZRHHMG\t37.350353240967\t49.850353240967\n" + "YYFLSVIHDW\t-400.000000000000\t-387.500000000000\n" + "EVMLKCJBEV\t0.000000914462\t12.500000914462\n" + "HLIHYBTVZN\t0.000001439041\t12.500001439041\n" + "NXFSUWPNXH\t-199.648437500000\t-187.148437500000\n" + "TZODWKOCPF\t0.002563251997\t12.502563251997\n" + "PVKNCBWLNL\t0.397523656487\t12.897523656487\n" + "WQXYPOVFDB\t8.950848102570\t21.450848102570\n" + "NIJEEHRUGP\t448.000000000000\t460.500000000000\n" + "BTKVSBEGMI\t391.000000000000\t403.500000000000\n" + "NLKFNUHNRJ\t0.000000048669\t12.500000048669\n" + "BWVLOMPBET\t0.000038398017\t12.500038398017\n" + "KRIVOCUGPU\t2.929819107056\t15.429819107056\n" + "FIVQFNIZOS\t815.000000000000\t827.500000000000\n" + "SEPGIUQZHE\t0.017051883508\t12.517051883508\n" + "QHNOJIGFIN\t601.087127685547\t613.587127685547\n" + "QVZWEVQTQO\t-492.000000000000\t-479.500000000000\n" + "XTPNHTDCEB\t-966.000000000000\t-953.500000000000\n" + "XBBZVRLPTY\t70.810325622559\t83.310325622559\n" + "GYFUXCDKDW\t0.000837845349\t12.500837845349\n" + "DXCBJFRPXZ\t0.000008696692\t12.500008696692\n" + "XUNYQXTGNJ\t6.359375000000\t18.859375000000\n" + "LLEYMIWTCW\t792.242187500000\t804.742187500000\n" + "ORGFIEVMKP\t-726.000000000000\t-713.500000000000\n" + "GPYKKBMQMU\t0.000000000000\t12.500000000000\n" + "CIHCNPUGJO\t-762.240234375000\t-749.740234375000\n" + "EUKWMDNZZB\t0.000000031117\t12.500000031117\n" + "KOJSOLDYRO\t552.000000000000\t564.500000000000\n" + "PUNRPSMIFD\t73.947616577148\t86.447616577148\n" + "DKOEZBRQSQ\t-736.000000000000\t-723.500000000000\n" + "DIHHNSSTCR\t0.000002963134\t12.500002963134\n" + "PVQFULMERT\t0.000005817310\t12.500005817310\n" + "QBUYZVQQHS\t0.000000011688\t12.500000011688\n" + "PZPBHLNEJR\t-1003.625000000000\t-991.125000000000\n" + "IKDISGQFYQ\t351.509208679199\t364.009208679199\n" + "GPZNYVLTPK\t0.000702012621\t12.500702012621\n" + "QFNPOYNNCT\t0.000445737198\t12.500445737198\n" + "NSXHHDILEL\t0.001584486396\t12.501584486396\n" + "MMZSCJOUOU\t642.865234375000\t655.365234375000\n" + "ENFELWWRSL\t-934.268554687500\t-921.768554687500\n" + "QHGJBFQBBK\t103.908081054688\t116.408081054688\n" + "JZZYNPPBXB\t-944.000000000000\t-931.500000000000\n" + "RIIYMHOWKC\t0.000000542615\t12.500000542615\n" + "ZNLCNGZTOY\t0.000080315222\t12.500080315222\n" + "XRSFPVRQLG\t0.000000093226\t12.500000093226\n" + "ONNLITWGLF\t0.000000000000\t12.500000000000\n" + "QWPKLHTIIG\t-783.750000000000\t-771.250000000000\n" + "YYPDVRGRQG\t0.000376001219\t12.500376001219\n" + "PHKOWBVDEG\t0.000002666791\t12.500002666791\n" + "XGZMDJTHMH\t-670.500000000000\t-658.000000000000\n" + "VZHCNXZEQG\t-524.507812500000\t-512.007812500000\n" + "LUCFTLNKYT\t0.000000001835\t12.500000001835\n";
            Assert.assertEquals(expected, sink.toString());
        }
    }
}

Example 7 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method selectColumns0.

private RecordSource selectColumns0(final RecordSource recordSource, QueryModel model) throws ParserException {
    final ObjList<QueryColumn> columns = model.getColumns();
    final CharSequenceIntHashMap columnNameHistogram = model.getColumnNameHistogram();
    final RecordMetadata meta = recordSource.getMetadata();
    this.outerVirtualColumns.clear();
    this.innerVirtualColumn.clear();
    this.aggregators.clear();
    this.selectedColumns.clear();
    this.selectedColumnAliases.clear();
    this.groupKeyColumns.clear();
    this.aggregateColumnSequence = 0;
    this.analyticColumns.clear();
    RecordSource rs = recordSource;
    try {
        // create virtual columns from select list
        for (int i = 0, k = columns.size(); i < k; i++) {
            final QueryColumn qc = columns.getQuick(i);
            final ExprNode node = qc.getAst();
            final boolean analytic = qc instanceof AnalyticColumn;
            if (!analytic && node.type == ExprNode.LITERAL) {
                // check literal column validity
                if (meta.getColumnIndexQuiet(node.token) == -1) {
                    throw QueryError.invalidColumn(node.position, node.token);
                }
                // check if this column is exists in more than one journal/result set
                if (columnNameHistogram.get(node.token) > 0) {
                    throw QueryError.ambiguousColumn(node.position);
                }
                // add to selected columns
                selectedColumns.add(node.token);
                addAlias(node.position, qc.getName());
                groupKeyColumns.add(node.token);
                continue;
            }
            // generate missing alias for everything else
            if (qc.getAlias() == null) {
                qc.of(createAlias(aggregateColumnSequence++), node.position, node);
            }
            selectedColumns.add(qc.getAlias());
            addAlias(node.position, qc.getAlias());
            // outright aggregate
            if (!analytic && node.type == ExprNode.FUNCTION && FunctionFactories.isAggregate(node.token)) {
                aggregators.add(qc);
                continue;
            }
            // check if this expression references aggregate function
            if (node.type == ExprNode.OPERATION || node.type == ExprNode.FUNCTION) {
                int beforeSplit = aggregators.size();
                splitAggregates(node, aggregators);
                if (beforeSplit < aggregators.size()) {
                    outerVirtualColumns.add(qc);
                    continue;
                }
            }
            if (analytic) {
                if (qc.getAst().type != ExprNode.FUNCTION) {
                    // todo: not hit by test
                    throw QueryError.$(qc.getAst().position, "Analytic function expected");
                }
                if (aggregators.size() > 0) {
                    throw QueryError.$(qc.getAst().position, "Analytic function is not allowed in context of aggregation. Use sub-query.");
                }
                AnalyticColumn ac = (AnalyticColumn) qc;
                analyticColumns.add(ac);
            } else {
                // this is either a constant or non-aggregate expression
                // either case is a virtual column
                innerVirtualColumn.add(qc);
            }
        }
        // if virtual columns are present, create record source to calculate them
        if (innerVirtualColumn.size() > 0) {
            // this is either a constant or non-aggregate expression
            // either case is a virtual column
            ObjList<VirtualColumn> virtualColumns = new ObjList<>();
            final String timestampName;
            if (model.getSampleBy() != null) {
                timestampName = meta.getColumnName(getTimestampIndex(model, model.getTimestamp(), meta));
            } else {
                timestampName = null;
            }
            for (int i = 0, n = innerVirtualColumn.size(); i < n; i++) {
                QueryColumn qc = innerVirtualColumn.getQuick(i);
                if (Chars.equalsNc(qc.getAlias(), timestampName)) {
                    throw QueryError.$(qc.getAliasPosition() == -1 ? qc.getAst().position : qc.getAliasPosition(), "Alias clashes with implicit sample column name");
                }
                VirtualColumn vc = virtualColumnBuilder.createVirtualColumn(model, qc.getAst(), meta);
                vc.setName(qc.getAlias());
                virtualColumns.add(vc);
                groupKeyColumns.add(qc.getAlias());
            }
            rs = new VirtualColumnRecordSource(rs, virtualColumns);
        }
        // if aggregators present, wrap record source into group-by source
        if (aggregators.size() > 0) {
            rs = compileAggregates(rs, model);
        } else {
            ExprNode sampleBy = model.getSampleBy();
            if (sampleBy != null) {
                throw QueryError.$(sampleBy.position, "There are no aggregation columns");
            }
        }
        if (outerVirtualColumns.size() > 0) {
            rs = compileOuterVirtualColumns(rs, model);
        }
        if (analyticColumns.size() > 0) {
            rs = compileAnalytic(rs, model);
        }
        if (selectedColumns.size() > 0) {
            // wrap underlying record source into selected columns source.
            rs = new SelectedColumnsRecordSource(rs, selectedColumns, selectedColumnAliases);
        }
        return rs;
    } catch (ParserException e) {
        Misc.free(rs);
        throw e;
    }
}

Example 8 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method compileAnalytic.

private RecordSource compileAnalytic(RecordSource rs, QueryModel model) throws ParserException {
    final int n = analyticColumns.size();
    grouppedAnalytic.clear();
    final RecordMetadata metadata = rs.getMetadata();
    ObjList<AnalyticFunction> naturalOrderFunctions = null;
    boolean needCache = false;
    for (int i = 0; i < n; i++) {
        AnalyticColumn col = analyticColumns.getQuick(i);
        ObjList<VirtualColumn> partitionBy = null;
        int psz = col.getPartitionBy().size();
        if (psz > 0) {
            partitionBy = new ObjList<>(psz);
            for (int j = 0; j < psz; j++) {
                partitionBy.add(virtualColumnBuilder.createVirtualColumn(model, col.getPartitionBy().getQuick(j), metadata));
            }
        }
        ExprNode ast = col.getAst();
        if (ast.paramCount > 1) {
            throw QueryError.$(col.getAst().position, "Too many arguments");
        }
        VirtualColumn valueColumn;
        if (ast.paramCount == 1) {
            valueColumn = virtualColumnBuilder.createVirtualColumn(model, col.getAst().rhs, metadata);
            valueColumn.setName(col.getAlias());
        } else {
            valueColumn = null;
        }
        final int osz = col.getOrderBy().size();
        AnalyticFunction f = AnalyticFunctionFactories.newInstance(configuration, ast.token, valueColumn, col.getAlias(), partitionBy, rs.supportsRowIdAccess(), osz > 0);
        if (f == null) {
            Misc.free(rs);
            throw QueryError.$(col.getAst().position, "Unknown function");
        }
        CharSequenceIntHashMap orderHash = model.getOrderHash();
        boolean dismissOrder;
        if (osz > 0 && orderHash.size() > 0) {
            dismissOrder = true;
            for (int j = 0; j < osz; j++) {
                ExprNode node = col.getOrderBy().getQuick(j);
                int direction = col.getOrderByDirection().getQuick(j);
                if (orderHash.get(node.token) != direction) {
                    dismissOrder = false;
                }
            }
        } else {
            dismissOrder = false;
        }
        if (osz > 0 && !dismissOrder) {
            IntList order = toOrderIndices(metadata, col.getOrderBy(), col.getOrderByDirection());
            ObjList<AnalyticFunction> funcs = grouppedAnalytic.get(order);
            if (funcs == null) {
                grouppedAnalytic.put(order, funcs = new ObjList<>());
            }
            funcs.add(f);
            needCache = true;
        } else {
            if (naturalOrderFunctions == null) {
                naturalOrderFunctions = new ObjList<>();
            }
            needCache = needCache || f.getType() != AnalyticFunction.STREAM;
            naturalOrderFunctions.add(f);
        }
    }
    if (needCache) {
        final ObjList<RecordComparator> analyticComparators = new ObjList<>(grouppedAnalytic.size());
        final ObjList<ObjList<AnalyticFunction>> functionGroups = new ObjList<>(grouppedAnalytic.size());
        for (ObjObjHashMap.Entry<IntList, ObjList<AnalyticFunction>> e : grouppedAnalytic) {
            analyticComparators.add(cc.compile(metadata, e.key));
            functionGroups.add(e.value);
        }
        if (naturalOrderFunctions != null) {
            analyticComparators.add(null);
            functionGroups.add(naturalOrderFunctions);
        }
        if (rs.supportsRowIdAccess()) {
            return new CachedRowAnalyticRecordSource(configuration.getDbAnalyticWindowPage(), rs, analyticComparators, functionGroups);
        }
        return new CachedAnalyticRecordSource(configuration.getDbAnalyticWindowPage(), configuration.getDbSortKeyPage(), rs, analyticComparators, functionGroups);
    } else {
        assert naturalOrderFunctions != null;
        return new AnalyticRecordSource(rs, naturalOrderFunctions);
    }
}

Example 9 with VirtualColumn

use of com.questdb.ql.ops.VirtualColumn in project questdb by bluestreak01.

the class QueryCompiler method joinModelIsFalse.

private boolean joinModelIsFalse(QueryModel model) throws ParserException {
    ExprNode current = null;
    IntHashSet constants = model.getParsedWhereConsts();
    ObjList<ExprNode> whereNodes = model.getParsedWhere();
    for (int i = 0, n = constants.size(); i < n; i++) {
        current = concatFilters(current, whereNodes.getQuick(constants.get(i)));
    }
    if (current == null) {
        return false;
    }
    VirtualColumn col = virtualColumnBuilder.createVirtualColumn(model, current, null);
    if (col.isConstant()) {
        if (col.getType() != ColumnType.BOOLEAN) {
            throw QueryError.$(current.position, "Boolean expression expected");
        }
        return !col.getBool(null);
    } else {
        throw QueryError.$(0, "Internal error: expected constant");
    }
}