Examples with ElementSymbol org.teiid.query.sql.symbol.ElementSymbol used on opensource projects

Example 31 with ElementSymbol

use of org.teiid.query.sql.symbol.ElementSymbol in project teiid by teiid.

the class SubqueryAwareEvaluator method evaluateSubquery.

@Override
protected ValueIterator evaluateSubquery(SubqueryContainer<?> container, List<?> tuple) throws TeiidProcessingException, BlockedException, TeiidComponentException {
    ContextReference ref = (ContextReference) container;
    String key = ref.getContextSymbol();
    SubqueryState state = this.subqueries.get(key);
    if (state == null) {
        String otherKey = commands.get(container.getCommand());
        if (otherKey != null) {
            state = this.subqueries.get(otherKey);
            if (state != null) {
                key = otherKey;
            }
        }
    }
    if (state == null) {
        state = new SubqueryState();
        state.plan = container.getCommand().getProcessorPlan().clone();
        if (container.getCommand().getCorrelatedReferences() != null) {
            for (ElementSymbol es : container.getCommand().getCorrelatedReferences().getKeys()) {
                if (DataTypeManager.isNonComparable(DataTypeManager.getDataTypeName(es.getType()))) {
                    state.comparable = false;
                    break;
                }
            }
        }
        this.subqueries.put(key, state);
        this.commands.put(container.getCommand(), key);
    }
    SymbolMap correlatedRefs = container.getCommand().getCorrelatedReferences();
    VariableContext currentContext = null;
    boolean shouldClose = false;
    boolean deterministic = true;
    if (state.processor != null && correlatedRefs != null) {
        Determinism determinism = state.processor.getContext().getDeterminismLevel();
        deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
    }
    boolean removeBuffer = true;
    if (correlatedRefs != null) {
        currentContext = new VariableContext();
        for (Map.Entry<ElementSymbol, Expression> entry : container.getCommand().getCorrelatedReferences().asMap().entrySet()) {
            currentContext.setValue(entry.getKey(), evaluate(entry.getValue(), tuple));
        }
        List<Object> refValues = currentContext.getLocalValues();
        if (!refValues.equals(state.refValues)) {
            if (state.comparable && deterministic) {
                if (state.processor != null) {
                    // cache the old value
                    TupleBuffer tb = state.collector.collectTuples();
                    // recheck determinism as the plan may not have been fully processed by the initial check
                    Determinism determinism = state.processor.getContext().getDeterminismLevel();
                    deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
                    if (deterministic) {
                        // allowed to track up to 4x the maximum results size
                        maxTuples = Math.max((int) Math.min(Integer.MAX_VALUE, tb.getRowCount() << 2), maxTuples);
                        ArrayList<Object> cacheKey = new ArrayList<Object>(state.refValues);
                        cacheKey.add(key);
                        // ensure that we aren't leaving large last batches in memory
                        tb.saveBatch();
                        this.cache.put(cacheKey, tb);
                        removeBuffer = false;
                        this.currentTuples += tb.getRowCount();
                        while (this.currentTuples > maxTuples && !cache.isEmpty()) {
                            Iterator<Map.Entry<List<?>, TupleBuffer>> i = this.cache.entrySet().iterator();
                            Map.Entry<List<?>, TupleBuffer> entry = i.next();
                            TupleBuffer buffer = entry.getValue();
                            if (buffer.getRowCount() <= 2) {
                                this.smallCache.put(entry.getKey(), buffer);
                            } else {
                                buffer.remove();
                            }
                            this.currentTuples -= buffer.getRowCount();
                            i.remove();
                        }
                    }
                }
                // find if we have cached values
                List<Object> cacheKey = new ArrayList<Object>(refValues);
                cacheKey.add(key);
                TupleBuffer cachedResult = cache.get(cacheKey);
                if (cachedResult == null) {
                    cachedResult = smallCache.get(cacheKey);
                }
                if (cachedResult != null) {
                    state.close(false);
                    return new TupleSourceValueIterator(cachedResult.createIndexedTupleSource(), 0);
                }
            }
            state.refValues = refValues;
            shouldClose = true;
        }
    }
    if (shouldClose || (!deterministic && !state.blocked)) {
        state.close(removeBuffer);
    }
    state.blocked = true;
    if (state.processor == null) {
        CommandContext subContext = context.clone();
        state.plan.reset();
        state.processor = new QueryProcessor(state.plan, subContext, manager, this.dataMgr);
        if (currentContext != null) {
            state.processor.getContext().pushVariableContext(currentContext);
        }
        state.collector = state.processor.createBatchCollector();
    }
    TupleSourceValueIterator iter = new TupleSourceValueIterator(state.collector.collectTuples().createIndexedTupleSource(), 0);
    state.blocked = false;
    return iter;
}

Example 32 with ElementSymbol

use of org.teiid.query.sql.symbol.ElementSymbol in project teiid by teiid.

the class SubqueryAwareEvaluator method evaluatePushdown.

/**
 * Implements must pushdown function handling if supported by the source.
 *
 * The basic strategy is to create a dummy subquery to represent the evaluation
 */
@Override
protected Object evaluatePushdown(Function function, List<?> tuple, Object[] values) throws TeiidComponentException, TeiidProcessingException {
    final FunctionDescriptor fd = function.getFunctionDescriptor();
    if (fd.getMethod() == null) {
        throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
    }
    String schema = null;
    if (fd.getMethod().getParent() == null || !fd.getMethod().getParent().isPhysical()) {
        // find a suitable target
        // TODO: do better than a linear search
        VDBMetaData vdb = this.context.getVdb();
        CapabilitiesFinder capabiltiesFinder = this.context.getQueryProcessorFactory().getCapabiltiesFinder();
        for (ModelMetaData mmd : vdb.getModelMetaDatas().values()) {
            if (!mmd.isSource()) {
                continue;
            }
            SourceCapabilities caps = capabiltiesFinder.findCapabilities(mmd.getName());
            if (caps.supportsCapability(Capability.SELECT_WITHOUT_FROM) && caps.supportsFunction(fd.getMethod().getFullName())) {
                schema = mmd.getName();
                break;
            }
        }
        if (schema == null) {
            throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
        }
    } else {
        if (!CapabilitiesUtil.supports(Capability.SELECT_WITHOUT_FROM, fd.getMethod().getParent(), context.getMetadata(), context.getQueryProcessorFactory().getCapabiltiesFinder())) {
            if (elements != null) {
                Integer index = (Integer) elements.get(function);
                if (index != null) {
                    return tuple.get(index.intValue());
                }
            }
            throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
        }
        schema = fd.getSchema();
    }
    ScalarSubquery ss = null;
    if (functionState != null) {
        ss = functionState.get(function);
    }
    Expression[] functionArgs = new Expression[values.length];
    for (int i = 0; i < values.length; i++) {
        functionArgs[i] = new Constant(values[i]);
    }
    if (ss == null) {
        final Query command = new Query();
        Select select = new Select();
        command.setSelect(select);
        Function f = new Function(function.getName(), functionArgs);
        f.setType(function.getType());
        f.setFunctionDescriptor(fd);
        select.addSymbol(f);
        ss = new ScalarSubquery(command);
        SymbolMap correlatedReferences = new SymbolMap();
        Collection<ElementSymbol> elements = ElementCollectorVisitor.getElements(function, true);
        if (!elements.isEmpty()) {
            for (ElementSymbol es : elements) {
                correlatedReferences.addMapping(es, es);
            }
            command.setCorrelatedReferences(correlatedReferences);
        }
        command.setProcessorPlan(new SimpleProcessorPlan(command, schema, fd, Arrays.asList(new Constant(null, fd.getReturnType()))));
    } else {
        ((Function) ((ExpressionSymbol) ss.getCommand().getProjectedSymbols().get(0)).getExpression()).setArgs(functionArgs);
    }
    if (functionState == null) {
        this.functionState = new HashMap<Function, ScalarSubquery>(2);
    }
    functionState.put(function, ss);
    return internalEvaluate(ss, tuple);
}

Example 33 with ElementSymbol

use of org.teiid.query.sql.symbol.ElementSymbol in project teiid by teiid.

the class WindowFunctionProjectNode method buildResults.

/**
 * Build the results by maintaining indexes that either map
 * rowid->values
 * or
 * rowid->partitionid and partitionid->values
 *
 * TODO use the size hint for tree balancing
 */
private void buildResults() throws TeiidComponentException, TeiidProcessingException, FunctionExecutionException, ExpressionEvaluationException {
    List<Map.Entry<WindowSpecification, WindowSpecificationInfo>> specs = new ArrayList<Map.Entry<WindowSpecification, WindowSpecificationInfo>>(windows.entrySet());
    for (int specIndex = 0; specIndex < specs.size(); specIndex++) {
        Map.Entry<WindowSpecification, WindowSpecificationInfo> entry = specs.get(specIndex);
        WindowSpecificationInfo info = entry.getValue();
        IndexedTupleSource specificationTs = tb.createIndexedTupleSource();
        boolean multiGroup = false;
        int[] partitionIndexes = null;
        int[] orderIndexes = null;
        // if there is partitioning or ordering, then sort
        if (!info.orderType.isEmpty()) {
            multiGroup = true;
            int[] sortKeys = new int[info.orderType.size()];
            int i = 0;
            if (!info.groupIndexes.isEmpty()) {
                for (Integer sortIndex : info.groupIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                partitionIndexes = Arrays.copyOf(sortKeys, info.groupIndexes.size());
            }
            if (!info.sortIndexes.isEmpty()) {
                for (Integer sortIndex : info.sortIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                orderIndexes = Arrays.copyOfRange(sortKeys, info.groupIndexes.size(), info.groupIndexes.size() + info.sortIndexes.size());
            }
            if (!info.functions.isEmpty()) {
                // $NON-NLS-1$
                ElementSymbol key = new ElementSymbol("rowId");
                key.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                // $NON-NLS-1$
                ElementSymbol value = new ElementSymbol("partitionId");
                value.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                List<ElementSymbol> elements = Arrays.asList(key, value);
                partitionMapping[specIndex] = this.getBufferManager().createSTree(elements, this.getConnectionID(), 1);
            }
            SortUtility su = new SortUtility(null, Mode.SORT, this.getBufferManager(), this.getConnectionID(), tb.getSchema(), info.orderType, info.nullOrderings, sortKeys);
            su.setWorkingBuffer(tb);
            su.setNonBlocking(true);
            TupleBuffer sorted = su.sort();
            specificationTs = sorted.createIndexedTupleSource(true);
        }
        List<AggregateFunction> aggs = initializeAccumulators(info.functions, specIndex, false);
        List<AggregateFunction> rowValueAggs = initializeAccumulators(info.rowValuefunctions, specIndex, true);
        int groupId = 0;
        List<?> lastRow = null;
        while (specificationTs.hasNext()) {
            List<?> tuple = specificationTs.nextTuple();
            if (multiGroup) {
                if (lastRow != null) {
                    boolean samePartition = GroupingNode.sameGroup(partitionIndexes, tuple, lastRow) == -1;
                    if (!aggs.isEmpty() && (!samePartition || GroupingNode.sameGroup(orderIndexes, tuple, lastRow) != -1)) {
                        saveValues(specIndex, aggs, groupId, samePartition, false);
                        groupId++;
                    }
                    saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), samePartition, true);
                }
                if (!aggs.isEmpty()) {
                    List<Object> partitionTuple = Arrays.asList(tuple.get(tuple.size() - 1), groupId);
                    partitionMapping[specIndex].insert(partitionTuple, InsertMode.NEW, -1);
                }
            }
            for (AggregateFunction function : aggs) {
                function.addInput(tuple, getContext());
            }
            for (AggregateFunction function : rowValueAggs) {
                function.addInput(tuple, getContext());
            }
            lastRow = tuple;
        }
        if (lastRow != null) {
            saveValues(specIndex, aggs, groupId, true, false);
            saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), true, true);
        }
    }
}

Example 34 with ElementSymbol

use of org.teiid.query.sql.symbol.ElementSymbol in project teiid by teiid.

the class WindowFunctionProjectNode method saveInput.

/**
 * Save the input generating any necessary expressions and adding a row id
 * @param collectedExpressions
 * @return
 * @throws TeiidComponentException
 * @throws TeiidProcessingException
 */
private void saveInput() throws TeiidComponentException, TeiidProcessingException {
    if (inputTs == null) {
        List<Expression> collectedExpressions = new ArrayList<Expression>(expressionIndexes.keySet());
        Evaluator eval = new Evaluator(elementMap, getDataManager(), getContext());
        final RelationalNode sourceNode = this.getChildren()[0];
        inputTs = new ProjectingTupleSource(sourceNode, eval, collectedExpressions, elementMap) {

            int index = 0;

            @Override
            public List<Object> nextTuple() throws TeiidComponentException, TeiidProcessingException {
                List<Object> tuple = super.nextTuple();
                if (tuple != null) {
                    tuple.add(index++);
                }
                return tuple;
            }
        };
        List<ElementSymbol> schema = new ArrayList<ElementSymbol>(collectedExpressions.size() + 1);
        int index = 0;
        for (Expression ex : collectedExpressions) {
            ElementSymbol es = new ElementSymbol(String.valueOf(index++));
            es.setType(ex.getType());
            schema.add(es);
        }
        // add in the row id
        ElementSymbol es = new ElementSymbol(String.valueOf(index++));
        es.setType(DataTypeManager.DefaultDataClasses.INTEGER);
        schema.add(es);
        tb = this.getBufferManager().createTupleBuffer(schema, this.getConnectionID(), TupleSourceType.PROCESSOR);
    }
    List<?> tuple = null;
    while ((tuple = inputTs.nextTuple()) != null) {
        tb.addTuple(tuple);
    }
    tb.close();
    inputTs.closeSource();
    inputTs = null;
}

Example 35 with ElementSymbol

use of org.teiid.query.sql.symbol.ElementSymbol in project teiid by teiid.

the class WindowFunctionProjectNode method initializeAccumulators.

/**
 * @param functions
 * @param specIndex
 * @param rowValues
 * @return
 */
private List<AggregateFunction> initializeAccumulators(List<WindowFunctionInfo> functions, int specIndex, boolean rowValues) {
    List<AggregateFunction> aggs = new ArrayList<AggregateFunction>(functions.size());
    if (functions.isEmpty()) {
        return aggs;
    }
    List<ElementSymbol> elements = new ArrayList<ElementSymbol>(functions.size());
    // $NON-NLS-1$
    ElementSymbol key = new ElementSymbol("key");
    key.setType(DataTypeManager.DefaultDataClasses.INTEGER);
    elements.add(key);
    for (WindowFunctionInfo wfi : functions) {
        aggs.add(GroupingNode.initAccumulator(wfi.function.getFunction(), this, expressionIndexes));
        Class<?> outputType = wfi.function.getType();
        if (wfi.function.getFunction().getAggregateFunction() == Type.LEAD || wfi.function.getFunction().getAggregateFunction() == Type.LAG) {
            outputType = DataTypeManager.getArrayType(DataTypeManager.DefaultDataClasses.OBJECT);
        }
        // $NON-NLS-1$
        ElementSymbol value = new ElementSymbol("val");
        value.setType(outputType);
        elements.add(value);
    }
    if (!rowValues) {
        valueMapping[specIndex] = this.getBufferManager().createSTree(elements, this.getConnectionID(), 1);
    } else {
        rowValueMapping[specIndex] = this.getBufferManager().createSTree(elements, this.getConnectionID(), 1);
    }
    return aggs;
}