Examples with IndexedTupleSource org.teiid.common.buffer.IndexedTupleSource used on opensource projects

Example 1 with IndexedTupleSource

use of org.teiid.common.buffer.IndexedTupleSource in project teiid by teiid.

the class WindowFunctionProjectNode method buildResults.

/**
 * Build the results by maintaining indexes that either map
 * rowid->values
 * or
 * rowid->partitionid and partitionid->values
 *
 * TODO use the size hint for tree balancing
 */
private void buildResults() throws TeiidComponentException, TeiidProcessingException, FunctionExecutionException, ExpressionEvaluationException {
    List<Map.Entry<WindowSpecification, WindowSpecificationInfo>> specs = new ArrayList<Map.Entry<WindowSpecification, WindowSpecificationInfo>>(windows.entrySet());
    for (int specIndex = 0; specIndex < specs.size(); specIndex++) {
        Map.Entry<WindowSpecification, WindowSpecificationInfo> entry = specs.get(specIndex);
        WindowSpecificationInfo info = entry.getValue();
        IndexedTupleSource specificationTs = tb.createIndexedTupleSource();
        boolean multiGroup = false;
        int[] partitionIndexes = null;
        int[] orderIndexes = null;
        // if there is partitioning or ordering, then sort
        if (!info.orderType.isEmpty()) {
            multiGroup = true;
            int[] sortKeys = new int[info.orderType.size()];
            int i = 0;
            if (!info.groupIndexes.isEmpty()) {
                for (Integer sortIndex : info.groupIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                partitionIndexes = Arrays.copyOf(sortKeys, info.groupIndexes.size());
            }
            if (!info.sortIndexes.isEmpty()) {
                for (Integer sortIndex : info.sortIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                orderIndexes = Arrays.copyOfRange(sortKeys, info.groupIndexes.size(), info.groupIndexes.size() + info.sortIndexes.size());
            }
            if (!info.functions.isEmpty()) {
                // $NON-NLS-1$
                ElementSymbol key = new ElementSymbol("rowId");
                key.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                // $NON-NLS-1$
                ElementSymbol value = new ElementSymbol("partitionId");
                value.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                List<ElementSymbol> elements = Arrays.asList(key, value);
                partitionMapping[specIndex] = this.getBufferManager().createSTree(elements, this.getConnectionID(), 1);
            }
            SortUtility su = new SortUtility(null, Mode.SORT, this.getBufferManager(), this.getConnectionID(), tb.getSchema(), info.orderType, info.nullOrderings, sortKeys);
            su.setWorkingBuffer(tb);
            su.setNonBlocking(true);
            TupleBuffer sorted = su.sort();
            specificationTs = sorted.createIndexedTupleSource(true);
        }
        List<AggregateFunction> aggs = initializeAccumulators(info.functions, specIndex, false);
        List<AggregateFunction> rowValueAggs = initializeAccumulators(info.rowValuefunctions, specIndex, true);
        int groupId = 0;
        List<?> lastRow = null;
        while (specificationTs.hasNext()) {
            List<?> tuple = specificationTs.nextTuple();
            if (multiGroup) {
                if (lastRow != null) {
                    boolean samePartition = GroupingNode.sameGroup(partitionIndexes, tuple, lastRow) == -1;
                    if (!aggs.isEmpty() && (!samePartition || GroupingNode.sameGroup(orderIndexes, tuple, lastRow) != -1)) {
                        saveValues(specIndex, aggs, groupId, samePartition, false);
                        groupId++;
                    }
                    saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), samePartition, true);
                }
                if (!aggs.isEmpty()) {
                    List<Object> partitionTuple = Arrays.asList(tuple.get(tuple.size() - 1), groupId);
                    partitionMapping[specIndex].insert(partitionTuple, InsertMode.NEW, -1);
                }
            }
            for (AggregateFunction function : aggs) {
                function.addInput(tuple, getContext());
            }
            for (AggregateFunction function : rowValueAggs) {
                function.addInput(tuple, getContext());
            }
            lastRow = tuple;
        }
        if (lastRow != null) {
            saveValues(specIndex, aggs, groupId, true, false);
            saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), true, true);
        }
    }
}

Example 2 with IndexedTupleSource

use of org.teiid.common.buffer.IndexedTupleSource in project teiid by teiid.

the class DependentValueSource method getValueIterator.

/**
 * @throws TeiidComponentException
 * @see org.teiid.query.sql.util.ValueIteratorSource#getValueIterator(org.teiid.query.sql.symbol.Expression)
 */
public TupleSourceValueIterator getValueIterator(Expression valueExpression) throws TeiidComponentException {
    IndexedTupleSource its = buffer.createIndexedTupleSource();
    int index = 0;
    if (valueExpression != null) {
        if (valueExpression instanceof Array) {
            final Array array = (Array) valueExpression;
            List<Expression> exprs = array.getExpressions();
            final int[] indexes = new int[exprs.size()];
            for (int i = 0; i < exprs.size(); i++) {
                indexes[i] = getIndex(exprs.get(i));
            }
            return new TupleSourceValueIterator(its, index) {

                @Override
                public Object next() throws TeiidComponentException {
                    List<?> tuple = super.nextTuple();
                    Object[] a = (Object[]) java.lang.reflect.Array.newInstance(array.getComponentType(), indexes.length);
                    for (int i = 0; i < indexes.length; i++) {
                        a[i] = tuple.get(indexes[i]);
                        if (a[i] == null) {
                            // TODO: this is a hack
                            return null;
                        }
                    }
                    return new ArrayImpl(a);
                }
            };
        }
        index = getIndex(valueExpression);
    }
    return new TupleSourceValueIterator(its, index);
}

Example 3 with IndexedTupleSource

use of org.teiid.common.buffer.IndexedTupleSource in project teiid by teiid.

the class NestedTableJoinStrategy method process.

@Override
protected void process() throws TeiidComponentException, TeiidProcessingException {
    IndexedTupleSource its = leftSource.getIterator();
    while (its.hasNext() || leftSource.getCurrentTuple() != null) {
        List<?> leftTuple = leftSource.getCurrentTuple();
        if (leftTuple == null) {
            leftTuple = leftSource.saveNext();
        }
        updateContext(leftTuple, leftSource.getSource().getElements());
        if (rightMap != null && !rightSource.open) {
            for (Map.Entry<ElementSymbol, Expression> entry : rightMap.asMap().entrySet()) {
                joinNode.getContext().getVariableContext().setValue(entry.getKey(), eval.evaluate(entry.getValue(), null));
            }
            rightSource.getSource().reset();
            super.openRight();
        }
        IndexedTupleSource right = rightSource.getIterator();
        while (right.hasNext() || rightSource.getCurrentTuple() != null) {
            List<?> rightTuple = rightSource.getCurrentTuple();
            if (rightTuple == null) {
                rightTuple = rightSource.saveNext();
            }
            List<?> outputTuple = outputTuple(this.leftSource.getCurrentTuple(), this.rightSource.getCurrentTuple());
            boolean matches = this.joinNode.matchesCriteria(outputTuple);
            rightSource.saveNext();
            if (matches) {
                outerMatched = true;
                joinNode.addBatchRow(outputTuple);
            }
        }
        if (!outerMatched && this.joinNode.getJoinType() == JoinType.JOIN_LEFT_OUTER) {
            joinNode.addBatchRow(outputTuple(this.leftSource.getCurrentTuple(), this.rightSource.getOuterVals()));
        }
        outerMatched = false;
        if (rightMap == null) {
            rightSource.getIterator().setPosition(1);
        } else {
            rightSource.close();
            for (Map.Entry<ElementSymbol, Expression> entry : rightMap.asMap().entrySet()) {
                joinNode.getContext().getVariableContext().remove(entry.getKey());
            }
        }
        leftSource.saveNext();
        updateContext(null, leftSource.getSource().getElements());
    }
}

Example 4 with IndexedTupleSource

use of org.teiid.common.buffer.IndexedTupleSource in project teiid by teiid.

the class EnhancedSortMergeJoinStrategy method createIndex.

/**
 * Create an index of the smaller size
 */
public void createIndex(SourceState state, SortOption sortOption) throws TeiidComponentException, TeiidProcessingException {
    // this is inefficient as it fully buffers, then builds the index.  if possible
    // we should build off of the streaming batches
    IndexedTupleSource its = state.getTupleBuffer().createIndexedTupleSource(!joinNode.isDependent());
    boolean sorted = sortOption == SortOption.ALREADY_SORTED;
    int[] expressionIndexes = state.getExpressionIndexes();
    int keyLength = expressionIndexes.length;
    List elements = state.getSource().getOutputElements();
    // TODO: minimize reordering, or at least detect when it's not necessary
    LinkedHashSet<Integer> used = new LinkedHashSet<Integer>();
    for (int i : expressionIndexes) {
        used.add(i);
    }
    int[] reorderedSortIndex = Arrays.copyOf(expressionIndexes, keyLength + elements.size() - used.size());
    int j = keyLength;
    for (int i = 0; i < elements.size(); i++) {
        if (!used.contains(i)) {
            reorderedSortIndex[j++] = i;
        }
    }
    List<Expression> reordered = RelationalNode.projectTuple(reorderedSortIndex, elements);
    if (sortOption == SortOption.SORT_DISTINCT) {
        keyLength = elements.size();
    } else if (!state.isExpresssionDistinct()) {
        // need to add a rowid, just in case
        reordered = new ArrayList<Expression>(reordered);
        // $NON-NLS-1$
        ElementSymbol id = new ElementSymbol("rowId");
        id.setType(DataTypeManager.DefaultDataClasses.INTEGER);
        reordered.add(keyLength, id);
        keyLength++;
    }
    index = this.joinNode.getBufferManager().createSTree(reordered, this.joinNode.getConnectionID(), keyLength);
    index.setPreferMemory(true);
    if (sortOption == SortOption.SORT_DISTINCT) {
        index.getComparator().setDistinctIndex(expressionIndexes.length);
    } else if (!state.isExpresssionDistinct()) {
        index.getComparator().setDistinctIndex(keyLength - 2);
    }
    int rowId = 0;
    List<?> lastTuple = null;
    boolean sortedDistinct = sorted && !state.isExpresssionDistinct();
    int sizeHint = index.getExpectedHeight(state.getRowCount());
    index.setBatchInsert(sorted);
    outer: while (its.hasNext()) {
        // detect if sorted and distinct
        List<?> originalTuple = its.nextTuple();
        // remove the tuple if it has null
        for (int i : expressionIndexes) {
            if (originalTuple.get(i) == null) {
                continue outer;
            }
        }
        if (sortedDistinct && lastTuple != null && this.compare(lastTuple, originalTuple, expressionIndexes, expressionIndexes) == 0) {
            sortedDistinct = false;
        }
        lastTuple = originalTuple;
        List<Object> tuple = (List<Object>) RelationalNode.projectTuple(reorderedSortIndex, originalTuple);
        if (!state.isExpresssionDistinct() && sortOption != SortOption.SORT_DISTINCT) {
            tuple.add(keyLength - 1, rowId++);
        }
        index.insert(tuple, sorted ? InsertMode.ORDERED : InsertMode.NEW, sizeHint);
    }
    if (!sorted) {
        index.compact();
    } else {
        index.setBatchInsert(false);
    }
    its.closeSource();
    this.reverseIndexes = new int[elements.size()];
    for (int i = 0; i < reorderedSortIndex.length; i++) {
        int oldIndex = reorderedSortIndex[i];
        this.reverseIndexes[oldIndex] = i + ((!state.isExpresssionDistinct() && (i >= keyLength - 1) && sortOption != SortOption.SORT_DISTINCT) ? 1 : 0);
    }
    // TODO: this logic doesn't catch distinct join expressions when using sort distinct very well
    if (!state.isExpresssionDistinct() && ((!sorted && index.getComparator().isDistinct()) || (sorted && sortedDistinct))) {
        if (sortOption != SortOption.SORT_DISTINCT) {
            this.index.removeRowIdFromKey();
        }
        state.markExpressionsDistinct(true);
    }
    keyTs = new SingleTupleSource();
    keyTs.indexes = this.notSortedSource.getExpressionIndexes();
    tb = new TupleBrowser(this.index, keyTs, OrderBy.ASC);
}