Examples with SequenceResultIterator org.apache.phoenix.iterate.SequenceResultIterator used on opensource projects

Example 1 with SequenceResultIterator

use of org.apache.phoenix.iterate.SequenceResultIterator in project phoenix by apache.

the class AggregatePlan method newIterator.

@Override
protected ResultIterator newIterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException {
    if (groupBy.isEmpty()) {
        UngroupedAggregateRegionObserver.serializeIntoScan(scan);
    } else {
        // Set attribute with serialized expressions for coprocessor
        GroupedAggregateRegionObserver.serializeIntoScan(scan, groupBy.getScanAttribName(), groupBy.getKeyExpressions());
        if (limit != null && orderBy.getOrderByExpressions().isEmpty() && having == null && ((statement.isDistinct() && !statement.isAggregate()) || (!statement.isDistinct() && (context.getAggregationManager().isEmpty() || BaseScannerRegionObserver.KEY_ORDERED_GROUP_BY_EXPRESSIONS.equals(groupBy.getScanAttribName()))))) {
            /*
                 * Optimization to early exit from the scan for a GROUP BY or DISTINCT with a LIMIT.
                 * We may exit early according to the LIMIT specified if the query has:
                 * 1) No ORDER BY clause (or the ORDER BY was optimized out). We cannot exit
                 *    early if there's an ORDER BY because the first group may be found last
                 *    in the scan.
                 * 2) No HAVING clause, since we execute the HAVING on the client side. The LIMIT
                 *    needs to be evaluated *after* the HAVING.
                 * 3) DISTINCT clause with no GROUP BY. We cannot exit early if there's a
                 *    GROUP BY, as the GROUP BY is processed on the client-side post aggregation
                 *    if a DISTNCT has a GROUP BY. Otherwise, since there are no aggregate
                 *    functions in a DISTINCT, we can exit early regardless of if the
                 *    groups are in row key order or unordered.
                 * 4) GROUP BY clause with no aggregate functions. This is in the same category
                 *    as (3). If we're using aggregate functions, we need to look at all the
                 *    rows, as otherwise we'd exit early with incorrect aggregate function
                 *    calculations.
                 * 5) GROUP BY clause along the pk axis, as the rows are processed in row key
                 *    order, so we can early exit, even when aggregate functions are used, as
                 *    the rows in the group are contiguous.
                 */
            scan.setAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT, PInteger.INSTANCE.toBytes(limit + (offset == null ? 0 : offset)));
        }
    }
    BaseResultIterators iterators = isSerial ? new SerialIterators(this, null, null, wrapParallelIteratorFactory(), scanGrouper, scan) : new ParallelIterators(this, null, wrapParallelIteratorFactory(), scan, false);
    estimatedRows = iterators.getEstimatedRowCount();
    estimatedSize = iterators.getEstimatedByteCount();
    splits = iterators.getSplits();
    scans = iterators.getScans();
    AggregatingResultIterator aggResultIterator;
    // No need to merge sort for ungrouped aggregation
    if (groupBy.isEmpty() || groupBy.isUngroupedAggregate()) {
        aggResultIterator = new UngroupedAggregatingResultIterator(new ConcatResultIterator(iterators), aggregators);
    // If salted or local index we still need a merge sort as we'll potentially have multiple group by keys that aren't contiguous.
    } else if (groupBy.isOrderPreserving() && !(this.getTableRef().getTable().getBucketNum() != null || this.getTableRef().getTable().getIndexType() == IndexType.LOCAL)) {
        aggResultIterator = new RowKeyOrderedAggregateResultIterator(iterators, aggregators);
    } else {
        aggResultIterator = new GroupedAggregatingResultIterator(new MergeSortRowKeyResultIterator(iterators, 0, this.getOrderBy() == OrderBy.REV_ROW_KEY_ORDER_BY), aggregators);
    }
    if (having != null) {
        aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having);
    }
    if (statement.isDistinct() && statement.isAggregate()) {
        // Dedup on client if select distinct and aggregation
        aggResultIterator = new DistinctAggregatingResultIterator(aggResultIterator, getProjector());
    }
    ResultIterator resultScanner = aggResultIterator;
    if (orderBy.getOrderByExpressions().isEmpty()) {
        if (offset != null) {
            resultScanner = new OffsetResultIterator(aggResultIterator, offset);
        }
        if (limit != null) {
            resultScanner = new LimitingResultIterator(resultScanner, limit);
        }
    } else {
        int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
        resultScanner = new OrderedAggregatingResultIterator(aggResultIterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset);
    }
    if (context.getSequenceManager().getSequenceCount() > 0) {
        resultScanner = new SequenceResultIterator(resultScanner, context.getSequenceManager());
    }
    return resultScanner;
}

Example 2 with SequenceResultIterator

use of org.apache.phoenix.iterate.SequenceResultIterator in project phoenix by apache.

the class ClientScanPlan method iterator.

@Override
public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException {
    ResultIterator iterator = delegate.iterator(scanGrouper, scan);
    if (where != null) {
        iterator = new FilterResultIterator(iterator, where);
    }
    if (!orderBy.getOrderByExpressions().isEmpty()) {
        // TopN
        int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
        iterator = new OrderedResultIterator(iterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset, projector.getEstimatedRowByteSize());
    } else {
        if (offset != null) {
            iterator = new OffsetResultIterator(iterator, offset);
        }
        if (limit != null) {
            iterator = new LimitingResultIterator(iterator, limit);
        }
    }
    if (context.getSequenceManager().getSequenceCount() > 0) {
        iterator = new SequenceResultIterator(iterator, context.getSequenceManager());
    }
    return iterator;
}

Example 3 with SequenceResultIterator

use of org.apache.phoenix.iterate.SequenceResultIterator in project phoenix by apache.

the class LiteralResultIterationPlan method newIterator.

@Override
protected ResultIterator newIterator(ParallelScanGrouper scanGrouper, Scan scan, final Map<ImmutableBytesPtr, ServerCache> caches) throws SQLException {
    ResultIterator scanner = new ResultIterator() {

        private final Iterator<Tuple> tupleIterator = tuples.iterator();

        private boolean closed = false;

        private int count = 0;

        private int offsetCount = 0;

        @Override
        public void close() throws SQLException {
            SQLCloseables.closeAll(caches.values());
            this.closed = true;
        }

        @Override
        public Tuple next() throws SQLException {
            while (!this.closed && (offset != null && offsetCount < offset) && tupleIterator.hasNext()) {
                offsetCount++;
                tupleIterator.next();
            }
            if (!this.closed && (limit == null || count++ < limit) && tupleIterator.hasNext()) {
                return tupleIterator.next();
            }
            return null;
        }

        @Override
        public void explain(List<String> planSteps) {
        }
    };
    if (context.getSequenceManager().getSequenceCount() > 0) {
        scanner = new SequenceResultIterator(scanner, context.getSequenceManager());
    }
    return scanner;
}

Example 4 with SequenceResultIterator

use of org.apache.phoenix.iterate.SequenceResultIterator in project phoenix by apache.

the class AggregatePlan method newIterator.

@Override
protected ResultIterator newIterator(ParallelScanGrouper scanGrouper, Scan scan, Map<ImmutableBytesPtr, ServerCache> caches) throws SQLException {
    if (groupBy.isEmpty()) {
        UngroupedAggregateRegionObserver.serializeIntoScan(scan);
    } else {
        // Set attribute with serialized expressions for coprocessor
        GroupedAggregateRegionObserver.serializeIntoScan(scan, groupBy.getScanAttribName(), groupBy.getKeyExpressions());
        if (limit != null && orderBy.getOrderByExpressions().isEmpty() && having == null && ((statement.isDistinct() && !statement.isAggregate()) || (!statement.isDistinct() && (context.getAggregationManager().isEmpty() || BaseScannerRegionObserver.KEY_ORDERED_GROUP_BY_EXPRESSIONS.equals(groupBy.getScanAttribName()))))) {
            /*
                 * Optimization to early exit from the scan for a GROUP BY or DISTINCT with a LIMIT.
                 * We may exit early according to the LIMIT specified if the query has:
                 * 1) No ORDER BY clause (or the ORDER BY was optimized out). We cannot exit
                 *    early if there's an ORDER BY because the first group may be found last
                 *    in the scan.
                 * 2) No HAVING clause, since we execute the HAVING on the client side. The LIMIT
                 *    needs to be evaluated *after* the HAVING.
                 * 3) DISTINCT clause with no GROUP BY. We cannot exit early if there's a
                 *    GROUP BY, as the GROUP BY is processed on the client-side post aggregation
                 *    if a DISTNCT has a GROUP BY. Otherwise, since there are no aggregate
                 *    functions in a DISTINCT, we can exit early regardless of if the
                 *    groups are in row key order or unordered.
                 * 4) GROUP BY clause with no aggregate functions. This is in the same category
                 *    as (3). If we're using aggregate functions, we need to look at all the
                 *    rows, as otherwise we'd exit early with incorrect aggregate function
                 *    calculations.
                 * 5) GROUP BY clause along the pk axis, as the rows are processed in row key
                 *    order, so we can early exit, even when aggregate functions are used, as
                 *    the rows in the group are contiguous.
                 */
            scan.setAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT, PInteger.INSTANCE.toBytes(limit + (offset == null ? 0 : offset)));
        }
    }
    BaseResultIterators iterators = isSerial ? new SerialIterators(this, null, null, wrapParallelIteratorFactory(), scanGrouper, scan, caches, dataPlan) : new ParallelIterators(this, null, wrapParallelIteratorFactory(), scan, false, caches, dataPlan);
    estimatedRows = iterators.getEstimatedRowCount();
    estimatedSize = iterators.getEstimatedByteCount();
    estimateInfoTimestamp = iterators.getEstimateInfoTimestamp();
    splits = iterators.getSplits();
    scans = iterators.getScans();
    AggregatingResultIterator aggResultIterator;
    // No need to merge sort for ungrouped aggregation
    if (groupBy.isEmpty() || groupBy.isUngroupedAggregate()) {
        aggResultIterator = new UngroupedAggregatingResultIterator(new ConcatResultIterator(iterators), aggregators);
    // If salted or local index we still need a merge sort as we'll potentially have multiple group by keys that aren't contiguous.
    } else if (groupBy.isOrderPreserving() && !(this.getTableRef().getTable().getBucketNum() != null || this.getTableRef().getTable().getIndexType() == IndexType.LOCAL)) {
        aggResultIterator = new RowKeyOrderedAggregateResultIterator(iterators, aggregators);
    } else {
        aggResultIterator = new GroupedAggregatingResultIterator(new MergeSortRowKeyResultIterator(iterators, 0, this.getOrderBy() == OrderBy.REV_ROW_KEY_ORDER_BY), aggregators);
    }
    if (having != null) {
        aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having);
    }
    if (statement.isDistinct() && statement.isAggregate()) {
        // Dedup on client if select distinct and aggregation
        aggResultIterator = new DistinctAggregatingResultIterator(aggResultIterator, getProjector());
    }
    ResultIterator resultScanner = aggResultIterator;
    if (orderBy.getOrderByExpressions().isEmpty()) {
        if (offset != null) {
            resultScanner = new OffsetResultIterator(aggResultIterator, offset);
        }
        if (limit != null) {
            resultScanner = new LimitingResultIterator(resultScanner, limit);
        }
    } else {
        int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
        resultScanner = new OrderedAggregatingResultIterator(aggResultIterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset);
    }
    if (context.getSequenceManager().getSequenceCount() > 0) {
        resultScanner = new SequenceResultIterator(resultScanner, context.getSequenceManager());
    }
    return resultScanner;
}

Example 5 with SequenceResultIterator

use of org.apache.phoenix.iterate.SequenceResultIterator in project phoenix by apache.

the class ClientAggregatePlan method iterator.

@Override
public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException {
    ResultIterator iterator = delegate.iterator(scanGrouper, scan);
    if (where != null) {
        iterator = new FilterResultIterator(iterator, where);
    }
    AggregatingResultIterator aggResultIterator;
    if (groupBy.isEmpty()) {
        aggResultIterator = new ClientUngroupedAggregatingResultIterator(LookAheadResultIterator.wrap(iterator), serverAggregators);
        aggResultIterator = new UngroupedAggregatingResultIterator(LookAheadResultIterator.wrap(aggResultIterator), clientAggregators);
    } else {
        if (!groupBy.isOrderPreserving()) {
            int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
            List<Expression> keyExpressions = groupBy.getKeyExpressions();
            List<OrderByExpression> keyExpressionOrderBy = Lists.newArrayListWithExpectedSize(keyExpressions.size());
            for (Expression keyExpression : keyExpressions) {
                keyExpressionOrderBy.add(new OrderByExpression(keyExpression, false, true));
            }
            iterator = new OrderedResultIterator(iterator, keyExpressionOrderBy, thresholdBytes, null, null, projector.getEstimatedRowByteSize());
        }
        aggResultIterator = new ClientGroupedAggregatingResultIterator(LookAheadResultIterator.wrap(iterator), serverAggregators, groupBy.getKeyExpressions());
        aggResultIterator = new GroupedAggregatingResultIterator(LookAheadResultIterator.wrap(aggResultIterator), clientAggregators);
    }
    if (having != null) {
        aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having);
    }
    if (statement.isDistinct() && statement.isAggregate()) {
        // Dedup on client if select distinct and aggregation
        aggResultIterator = new DistinctAggregatingResultIterator(aggResultIterator, getProjector());
    }
    ResultIterator resultScanner = aggResultIterator;
    if (orderBy.getOrderByExpressions().isEmpty()) {
        if (offset != null) {
            resultScanner = new OffsetResultIterator(resultScanner, offset);
        }
        if (limit != null) {
            resultScanner = new LimitingResultIterator(resultScanner, limit);
        }
    } else {
        int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
        resultScanner = new OrderedAggregatingResultIterator(aggResultIterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset);
    }
    if (context.getSequenceManager().getSequenceCount() > 0) {
        resultScanner = new SequenceResultIterator(resultScanner, context.getSequenceManager());
    }
    return resultScanner;
}