Examples with AbstractPagingIterable com.palantir.util.paging.AbstractPagingIterable used on opensource projects

Example 1 with AbstractPagingIterable

use of com.palantir.util.paging.AbstractPagingIterable in project atlasdb by palantir.

the class DbKvs method getRangeOfTimestamps.

/**
 * @param tableRef the name of the table to read from.
 * @param rangeRequest the range to load.
 * @param timestamp the maximum timestamp to load.
 *
 * @return Each row that has fewer than maxRangeOfTimestampsBatchSize entries is guaranteed to be returned in a
 * single RowResult. If a row has more than maxRangeOfTimestampsBatchSize results, it will potentially be split
 * into multiple RowResults, by finishing the current column; see example below. Note that:
 *  1) this may cause a RowResult to have more than maxRangeOfTimestampsBatchSize entries
 *  2) this may still finish a row, in which case there is going to be only one RowResult for that row.
 * It is, furthermore,  guaranteed that the columns will be read in ascending order
 *
 * E.g., for the following table, rangeRequest taking all rows in ascending order,
 * maxRangeOfTimestampsBatchSize == 5, and timestamp 10:
 *
 *           a     |     b     |     c     |     d
 *     ------------------------------------------------
 *   a | (1, 2, 3) | (1, 2, 3) | (4, 5, 6) | (4, 5, 6)|
 *     ------------------------------------------------
 *   b | (1, 3, 5) |     -     | (1)       |     -    |
 *     ------------------------------------------------
 *   c | (1, 2)    | (1, 2)    | (4, 5, 6) | (4, 5, 6)|
 *     ------------------------------------------------
 *   d | (1, 3, 5) |     -     | (1, 2, 3) |     -    |
 *     ------------------------------------------------
 *   e | (1, 3)    |     -     |     -     |     -    |
 *     ------------------------------------------------
 *
 * The RowResults will be:
 *   1. (a, (a -> 1, 2, 3; b -> 1, 2, 3))
 *   2. (a, (c -> 4, 5, 6; d -> 4, 5, 6))
 *
 *   3. (b, (a -> 1, 3, 5; b -> 1))
 *
 *   4. (c, (a -> 1, 2, b -> 1, 2; c -> 4, 5, 6))
 *   5. (c, (d -> 4, 5, 6))
 *
 *   6. (d, (a -> 1, 3, 5, c -> 1, 2, 3))
 *
 *   7. (e, (a -> 1, 3))
 */
@Override
public ClosableIterator<RowResult<Set<Long>>> getRangeOfTimestamps(TableReference tableRef, RangeRequest rangeRequest, long timestamp) {
    Iterable<RowResult<Set<Long>>> rows = new AbstractPagingIterable<RowResult<Set<Long>>, TokenBackedBasicResultsPage<RowResult<Set<Long>>, Token>>() {

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Set<Long>>, Token> getFirstPage() {
            return getTimestampsPage(tableRef, rangeRequest, timestamp, maxRangeOfTimestampsBatchSize, Token.INITIAL);
        }

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Set<Long>>, Token> getNextPage(TokenBackedBasicResultsPage<RowResult<Set<Long>>, Token> previous) {
            Token token = previous.getTokenForNextPage();
            RangeRequest newRange = rangeRequest.getBuilder().startRowInclusive(token.row()).build();
            return getTimestampsPage(tableRef, newRange, timestamp, maxRangeOfTimestampsBatchSize, token);
        }
    };
    return ClosableIterators.wrap(rows.iterator());
}

Example 2 with AbstractPagingIterable

use of com.palantir.util.paging.AbstractPagingIterable in project atlasdb by palantir.

the class DbKvs method getRowColumnRange.

private Iterator<Map.Entry<Cell, Value>> getRowColumnRange(TableReference tableRef, byte[] row, BatchColumnRangeSelection batchColumnRangeSelection, long timestamp) {
    List<byte[]> rowList = ImmutableList.of(row);
    return ClosableIterators.wrap(new AbstractPagingIterable<Entry<Cell, Value>, TokenBackedBasicResultsPage<Entry<Cell, Value>, byte[]>>() {

        @Override
        protected TokenBackedBasicResultsPage<Entry<Cell, Value>, byte[]> getFirstPage() throws Exception {
            return page(batchColumnRangeSelection.getStartCol());
        }

        @Override
        protected TokenBackedBasicResultsPage<Map.Entry<Cell, Value>, byte[]> getNextPage(TokenBackedBasicResultsPage<Map.Entry<Cell, Value>, byte[]> previous) throws Exception {
            return page(previous.getTokenForNextPage());
        }

        TokenBackedBasicResultsPage<Map.Entry<Cell, Value>, byte[]> page(byte[] startCol) throws Exception {
            BatchColumnRangeSelection range = BatchColumnRangeSelection.create(startCol, batchColumnRangeSelection.getEndCol(), batchColumnRangeSelection.getBatchHint());
            List<Map.Entry<Cell, Value>> nextPage = Iterables.getOnlyElement(extractRowColumnRangePage(tableRef, range, timestamp, rowList).values());
            if (nextPage.isEmpty()) {
                return SimpleTokenBackedResultsPage.create(startCol, ImmutableList.<Entry<Cell, Value>>of(), false);
            }
            byte[] lastCol = nextPage.get(nextPage.size() - 1).getKey().getColumnName();
            if (isEndOfColumnRange(lastCol, batchColumnRangeSelection.getEndCol())) {
                return SimpleTokenBackedResultsPage.create(lastCol, nextPage, false);
            }
            byte[] nextCol = RangeRequests.nextLexicographicName(lastCol);
            return SimpleTokenBackedResultsPage.create(nextCol, nextPage, true);
        }
    }.iterator());
}

Example 3 with AbstractPagingIterable

use of com.palantir.util.paging.AbstractPagingIterable in project atlasdb by palantir.

the class CqlKeyValueService method getRangeWithPageCreator.

public <T> ClosableIterator<RowResult<T>> getRangeWithPageCreator(TableReference tableRef, RangeRequest rangeRequest, long timestamp, com.datastax.driver.core.ConsistencyLevel consistency, Supplier<ResultsExtractor<T>> resultsExtractor) {
    if (rangeRequest.isReverse()) {
        throw new UnsupportedOperationException();
    }
    if (rangeRequest.isEmptyRange()) {
        return ClosableIterators.wrap(ImmutableList.<RowResult<T>>of().iterator());
    }
    final int batchHint = rangeRequest.getBatchHint() == null ? 100 : rangeRequest.getBatchHint();
    final ColumnSelection selection = rangeRequest.getColumnNames().isEmpty() ? ColumnSelection.all() : ColumnSelection.create(rangeRequest.getColumnNames());
    final byte[] endExclusive = rangeRequest.getEndExclusive();
    final StringBuilder bindQuery = new StringBuilder();
    bindQuery.append("SELECT * FROM " + getFullTableName(tableRef) + " WHERE token(" + fieldNameProvider.row() + ") >= token(?) ");
    if (endExclusive.length > 0) {
        bindQuery.append("AND token(" + fieldNameProvider.row() + ") < token(?) ");
    }
    bindQuery.append("LIMIT " + batchHint);
    final String getLastRowQuery = "SELECT * FROM " + getFullTableName(tableRef) + " WHERE " + fieldNameProvider.row() + " = ?";
    return ClosableIterators.wrap(new AbstractPagingIterable<RowResult<T>, TokenBackedBasicResultsPage<RowResult<T>, byte[]>>() {

        @Override
        protected TokenBackedBasicResultsPage<RowResult<T>, byte[]> getFirstPage() throws Exception {
            return getPage(rangeRequest.getStartInclusive());
        }

        @Override
        protected TokenBackedBasicResultsPage<RowResult<T>, byte[]> getNextPage(TokenBackedBasicResultsPage<RowResult<T>, byte[]> previous) throws Exception {
            return getPage(previous.getTokenForNextPage());
        }

        TokenBackedBasicResultsPage<RowResult<T>, byte[]> getPage(final byte[] startKey) throws Exception {
            BoundStatement boundStatement = getPreparedStatement(tableRef, bindQuery.toString(), session).setConsistencyLevel(consistency).bind();
            boundStatement.setBytes(0, ByteBuffer.wrap(startKey));
            if (endExclusive.length > 0) {
                boundStatement.setBytes(1, ByteBuffer.wrap(endExclusive));
            }
            ResultSet resultSet = session.execute(boundStatement);
            List<Row> rows = Lists.newArrayList(resultSet.all());
            cqlKeyValueServices.logTracedQuery(bindQuery.toString(), resultSet, session, cqlStatementCache.normalQuery);
            byte[] maxRow = null;
            ResultsExtractor<T> extractor = resultsExtractor.get();
            for (Row row : rows) {
                byte[] rowName = getRowName(row);
                if (maxRow == null) {
                    maxRow = rowName;
                } else {
                    maxRow = PtBytes.BYTES_COMPARATOR.max(maxRow, rowName);
                }
            }
            if (maxRow == null) {
                return new SimpleTokenBackedResultsPage<>(endExclusive, ImmutableList.of(), false);
            }
            // get the rest of the last row
            BoundStatement boundLastRow = getPreparedStatement(tableRef, getLastRowQuery, session).bind();
            boundLastRow.setBytes(fieldNameProvider.row(), ByteBuffer.wrap(maxRow));
            try {
                resultSet = session.execute(boundLastRow);
            } catch (com.datastax.driver.core.exceptions.UnavailableException e) {
                throw new InsufficientConsistencyException("This operation requires all Cassandra" + " nodes to be up and available.", e);
            }
            rows.addAll(resultSet.all());
            cqlKeyValueServices.logTracedQuery(getLastRowQuery, resultSet, session, cqlStatementCache.normalQuery);
            for (Row row : rows) {
                extractor.internalExtractResult(timestamp, selection, getRowName(row), getColName(row), getValue(row), getTs(row));
            }
            SortedMap<byte[], SortedMap<byte[], T>> resultsByRow = Cells.breakCellsUpByRow(extractor.asMap());
            return ResultsExtractor.getRowResults(endExclusive, maxRow, resultsByRow);
        }
    }.iterator());
}

Example 4 with AbstractPagingIterable

use of com.palantir.util.paging.AbstractPagingIterable in project atlasdb by palantir.

the class JdbcKeyValueService method getRange.

@Override
public ClosableIterator<RowResult<Value>> getRange(final TableReference tableRef, final RangeRequest rangeRequest, final long timestamp) {
    Iterable<RowResult<Value>> iter = new AbstractPagingIterable<RowResult<Value>, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>>() {

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Value>, byte[]> getFirstPage() {
            return getPageWithValues(tableRef, rangeRequest, timestamp);
        }

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Value>, byte[]> getNextPage(TokenBackedBasicResultsPage<RowResult<Value>, byte[]> previous) {
            byte[] startRow = previous.getTokenForNextPage();
            RangeRequest newRange = rangeRequest.getBuilder().startRowInclusive(startRow).build();
            return getPageWithValues(tableRef, newRange, timestamp);
        }
    };
    return ClosableIterators.wrap(iter.iterator());
}

Example 5 with AbstractPagingIterable

use of com.palantir.util.paging.AbstractPagingIterable in project atlasdb by palantir.

the class JdbcKeyValueService method getRangeOfTimestamps.

@Override
public ClosableIterator<RowResult<Set<Long>>> getRangeOfTimestamps(final TableReference tableRef, final RangeRequest rangeRequest, final long timestamp) {
    Iterable<RowResult<Set<Long>>> iter = new AbstractPagingIterable<RowResult<Set<Long>>, TokenBackedBasicResultsPage<RowResult<Set<Long>>, byte[]>>() {

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Set<Long>>, byte[]> getFirstPage() {
            return getPageWithTimestamps(tableRef, rangeRequest, timestamp);
        }

        @Override
        protected TokenBackedBasicResultsPage<RowResult<Set<Long>>, byte[]> getNextPage(TokenBackedBasicResultsPage<RowResult<Set<Long>>, byte[]> previous) {
            byte[] startRow = previous.getTokenForNextPage();
            RangeRequest newRange = rangeRequest.getBuilder().startRowInclusive(startRow).build();
            return getPageWithTimestamps(tableRef, newRange, timestamp);
        }
    };
    return ClosableIterators.wrap(iter.iterator());
}