Examples with BatchingVisitable com.palantir.common.base.BatchingVisitable used on opensource projects

Example 6 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class AbstractSerializableTransactionTest method testColumnRangeReadWriteEmptyRange.

@Test
public void testColumnRangeReadWriteEmptyRange() {
    byte[] row = PtBytes.toBytes("row1");
    Transaction t1 = startTransaction();
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange = t1.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.toBytes("col"), PtBytes.toBytes("col0"), 1));
    assertNull(BatchingVisitables.getFirst(Iterables.getOnlyElement(columnRange.values())));
    // Write to avoid the read only path.
    put(t1, "row1_1", "col0", "v0");
    Transaction t2 = startTransaction();
    put(t2, "row1", "col", "v0");
    t2.commit();
    try {
        t1.commit();
        fail();
    } catch (TransactionSerializableConflictException e) {
    // expected
    }
}

Example 7 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class SnapshotTransaction method getRanges.

@Override
public Iterable<BatchingVisitable<RowResult<byte[]>>> getRanges(final TableReference tableRef, Iterable<RangeRequest> rangeRequests) {
    checkGetPreconditions(tableRef);
    if (perfLogger.isDebugEnabled()) {
        perfLogger.debug("Passed {} ranges to getRanges({}, {})", Iterables.size(rangeRequests), tableRef, rangeRequests);
    }
    if (!Iterables.isEmpty(rangeRequests)) {
        hasReads = true;
    }
    return FluentIterable.from(Iterables.partition(rangeRequests, BATCH_SIZE_GET_FIRST_PAGE)).transformAndConcat(input -> {
        Timer.Context timer = getTimer("processedRangeMillis").time();
        Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> firstPages = keyValueService.getFirstBatchForRanges(tableRef, input, getStartTimestamp());
        validateExternalAndCommitLocksIfNecessary(tableRef, getStartTimestamp());
        SortedMap<Cell, byte[]> postFiltered = postFilterPages(tableRef, firstPages.values());
        List<BatchingVisitable<RowResult<byte[]>>> ret = Lists.newArrayListWithCapacity(input.size());
        for (RangeRequest rangeRequest : input) {
            TokenBackedBasicResultsPage<RowResult<Value>, byte[]> prePostFilter = firstPages.get(rangeRequest);
            byte[] nextStartRowName = getNextStartRowName(rangeRequest, prePostFilter);
            List<Entry<Cell, byte[]>> mergeIterators = getPostFilteredWithLocalWrites(tableRef, postFiltered, rangeRequest, prePostFilter.getResults(), nextStartRowName);
            ret.add(new AbstractBatchingVisitable<RowResult<byte[]>>() {

                @Override
                protected <K extends Exception> void batchAcceptSizeHint(int batchSizeHint, ConsistentVisitor<RowResult<byte[]>, K> visitor) throws K {
                    checkGetPreconditions(tableRef);
                    final Iterator<RowResult<byte[]>> rowResults = Cells.createRowView(mergeIterators);
                    while (rowResults.hasNext()) {
                        if (!visitor.visit(ImmutableList.of(rowResults.next()))) {
                            return;
                        }
                    }
                    if ((nextStartRowName.length == 0) || !prePostFilter.moreResultsAvailable()) {
                        return;
                    }
                    RangeRequest newRange = rangeRequest.getBuilder().startRowInclusive(nextStartRowName).build();
                    getRange(tableRef, newRange).batchAccept(batchSizeHint, visitor);
                }
            });
        }
        long processedRangeMillis = TimeUnit.NANOSECONDS.toMillis(timer.stop());
        log.trace("Processed {} range requests for {} in {}ms", SafeArg.of("numRequests", input.size()), LoggingArgs.tableRef(tableRef), SafeArg.of("millis", processedRangeMillis));
        return ret;
    });
}

Example 8 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class SnapshotTransaction method getRowsColumnRange.

@Override
public Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> getRowsColumnRange(TableReference tableRef, Iterable<byte[]> rows, BatchColumnRangeSelection columnRangeSelection) {
    checkGetPreconditions(tableRef);
    if (Iterables.isEmpty(rows)) {
        return ImmutableMap.of();
    }
    hasReads = true;
    Map<byte[], RowColumnRangeIterator> rawResults = keyValueService.getRowsColumnRange(tableRef, rows, columnRangeSelection, getStartTimestamp());
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> postFilteredResults = Maps.newHashMapWithExpectedSize(rawResults.size());
    for (Entry<byte[], RowColumnRangeIterator> e : rawResults.entrySet()) {
        byte[] row = e.getKey();
        RowColumnRangeIterator rawIterator = e.getValue();
        Iterator<Map.Entry<Cell, byte[]>> postFilteredIterator = getPostFilteredColumns(tableRef, columnRangeSelection, row, rawIterator);
        postFilteredResults.put(row, BatchingVisitableFromIterable.create(postFilteredIterator));
    }
    return postFilteredResults;
}

Example 9 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class SerializableTransaction method verifyColumnRanges.

private void verifyColumnRanges(Transaction readOnlyTransaction) {
    // verify each set of reads to ensure they are the same.
    for (Entry<TableReference, ConcurrentMap<byte[], ConcurrentMap<BatchColumnRangeSelection, byte[]>>> tableAndRange : columnRangeEndsByTable.entrySet()) {
        TableReference table = tableAndRange.getKey();
        Map<byte[], ConcurrentMap<BatchColumnRangeSelection, byte[]>> columnRangeEnds = tableAndRange.getValue();
        Map<Cell, byte[]> writes = writesByTable.get(table);
        Map<BatchColumnRangeSelection, List<byte[]>> rangesToRows = Maps.newHashMap();
        for (Entry<byte[], ConcurrentMap<BatchColumnRangeSelection, byte[]>> rowAndRangeEnds : columnRangeEnds.entrySet()) {
            byte[] row = rowAndRangeEnds.getKey();
            Map<BatchColumnRangeSelection, byte[]> rangeEnds = columnRangeEnds.get(row);
            for (Entry<BatchColumnRangeSelection, byte[]> e : rangeEnds.entrySet()) {
                BatchColumnRangeSelection range = e.getKey();
                byte[] rangeEnd = e.getValue();
                if (rangeEnd.length != 0 && !RangeRequests.isTerminalRow(false, rangeEnd)) {
                    range = BatchColumnRangeSelection.create(range.getStartCol(), RangeRequests.getNextStartRow(false, rangeEnd), range.getBatchHint());
                }
                if (rangesToRows.get(range) != null) {
                    rangesToRows.get(range).add(row);
                } else {
                    rangesToRows.put(range, ImmutableList.of(row));
                }
            }
        }
        for (Entry<BatchColumnRangeSelection, List<byte[]>> e : rangesToRows.entrySet()) {
            BatchColumnRangeSelection range = e.getKey();
            List<byte[]> rows = e.getValue();
            Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> result = readOnlyTransaction.getRowsColumnRange(table, rows, range);
            for (Entry<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> res : result.entrySet()) {
                byte[] row = res.getKey();
                BatchingVisitableView<Entry<Cell, byte[]>> bv = BatchingVisitableView.of(res.getValue());
                NavigableMap<Cell, ByteBuffer> readsInRange = Maps.transformValues(getReadsInColumnRange(table, row, range), input -> ByteBuffer.wrap(input));
                boolean isEqual = bv.transformBatch(input -> filterWritesFromCells(input, writes)).isEqual(readsInRange.entrySet());
                if (!isEqual) {
                    handleTransactionConflict(table);
                }
            }
        }
    }
}

Example 10 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class SnapshotTransactionTest method testTransactionAtomicity.

@Test
public void testTransactionAtomicity() throws Exception {
    // This test runs multiple transactions in parallel, with KeyValueService.put calls throwing
    // a RuntimeException from time to time and hanging other times. which effectively kills the
    // thread. We ensure that every transaction either adds 5 rows to the table or adds 0 rows
    // by checking at the end that the number of rows is a multiple of 5.
    final TableReference tableRef = TABLE;
    Random random = new Random(1);
    final UnstableKeyValueService unstableKvs = new UnstableKeyValueService(keyValueService, random);
    final TestTransactionManager unstableTransactionManager = new TestTransactionManagerImpl(unstableKvs, timestampService, lockClient, lockService, transactionService, conflictDetectionManager, sweepStrategyManager, sweepQueue);
    ScheduledExecutorService service = PTExecutors.newScheduledThreadPool(20);
    for (int i = 0; i < 30; i++) {
        final int threadNumber = i;
        service.schedule((Callable<Void>) () -> {
            if (threadNumber == 10) {
                unstableKvs.setRandomlyThrow(true);
            }
            if (threadNumber == 20) {
                unstableKvs.setRandomlyHang(true);
            }
            Transaction transaction = unstableTransactionManager.createNewTransaction();
            BatchingVisitable<RowResult<byte[]>> results = transaction.getRange(tableRef, RangeRequest.builder().build());
            final MutableInt nextIndex = new MutableInt(0);
            results.batchAccept(1, AbortingVisitors.batching((AbortingVisitor<RowResult<byte[]>, Exception>) row -> {
                byte[] dataBytes = row.getColumns().get(PtBytes.toBytes("data"));
                BigInteger dataValue = new BigInteger(dataBytes);
                nextIndex.setValue(Math.max(nextIndex.toInteger(), dataValue.intValue() + 1));
                return true;
            }));
            // rows to the table.
            for (int j = 0; j < 5; j++) {
                int rowNumber = nextIndex.toInteger() + j;
                Cell cell = Cell.create(PtBytes.toBytes("row" + rowNumber), PtBytes.toBytes("data"));
                transaction.put(tableRef, ImmutableMap.of(cell, BigInteger.valueOf(rowNumber).toByteArray()));
                Thread.yield();
            }
            transaction.commit();
            return null;
        }, i * 20, TimeUnit.MILLISECONDS);
    }
    service.shutdown();
    service.awaitTermination(1, TimeUnit.SECONDS);
    // Verify each table has a number of rows that's a multiple of 5
    Transaction verifyTransaction = txManager.createNewTransaction();
    BatchingVisitable<RowResult<byte[]>> results = verifyTransaction.getRange(tableRef, RangeRequest.builder().build());
    final MutableInt numRows = new MutableInt(0);
    results.batchAccept(1, AbortingVisitors.batching((AbortingVisitor<RowResult<byte[]>, Exception>) row -> {
        numRows.increment();
        return true;
    }));
    Assert.assertEquals(0, numRows.toInteger() % 5);
}