Examples with BatchingVisitable com.palantir.common.base.BatchingVisitable used on opensource projects

Example 1 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class Scrubber method runBackgroundScrubTask.

@VisibleForTesting
void runBackgroundScrubTask(final TransactionManager txManager) {
    log.debug("Starting scrub task");
    // Warning: Let T be the hard delete transaction that triggered a scrub, and let S be its
    // start timestamp.  If the locks for T happen to time out right after T checks that its
    // locks are held but right before T writes its commit timestamp (extremely rare case), AND
    // the unreadable timestamp is greater than S, then the scrub task could actually roll back
    // the hard delete transaction (forcing it to abort or retry).  Note that this doesn't affect
    // correctness, but could be an annoying edge cause that causes hard delete to take longer
    // than it otherwise would have.
    Long immutableTimestamp = immutableTimestampSupplier.get();
    Long unreadableTimestamp = unreadableTimestampSupplier.get();
    final long maxScrubTimestamp = aggressiveScrub ? immutableTimestamp : Math.min(unreadableTimestamp, immutableTimestamp);
    log.debug("Scrub task immutableTimestamp: {}, unreadableTimestamp: {}, maxScrubTimestamp: {}", immutableTimestamp, unreadableTimestamp, maxScrubTimestamp);
    final int batchSize = (int) Math.ceil(batchSizeSupplier.get() * ((double) threadCount / readThreadCount));
    List<byte[]> rangeBoundaries = Lists.newArrayList();
    rangeBoundaries.add(PtBytes.EMPTY_BYTE_ARRAY);
    if (readThreadCount > 1) {
        // This will actually partition into the closest higher power of 2 number of ranges.
        rangeBoundaries.addAll(Ordering.from(UnsignedBytes.lexicographicalComparator()).sortedCopy(new UniformRowNamePartitioner(ValueType.BLOB).getPartitions(readThreadCount - 1)));
    }
    rangeBoundaries.add(PtBytes.EMPTY_BYTE_ARRAY);
    List<Future<Void>> readerFutures = Lists.newArrayList();
    final AtomicInteger totalCellsRead = new AtomicInteger(0);
    for (int i = 0; i < rangeBoundaries.size() - 1; i++) {
        final byte[] startRow = rangeBoundaries.get(i);
        final byte[] endRow = rangeBoundaries.get(i + 1);
        readerFutures.add(readerExec.submit(() -> {
            BatchingVisitable<SortedMap<Long, Multimap<TableReference, Cell>>> scrubQueue = scrubberStore.getBatchingVisitableScrubQueue(maxScrubTimestamp, startRow, endRow);
            scrubQueue.batchAccept(batchSize, batch -> {
                for (SortedMap<Long, Multimap<TableReference, Cell>> cells : batch) {
                    // We may actually get more cells than the batch size. The batch size is used
                    // for pulling off the scrub queue, and a single entry in the scrub queue may
                    // match multiple tables. These will get broken down into smaller batches later
                    // on when we actually do deletes.
                    int numCellsRead = scrubSomeCells(cells, txManager, maxScrubTimestamp);
                    int totalRead = totalCellsRead.addAndGet(numCellsRead);
                    log.debug("Scrub task processed {} cells in a batch, total {} processed so far.", numCellsRead, totalRead);
                    if (!isScrubEnabled.get()) {
                        log.debug("Stopping scrub for banned hours.");
                        break;
                    }
                }
                return isScrubEnabled.get();
            });
            return null;
        }));
    }
    for (Future<Void> readerFuture : readerFutures) {
        Futures.getUnchecked(readerFuture);
    }
    log.debug("Scrub background task running at timestamp {} processed a total of {} cells", maxScrubTimestamp, totalCellsRead.get());
    log.debug("Finished scrub task");
}

Example 2 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class AbstractTransactionTest method testColumnRangePagingTransaction.

@Test
public void testColumnRangePagingTransaction() {
    Transaction t = startTransaction();
    int totalPuts = 101;
    byte[] row = PtBytes.toBytes("row1");
    // Record expected results using byte ordering
    ImmutableSortedMap.Builder<Cell, byte[]> writes = ImmutableSortedMap.orderedBy(Ordering.from(UnsignedBytes.lexicographicalComparator()).onResultOf(key -> key.getColumnName()));
    for (int i = 0; i < totalPuts; i++) {
        put(t, "row1", "col" + i, "v" + i);
        writes.put(Cell.create(row, PtBytes.toBytes("col" + i)), PtBytes.toBytes("v" + i));
    }
    t.commit();
    t = startTransaction();
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange = t.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY, 1));
    List<Map.Entry<Cell, byte[]>> expected = ImmutableList.copyOf(writes.build().entrySet());
    verifyMatchingResult(expected, row, columnRange);
    columnRange = t.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.toBytes("col"), PtBytes.EMPTY_BYTE_ARRAY, 1));
    verifyMatchingResult(expected, row, columnRange);
    columnRange = t.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.toBytes("col"), PtBytes.EMPTY_BYTE_ARRAY, 101));
    verifyMatchingResult(expected, row, columnRange);
    columnRange = t.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, RangeRequests.nextLexicographicName(expected.get(expected.size() - 1).getKey().getColumnName()), 1));
    verifyMatchingResult(expected, row, columnRange);
    columnRange = t.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, expected.get(expected.size() - 1).getKey().getColumnName(), 1));
    verifyMatchingResult(ImmutableList.copyOf(Iterables.limit(expected, 100)), row, columnRange);
}

Example 3 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class AbstractSerializableTransactionTest method testColumnRangeReadWriteConflictOnNewCell.

@Test
public void testColumnRangeReadWriteConflictOnNewCell() {
    byte[] row = PtBytes.toBytes("row1");
    writeColumns();
    Transaction t1 = startTransaction();
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange = t1.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY, 1));
    // Serializable transaction records only the first column as read.
    Map.Entry<Cell, byte[]> read = BatchingVisitables.getFirst(Iterables.getOnlyElement(columnRange.values()));
    assertEquals(Cell.create(row, PtBytes.toBytes("col0")), read.getKey());
    // Write to avoid the read only path.
    put(t1, "row1_1", "col0", "v0");
    Transaction t2 = startTransaction();
    // Write on the start of the range.
    put(t2, "row1", "col", "v");
    t2.commit();
    try {
        t1.commit();
        fail();
    } catch (TransactionSerializableConflictException e) {
    // expected
    }
}

Example 4 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class AbstractSerializableTransactionTest method testColumnRangeReadWriteConflict.

@Test
public void testColumnRangeReadWriteConflict() {
    byte[] row = PtBytes.toBytes("row1");
    writeColumns();
    Transaction t1 = startTransaction();
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange = t1.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY, 1));
    // Serializable transaction records only the first column as read.
    Map.Entry<Cell, byte[]> read = BatchingVisitables.getFirst(Iterables.getOnlyElement(columnRange.values()));
    assertEquals(Cell.create(row, PtBytes.toBytes("col0")), read.getKey());
    // Write to avoid the read only path.
    put(t1, "row1_1", "col0", "v0");
    Transaction t2 = startTransaction();
    put(t2, "row1", "col0", "v0_0");
    t2.commit();
    try {
        t1.commit();
        fail();
    } catch (TransactionSerializableConflictException e) {
    // expected
    }
}

Example 5 with BatchingVisitable

use of com.palantir.common.base.BatchingVisitable in project atlasdb by palantir.

the class AbstractSerializableTransactionTest method testColumnRangeReadWriteEmptyRangeUnread.

@Test
public void testColumnRangeReadWriteEmptyRangeUnread() {
    byte[] row = PtBytes.toBytes("row1");
    Transaction t1 = startTransaction();
    Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange = t1.getRowsColumnRange(TEST_TABLE, ImmutableList.of(row), BatchColumnRangeSelection.create(PtBytes.toBytes("col"), PtBytes.toBytes("col0"), 1));
    // Intentionally not reading anything from the result, so we shouldn't get a conflict.
    // Write to avoid the read only path.
    put(t1, "row1_1", "col0", "v0");
    Transaction t2 = startTransaction();
    put(t2, "row1", "col", "v0");
    t2.commit();
    t1.commit();
}