Search in sources :

Example 1 with Transaction

use of com.palantir.atlasdb.transaction.api.Transaction in project atlasdb by palantir.

the class Scrubber method runBackgroundScrubTask.

@VisibleForTesting
void runBackgroundScrubTask(final TransactionManager txManager) {
    log.debug("Starting scrub task");
    // Warning: Let T be the hard delete transaction that triggered a scrub, and let S be its
    // start timestamp.  If the locks for T happen to time out right after T checks that its
    // locks are held but right before T writes its commit timestamp (extremely rare case), AND
    // the unreadable timestamp is greater than S, then the scrub task could actually roll back
    // the hard delete transaction (forcing it to abort or retry).  Note that this doesn't affect
    // correctness, but could be an annoying edge cause that causes hard delete to take longer
    // than it otherwise would have.
    Long immutableTimestamp = immutableTimestampSupplier.get();
    Long unreadableTimestamp = unreadableTimestampSupplier.get();
    final long maxScrubTimestamp = aggressiveScrub ? immutableTimestamp : Math.min(unreadableTimestamp, immutableTimestamp);
    log.debug("Scrub task immutableTimestamp: {}, unreadableTimestamp: {}, maxScrubTimestamp: {}", immutableTimestamp, unreadableTimestamp, maxScrubTimestamp);
    final int batchSize = (int) Math.ceil(batchSizeSupplier.get() * ((double) threadCount / readThreadCount));
    List<byte[]> rangeBoundaries = Lists.newArrayList();
    rangeBoundaries.add(PtBytes.EMPTY_BYTE_ARRAY);
    if (readThreadCount > 1) {
        // This will actually partition into the closest higher power of 2 number of ranges.
        rangeBoundaries.addAll(Ordering.from(UnsignedBytes.lexicographicalComparator()).sortedCopy(new UniformRowNamePartitioner(ValueType.BLOB).getPartitions(readThreadCount - 1)));
    }
    rangeBoundaries.add(PtBytes.EMPTY_BYTE_ARRAY);
    List<Future<Void>> readerFutures = Lists.newArrayList();
    final AtomicInteger totalCellsRead = new AtomicInteger(0);
    for (int i = 0; i < rangeBoundaries.size() - 1; i++) {
        final byte[] startRow = rangeBoundaries.get(i);
        final byte[] endRow = rangeBoundaries.get(i + 1);
        readerFutures.add(readerExec.submit(() -> {
            BatchingVisitable<SortedMap<Long, Multimap<TableReference, Cell>>> scrubQueue = scrubberStore.getBatchingVisitableScrubQueue(maxScrubTimestamp, startRow, endRow);
            scrubQueue.batchAccept(batchSize, batch -> {
                for (SortedMap<Long, Multimap<TableReference, Cell>> cells : batch) {
                    // We may actually get more cells than the batch size. The batch size is used
                    // for pulling off the scrub queue, and a single entry in the scrub queue may
                    // match multiple tables. These will get broken down into smaller batches later
                    // on when we actually do deletes.
                    int numCellsRead = scrubSomeCells(cells, txManager, maxScrubTimestamp);
                    int totalRead = totalCellsRead.addAndGet(numCellsRead);
                    log.debug("Scrub task processed {} cells in a batch, total {} processed so far.", numCellsRead, totalRead);
                    if (!isScrubEnabled.get()) {
                        log.debug("Stopping scrub for banned hours.");
                        break;
                    }
                }
                return isScrubEnabled.get();
            });
            return null;
        }));
    }
    for (Future<Void> readerFuture : readerFutures) {
        Futures.getUnchecked(readerFuture);
    }
    log.debug("Scrub background task running at timestamp {} processed a total of {} cells", maxScrubTimestamp, totalCellsRead.get());
    log.debug("Finished scrub task");
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) TransactionService(com.palantir.atlasdb.transaction.service.TransactionService) Throwables(com.palantir.common.base.Throwables) LoggerFactory(org.slf4j.LoggerFactory) BatchingVisitable(com.palantir.common.base.BatchingVisitable) Future(java.util.concurrent.Future) HashMultimap(com.google.common.collect.HashMultimap) PTExecutors(com.palantir.common.concurrent.PTExecutors) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) TableReference(com.palantir.atlasdb.keyvalue.api.TableReference) AtlasDbConstants(com.palantir.atlasdb.AtlasDbConstants) ImmutableSet(com.google.common.collect.ImmutableSet) NamedThreadFactory(com.palantir.common.concurrent.NamedThreadFactory) Cell(com.palantir.atlasdb.keyvalue.api.Cell) Collection(java.util.Collection) Set(java.util.Set) TransactionManager(com.palantir.atlasdb.transaction.api.TransactionManager) Maps2(com.palantir.common.collect.Maps2) GuardedBy(javax.annotation.concurrent.GuardedBy) Sets(com.google.common.collect.Sets) UniformRowNamePartitioner(com.palantir.atlasdb.table.description.UniformRowNamePartitioner) List(java.util.List) ValueType(com.palantir.atlasdb.table.description.ValueType) Transaction(com.palantir.atlasdb.transaction.api.Transaction) Entry(java.util.Map.Entry) Builder(com.google.common.collect.ImmutableMultimap.Builder) KeyAlreadyExistsException(com.palantir.atlasdb.keyvalue.api.KeyAlreadyExistsException) SortedMap(java.util.SortedMap) Iterables(com.google.common.collect.Iterables) Supplier(com.google.common.base.Supplier) TransactionFailedRetriableException(com.palantir.atlasdb.transaction.api.TransactionFailedRetriableException) Callable(java.util.concurrent.Callable) Multimap(com.google.common.collect.Multimap) PtBytes(com.palantir.atlasdb.encoding.PtBytes) Multimaps(com.google.common.collect.Multimaps) Lists(com.google.common.collect.Lists) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ExecutorService(java.util.concurrent.ExecutorService) UnsignedBytes(com.google.common.primitives.UnsignedBytes) Logger(org.slf4j.Logger) Value(com.palantir.atlasdb.keyvalue.api.Value) ExecutorInheritableThreadLocal(com.palantir.common.concurrent.ExecutorInheritableThreadLocal) Maps(com.google.common.collect.Maps) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Futures(com.google.common.util.concurrent.Futures) TransactionConstants(com.palantir.atlasdb.transaction.impl.TransactionConstants) Ordering(com.google.common.collect.Ordering) KeyValueService(com.palantir.atlasdb.keyvalue.api.KeyValueService) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) TransactionType(com.palantir.atlasdb.transaction.api.Transaction.TransactionType) UniformRowNamePartitioner(com.palantir.atlasdb.table.description.UniformRowNamePartitioner) BatchingVisitable(com.palantir.common.base.BatchingVisitable) ArrayListMultimap(com.google.common.collect.ArrayListMultimap) HashMultimap(com.google.common.collect.HashMultimap) Multimap(com.google.common.collect.Multimap) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) TableReference(com.palantir.atlasdb.keyvalue.api.TableReference) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SortedMap(java.util.SortedMap) Future(java.util.concurrent.Future) Cell(com.palantir.atlasdb.keyvalue.api.Cell) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 2 with Transaction

use of com.palantir.atlasdb.transaction.api.Transaction in project atlasdb by palantir.

the class SnapshotTransactionTest method readRow.

private RowResult<byte[]> readRow(byte[] defaultRow) {
    Transaction readTransaction = txManager.createNewTransaction();
    SortedMap<byte[], RowResult<byte[]>> allRows = readTransaction.getRows(TABLE, ImmutableSet.of(defaultRow), ColumnSelection.all());
    return allRows.get(defaultRow);
}
Also used : RowResult(com.palantir.atlasdb.keyvalue.api.RowResult) Transaction(com.palantir.atlasdb.transaction.api.Transaction)

Example 3 with Transaction

use of com.palantir.atlasdb.transaction.api.Transaction in project atlasdb by palantir.

the class SnapshotTransactionTest method testWriteChangedConflictsNoThrow.

@Test
public void testWriteChangedConflictsNoThrow() {
    overrideConflictHandlerForTable(TABLE, ConflictHandler.RETRY_ON_VALUE_CHANGED);
    final Cell cell = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1"));
    Transaction t1 = txManager.createNewTransaction();
    Transaction t2 = txManager.createNewTransaction();
    t1.delete(TABLE, ImmutableSet.of(cell));
    t2.delete(TABLE, ImmutableSet.of(cell));
    t1.commit();
    t2.commit();
}
Also used : Transaction(com.palantir.atlasdb.transaction.api.Transaction) Cell(com.palantir.atlasdb.keyvalue.api.Cell) Test(org.junit.Test)

Example 4 with Transaction

use of com.palantir.atlasdb.transaction.api.Transaction in project atlasdb by palantir.

the class SnapshotTransactionTest method testWriteChangedConflictsThrow.

@Test
public void testWriteChangedConflictsThrow() {
    overrideConflictHandlerForTable(TABLE, ConflictHandler.RETRY_ON_VALUE_CHANGED);
    final Cell cell = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1"));
    Transaction t1 = txManager.createNewTransaction();
    Transaction t2 = txManager.createNewTransaction();
    t1.delete(TABLE, ImmutableSet.of(cell));
    t2.put(TABLE, ImmutableMap.of(cell, new byte[1]));
    t1.commit();
    try {
        t2.commit();
        fail();
    } catch (TransactionConflictException e) {
    // good
    }
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t1.delete(TABLE, ImmutableSet.of(cell));
    t2.put(TABLE, ImmutableMap.of(cell, new byte[1]));
    t2.commit();
    try {
        t1.commit();
        fail();
    } catch (TransactionConflictException e) {
    // good
    }
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t2.delete(TABLE, ImmutableSet.of(cell));
    t1.put(TABLE, ImmutableMap.of(cell, new byte[1]));
    t2.commit();
    try {
        t1.commit();
        fail();
    } catch (TransactionConflictException e) {
    // good
    }
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t2.delete(TABLE, ImmutableSet.of(cell));
    t1.put(TABLE, ImmutableMap.of(cell, new byte[1]));
    t1.commit();
    try {
        t2.commit();
        fail();
    } catch (TransactionConflictException e) {
    // good
    }
}
Also used : Transaction(com.palantir.atlasdb.transaction.api.Transaction) TransactionConflictException(com.palantir.atlasdb.transaction.api.TransactionConflictException) Cell(com.palantir.atlasdb.keyvalue.api.Cell) Test(org.junit.Test)

Example 5 with Transaction

use of com.palantir.atlasdb.transaction.api.Transaction in project atlasdb by palantir.

the class SnapshotTransactionTest method testTransactionWriteWriteConflicts.

@Test
public void testTransactionWriteWriteConflicts() throws Exception {
    // This test creates various types of conflicting writes and makes sure that write-write
    // conflicts are thrown when necessary, and not thrown when there actually isn't a conflict.
    Cell row1Column1 = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1"));
    Cell row1Column2 = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column2"));
    Cell row2Column1 = Cell.create(PtBytes.toBytes("row2"), PtBytes.toBytes("column1"));
    // First transaction commits first, second tries to commit same write
    Transaction t1 = txManager.createNewTransaction();
    Transaction t2 = txManager.createNewTransaction();
    t1.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t1.commit();
    try {
        t2.commit();
        assertTrue(false);
    } catch (TransactionConflictException e) {
    // We expect to catch this exception
    }
    // Second transaction commits first, first tries to commit same write
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t1.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.commit();
    try {
        t1.commit();
        assertTrue(false);
    } catch (TransactionConflictException e) {
    // We expect to catch this exception
    }
    // Transactions committing to different rows
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t1.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.put(TABLE1, ImmutableMap.of(row2Column1, BigInteger.valueOf(1).toByteArray()));
    t1.commit();
    t2.commit();
    // Transactions committing to different tables
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t1.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.put(TABLE2, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t1.commit();
    t2.commit();
    // Transactions committing to different columns in the same row
    t1 = txManager.createNewTransaction();
    t2 = txManager.createNewTransaction();
    t1.put(TABLE1, ImmutableMap.of(row1Column1, BigInteger.valueOf(1).toByteArray()));
    t2.put(TABLE1, ImmutableMap.of(row1Column2, BigInteger.valueOf(1).toByteArray()));
    t1.commit();
    t2.commit();
}
Also used : Transaction(com.palantir.atlasdb.transaction.api.Transaction) TransactionConflictException(com.palantir.atlasdb.transaction.api.TransactionConflictException) Cell(com.palantir.atlasdb.keyvalue.api.Cell) Test(org.junit.Test)

Aggregations

Transaction (com.palantir.atlasdb.transaction.api.Transaction)60 Test (org.junit.Test)51 Cell (com.palantir.atlasdb.keyvalue.api.Cell)26 Map (java.util.Map)18 RangeRequest (com.palantir.atlasdb.keyvalue.api.RangeRequest)14 TransactionSerializableConflictException (com.palantir.atlasdb.transaction.api.TransactionSerializableConflictException)14 BatchingVisitable (com.palantir.common.base.BatchingVisitable)14 ImmutableMap (com.google.common.collect.ImmutableMap)13 TransactionConflictException (com.palantir.atlasdb.transaction.api.TransactionConflictException)13 List (java.util.List)13 ImmutableSortedMap (com.google.common.collect.ImmutableSortedMap)12 PtBytes (com.palantir.atlasdb.encoding.PtBytes)10 ColumnSelection (com.palantir.atlasdb.keyvalue.api.ColumnSelection)10 RowResult (com.palantir.atlasdb.keyvalue.api.RowResult)10 ExecutionException (java.util.concurrent.ExecutionException)9 ExecutorService (java.util.concurrent.ExecutorService)9 Collectors (java.util.stream.Collectors)9 ImmutableSet (com.google.common.collect.ImmutableSet)8 Multimap (com.google.common.collect.Multimap)8 Multimaps (com.google.common.collect.Multimaps)8