Examples with TableReference com.palantir.atlasdb.keyvalue.api.TableReference used on opensource projects

Example 91 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class TableTasksTest method testDiffTask.

@Test
public void testDiffTask() throws InterruptedException {
    TableReference table1 = TableReference.createWithEmptyNamespace("table1");
    TableReference table2 = TableReference.createWithEmptyNamespace("table2");
    Random rand = new Random();
    kvs.createTable(table1, AtlasDbConstants.GENERIC_TABLE_METADATA);
    kvs.createTable(table2, AtlasDbConstants.GENERIC_TABLE_METADATA);
    Multimap<Integer, Integer> keys1 = HashMultimap.create();
    Multimap<Integer, Integer> keys2 = HashMultimap.create();
    int key = 0;
    for (int col = 0; col < 256; col++) {
        int randomInt = rand.nextInt(3);
        if (randomInt >= 1) {
            keys1.put(key, col);
            kvs.put(table1, ImmutableMap.of(Cell.create(new byte[] { (byte) key }, new byte[] { (byte) col }), new byte[] { 0 }), 1);
        }
        if (randomInt <= 1) {
            keys2.put(key, col);
            kvs.put(table2, ImmutableMap.of(Cell.create(new byte[] { (byte) key }, new byte[] { (byte) col }), new byte[] { 0 }), 1);
        }
        if (rand.nextBoolean()) {
            key++;
        }
    }
    TransactionServices.createTransactionService(kvs).putUnlessExists(1, 1);
    AtomicLong rowsOnlyInSource = new AtomicLong();
    AtomicLong rowsPartiallyInCommon = new AtomicLong();
    AtomicLong rowsCompletelyInCommon = new AtomicLong();
    AtomicLong rowsVisited = new AtomicLong();
    AtomicLong cellsOnlyInSource = new AtomicLong();
    AtomicLong cellsInCommon = new AtomicLong();
    DiffStats stats = new TableTasks.DiffStats(rowsOnlyInSource, rowsPartiallyInCommon, rowsCompletelyInCommon, rowsVisited, cellsOnlyInSource, cellsInCommon);
    TableTasks.diff(txManager, MoreExecutors.newDirectExecutorService(), table1, table2, 10, 1, stats, (transaction, partialDiff) -> Iterators.size(partialDiff));
    long sourceOnlyCells = 0;
    long commonCells = 0;
    for (Entry<Integer, Integer> cell : keys1.entries()) {
        if (keys2.containsEntry(cell.getKey(), cell.getValue())) {
            commonCells++;
        } else {
            sourceOnlyCells++;
        }
    }
    long disjointRows = 0;
    long partialRows = 0;
    long commonRows = 0;
    for (int k : keys1.keySet()) {
        if (Collections.disjoint(keys2.get(k), keys1.get(k))) {
            disjointRows++;
        } else if (keys2.get(k).containsAll(keys1.get(k))) {
            commonRows++;
        } else {
            partialRows++;
        }
    }
    Assert.assertEquals(commonCells, cellsInCommon.get());
    Assert.assertEquals(sourceOnlyCells, cellsOnlyInSource.get());
    Assert.assertEquals(disjointRows, rowsOnlyInSource.get());
    Assert.assertEquals(commonRows, rowsCompletelyInCommon.get());
    Assert.assertEquals(partialRows, rowsPartiallyInCommon.get());
    Assert.assertEquals(keys1.keySet().size(), rowsVisited.get());
}

Example 92 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class AbstractKeyValueService method multiPut.

/* (non-Javadoc)
     * @see com.palantir.atlasdb.keyvalue.api.KeyValueService#multiPut(java.util.Map, long)
     */
@Override
public void multiPut(Map<TableReference, ? extends Map<Cell, byte[]>> valuesByTable, final long timestamp) throws KeyAlreadyExistsException {
    List<Callable<Void>> callables = Lists.newArrayList();
    for (Entry<TableReference, ? extends Map<Cell, byte[]>> e : valuesByTable.entrySet()) {
        final TableReference table = e.getKey();
        // We sort here because some key value stores are more efficient if you store adjacent keys together.
        NavigableMap<Cell, byte[]> sortedMap = ImmutableSortedMap.copyOf(e.getValue());
        Iterable<List<Entry<Cell, byte[]>>> partitions = IterablePartitioner.partitionByCountAndBytes(sortedMap.entrySet(), getMultiPutBatchCount(), getMultiPutBatchSizeBytes(), table, entry -> entry == null ? 0 : entry.getValue().length + Cells.getApproxSizeOfCell(entry.getKey()));
        for (final List<Entry<Cell, byte[]>> p : partitions) {
            callables.add(() -> {
                String originalName = Thread.currentThread().getName();
                Thread.currentThread().setName("Atlas multiPut of " + p.size() + " cells into " + table);
                try {
                    put(table, Maps2.fromEntries(p), timestamp);
                    return null;
                } finally {
                    Thread.currentThread().setName(originalName);
                }
            });
        }
    }
    List<Future<Void>> futures;
    try {
        futures = executor.invokeAll(callables);
    } catch (InterruptedException e) {
        throw Throwables.throwUncheckedException(e);
    }
    for (Future<Void> future : futures) {
        try {
            future.get();
        } catch (InterruptedException e) {
            throw Throwables.throwUncheckedException(e);
        } catch (ExecutionException e) {
            throw Throwables.rewrapAndThrowUncheckedException(e.getCause());
        }
    }
}

Example 93 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class DualWriteKeyValueService method dropTables.

@Override
public void dropTables(Set<TableReference> tableRefs) {
    for (TableReference tableRef : tableRefs) {
        delegate1.dropTable(tableRef);
        delegate2.dropTable(tableRef);
    }
}

Example 94 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class SweepCommand method execute.

@Override
public int execute(final AtlasDbServices services) {
    SweepTaskRunner sweepRunner = services.getSweepTaskRunner();
    if (!((namespace != null) ^ (table != null) ^ sweepAllTables)) {
        printer.error("Specify one of --namespace, --table, or --all options.");
        return 1;
    }
    if ((namespace != null) && (row != null)) {
        printer.error("Cannot specify a start row (" + row + ") when sweeping multiple tables (in namespace " + namespace + ")");
        return 1;
    }
    Map<TableReference, byte[]> tableToStartRow = Maps.newHashMap();
    if (table != null) {
        TableReference tableToSweep = TableReference.createUnsafe(table);
        if (!services.getKeyValueService().getAllTableNames().contains(tableToSweep)) {
            printer.info("The table {} passed in to sweep does not exist", LoggingArgs.tableRef(tableToSweep));
            return 1;
        }
        byte[] startRow = PtBytes.EMPTY_BYTE_ARRAY;
        if (row != null) {
            startRow = decodeStartRow(row);
        }
        tableToStartRow.put(tableToSweep, startRow);
    } else if (namespace != null) {
        Set<TableReference> tablesInNamespace = services.getKeyValueService().getAllTableNames().stream().filter(tableRef -> tableRef.getNamespace().getName().equals(namespace)).collect(Collectors.toSet());
        for (TableReference tableInNamespace : tablesInNamespace) {
            tableToStartRow.put(tableInNamespace, new byte[0]);
        }
    } else if (sweepAllTables) {
        tableToStartRow.putAll(Maps.asMap(Sets.difference(services.getKeyValueService().getAllTableNames(), AtlasDbConstants.hiddenTables), Functions.constant(new byte[0])));
    }
    SweepBatchConfig batchConfig = getSweepBatchConfig();
    for (Map.Entry<TableReference, byte[]> entry : tableToStartRow.entrySet()) {
        final TableReference tableToSweep = entry.getKey();
        SweepResults accumulatedResults = SweepResults.createEmptySweepResult(Optional.of(entry.getValue()));
        while (accumulatedResults.getNextStartRow().isPresent()) {
            SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get());
            accumulatedResults = accumulatedResults.accumulateWith(newResults);
            printer.info("{} Swept from {} to {} in table {} in {} ms, examined {} cell values," + " deleted {} stale versions of those cells. Time elapsed since started sweeping:" + " {} ms. Total time sweeping this table: {} ms.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), UnsafeArg.of("startRow", encodeStartRow(accumulatedResults.getNextStartRow())), UnsafeArg.of("exclusiveEndRow", encodeEndRow(newResults.getNextStartRow())), LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken millis", newResults.getTimeInMillis()), SafeArg.of("cellTs pairs examined", newResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", newResults.getStaleValuesDeleted()), SafeArg.of("time elapsed", accumulatedResults.getTimeElapsedSinceStartedSweeping()), SafeArg.of("time sweeping", accumulatedResults.getTimeInMillis()));
            maybeSleep();
        }
        SweepResults finalAccumulatedResults = accumulatedResults;
        if (!dryRun) {
            services.getTransactionManager().runTaskWithRetry((TxTask) t -> {
                SweepPriorityTable priorityTable = SweepTableFactory.of().getSweepPriorityTable(t);
                SweepPriorityTable.SweepPriorityRow row1 = SweepPriorityTable.SweepPriorityRow.of(tableToSweep.getQualifiedName());
                priorityTable.putWriteCount(row1, 0L);
                priorityTable.putCellsExamined(row1, finalAccumulatedResults.getCellTsPairsExamined());
                priorityTable.putCellsDeleted(row1, finalAccumulatedResults.getStaleValuesDeleted());
                priorityTable.putLastSweepTime(row1, System.currentTimeMillis());
                return null;
            });
        }
        printer.info("{} Finished sweeping {}, examined {} cell values, deleted {} stale versions of those cells.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), LoggingArgs.tableRef(tableToSweep), SafeArg.of("cellTs pairs examined", finalAccumulatedResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", finalAccumulatedResults.getStaleValuesDeleted()));
        if (!dryRun && finalAccumulatedResults.getStaleValuesDeleted() > 0) {
            Stopwatch watch = Stopwatch.createStarted();
            services.getKeyValueService().compactInternally(tableToSweep);
            printer.info("Finished performing compactInternally on {} in {} ms.", LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken", watch.elapsed(TimeUnit.MILLISECONDS)));
        }
    }
    return 0;
}

Example 95 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class StatsTrackingKeyValueService method multiPut.

@Override
public void multiPut(Map<TableReference, ? extends Map<Cell, byte[]>> valuesByTable, long timestamp) {
    long start = System.currentTimeMillis();
    super.multiPut(valuesByTable, timestamp);
    long finish = System.currentTimeMillis();
    for (Entry<TableReference, ? extends Map<Cell, byte[]>> entry : valuesByTable.entrySet()) {
        TableReference tableRef = entry.getKey();
        Map<Cell, byte[]> values = entry.getValue();
        TableStats s = getTableStats(tableRef);
        s.totalPutMillis.addAndGet(finish - start);
        s.totalPutCalls.incrementAndGet();
        // Only update stats after put was successful.
        s.totalPutCells.addAndGet(values.size());
        for (Map.Entry<Cell, byte[]> e : values.entrySet()) {
            incrementPutBytes(s, e.getKey(), e.getValue());
        }
    }
}