Examples with TableReference com.palantir.atlasdb.keyvalue.api.TableReference used on opensource projects

Example 96 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class SweepStatsKeyValueService method flushWrites.

private void flushWrites(Multiset<TableReference> writes, Set<TableReference> clears) {
    if (writes.isEmpty() && clears.isEmpty()) {
        log.info("No writes to flush");
        return;
    }
    log.info("Flushing stats for {} writes and {} clears", SafeArg.of("writes", writes.size()), SafeArg.of("clears", clears.size()));
    log.trace("Flushing writes: {}", UnsafeArg.of("writes", writes));
    log.trace("Flushing clears: {}", UnsafeArg.of("clears", clears));
    try {
        Set<TableReference> tableNames = Sets.difference(writes.elementSet(), clears);
        Collection<byte[]> rows = Collections2.transform(Collections2.transform(tableNames, t -> t.getQualifiedName()), Functions.compose(Persistables.persistToBytesFunction(), SweepPriorityRow.fromFullTableNameFun()));
        Map<Cell, Value> oldWriteCounts = delegate().getRows(SWEEP_PRIORITY_TABLE, rows, SweepPriorityTable.getColumnSelection(SweepPriorityNamedColumn.WRITE_COUNT), Long.MAX_VALUE);
        Map<Cell, byte[]> newWriteCounts = Maps.newHashMapWithExpectedSize(writes.elementSet().size());
        byte[] col = SweepPriorityNamedColumn.WRITE_COUNT.getShortName();
        for (TableReference tableRef : tableNames) {
            Preconditions.checkState(!tableRef.getQualifiedName().startsWith(AtlasDbConstants.NAMESPACE_PREFIX), "The sweep stats kvs should wrap the namespace mapping kvs, not the other way around.");
            byte[] row = SweepPriorityRow.of(tableRef.getQualifiedName()).persistToBytes();
            Cell cell = Cell.create(row, col);
            Value oldValue = oldWriteCounts.get(cell);
            long oldCount = oldValue == null || oldValue.getContents().length == 0 ? 0 : SweepPriorityTable.WriteCount.BYTES_HYDRATOR.hydrateFromBytes(oldValue.getContents()).getValue();
            long newValue = clears.contains(tableRef) ? writes.count(tableRef) : oldCount + writes.count(tableRef);
            log.debug("Sweep priority for {} has {} writes (was {})", tableRef, newValue, oldCount);
            newWriteCounts.put(cell, SweepPriorityTable.WriteCount.of(newValue).persistValue());
        }
        long timestamp = timestampService.getFreshTimestamp();
        // Committing before writing is intentional, we want the start timestamp to
        // show up in the transaction table before we write do our writes.
        commit(timestamp);
        delegate().put(SWEEP_PRIORITY_TABLE, newWriteCounts, timestamp);
    } catch (RuntimeException e) {
        if (Thread.interrupted()) {
            return;
        }
        Set<TableReference> allTableNames = delegate().getAllTableNames();
        if (!allTableNames.contains(SWEEP_PRIORITY_TABLE) || !allTableNames.contains(TransactionConstants.TRANSACTION_TABLE)) {
            // ignore problems when sweep or transaction tables don't exist
            log.warn("Ignoring failed sweep stats flush due to ", e);
        }
        log.error("Unable to flush sweep stats for writes {} and clears {}: ", writes, clears, e);
        throw e;
    }
}

Example 97 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class LoggingArgs method tableRefs.

public static SafeAndUnsafeTableReferences tableRefs(Collection<TableReference> tableReferences) {
    List<TableReference> safeTableRefs = new ArrayList<>();
    List<TableReference> unsafeTableRefs = new ArrayList<>();
    for (TableReference tableRef : tableReferences) {
        if (logArbitrator.isTableReferenceSafe(tableRef)) {
            safeTableRefs.add(tableRef);
        } else {
            unsafeTableRefs.add(tableRef);
        }
    }
    return ImmutableSafeAndUnsafeTableReferences.builder().safeTableRefs(SafeArg.of("safeTableRefs", safeTableRefs)).unsafeTableRefs(UnsafeArg.of("unsafeTableRefs", unsafeTableRefs)).build();
}

Example 98 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class TestScrubQueueMigrationCommand method testScrubQueueMigration.

@Test
public void testScrubQueueMigration() {
    TableReference foo = TableReference.createWithEmptyNamespace("foo");
    TableReference bar = TableReference.createWithEmptyNamespace("bar");
    TableReference baz = TableReference.createWithEmptyNamespace("baz");
    Cell cell1 = Cell.create(new byte[] { 1, 2, 3 }, new byte[] { 4, 5, 6 });
    Cell cell2 = Cell.create(new byte[] { 7, 8, 9 }, new byte[] { 4, 5, 6 });
    Cell cell3 = Cell.create(new byte[] { 1, 2, 3 }, new byte[] { 7, 8, 9 });
    kvs.createTable(AtlasDbConstants.OLD_SCRUB_TABLE, new byte[] { 0 });
    kvs.putWithTimestamps(AtlasDbConstants.OLD_SCRUB_TABLE, ImmutableMultimap.of(cell1, Value.create("foo\0bar".getBytes(), 10), cell1, Value.create("baz".getBytes(), 20), cell2, Value.create("foo".getBytes(), 30), cell3, Value.create("foo\0bar\0baz".getBytes(), 40)));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ScrubQueueMigrationCommand.run(kvs, new PrintWriter(baos, true), 1000);
    BatchingVisitable<SortedMap<Long, Multimap<TableReference, Cell>>> visitable = scrubStore.getBatchingVisitableScrubQueue(Long.MAX_VALUE, PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY);
    Assert.assertEquals(ImmutableSortedMap.of(10L, ImmutableMultimap.of(foo, cell1, bar, cell1), 20L, ImmutableMultimap.of(baz, cell1), 30L, ImmutableMultimap.of(foo, cell2), 40L, ImmutableMultimap.of(foo, cell3, bar, cell3, baz, cell3)), Iterables.getOnlyElement(BatchingVisitables.copyToList(visitable)));
    String output = new String(baos.toByteArray());
    Assert.assertTrue(output, output.contains("Starting iteration 2"));
    Assert.assertTrue(output, output.contains("Moved 4 cells"));
    Assert.assertTrue(output, output.contains("into 7 cells"));
    Assert.assertTrue(output, output.contains("Finished all iterations"));
}

Example 99 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class TestKvsMigrationCommand method checkKvs.

private void checkKvs(AtlasDbServices services, int numTables, int numEntriesPerTable) {
    for (int i = 0; i < numTables; i++) {
        TableReference table = TableReference.create(Namespace.create("ns"), "table" + i);
        services.getKeyValueService().createTable(table, AtlasDbConstants.GENERIC_TABLE_METADATA);
        services.getTransactionManager().runTaskThrowOnConflict(t -> {
            ImmutableSet.Builder<Cell> toRead = ImmutableSet.builder();
            ImmutableMap.Builder<Cell, byte[]> expectedBuilder = ImmutableMap.builder();
            for (int j = 0; j < numEntriesPerTable; j++) {
                Cell cell = Cell.create(PtBytes.toBytes("row" + j), PtBytes.toBytes("col"));
                toRead.add(cell);
                expectedBuilder.put(cell, PtBytes.toBytes("val" + j));
            }
            Map<Cell, byte[]> expected = expectedBuilder.build();
            Map<Cell, byte[]> result = t.get(table, expected.keySet());
            Assert.assertEquals(expected.keySet(), result.keySet());
            for (Entry<Cell, byte[]> e : result.entrySet()) {
                Assert.assertArrayEquals(expected.get(e.getKey()), e.getValue());
            }
            return null;
        });
    }
}

Example 100 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class CassandraKeyValueServiceTableCreationIntegrationTest method testCreateTableCanRestoreLostMetadata.

@Test
public void testCreateTableCanRestoreLostMetadata() {
    // setup a basic table
    TableReference missingMetadataTable = TableReference.createFromFullyQualifiedName("test.metadata_missing");
    byte[] initialMetadata = new TableDefinition() {

        {
            rowName();
            rowComponent("blob", ValueType.BLOB);
            columns();
            column("bar", "b", ValueType.BLOB);
            conflictHandler(ConflictHandler.IGNORE_ALL);
            sweepStrategy(TableMetadataPersistence.SweepStrategy.NOTHING);
        }
    }.toTableMetadata().persistToBytes();
    kvs.createTable(missingMetadataTable, initialMetadata);
    // retrieve the metadata and see that it's the same as what we just put in
    byte[] existingMetadata = kvs.getMetadataForTable(missingMetadataTable);
    assertThat(initialMetadata, is(existingMetadata));
    // Directly get and delete the metadata (`get` necessary to get the fake timestamp putMetadataForTables used)
    Cell cell = Cell.create(missingMetadataTable.getQualifiedName().getBytes(StandardCharsets.UTF_8), "m".getBytes(StandardCharsets.UTF_8));
    Value persistedMetadata = Iterables.getLast(kvs.get(AtlasDbConstants.DEFAULT_METADATA_TABLE, ImmutableMap.of(cell, Long.MAX_VALUE)).values());
    kvs.delete(AtlasDbConstants.DEFAULT_METADATA_TABLE, ImmutableMultimap.of(cell, persistedMetadata.getTimestamp()));
    // pretend we started up again and did a createTable() for our existing table, that no longer has metadata
    kvs.createTable(missingMetadataTable, initialMetadata);
    // retrieve the metadata again and see that it's the same as what we just put in
    existingMetadata = kvs.getMetadataForTable(missingMetadataTable);
    assertThat(initialMetadata, is(existingMetadata));
}