Examples with TableReference com.palantir.atlasdb.keyvalue.api.TableReference used on opensource projects

Example 66 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class KeyValueServiceMigrator method setup.

/**
 * Drop and create tables before starting migration.
 */
public void setup() {
    /*
         * We do *not* want to drop tables that aren't getting migrated (since that would drop them
         * ON THE SOURCE KVS), but we *do* actually want to create metadata for those tables,
         * because they might be migrated later by dbcopy.
         */
    processMessage("dropping tables: " + getCreatableTableNames(toKvs), KvsMigrationMessageLevel.INFO);
    toKvs.dropTables(getCreatableTableNames(toKvs));
    Map<TableReference, byte[]> metadataByTableName = Maps.newHashMap();
    for (TableReference tableRef : getCreatableTableNames(fromKvs)) {
        processMessage("retrieving metadata for table " + tableRef, KvsMigrationMessageLevel.INFO);
        byte[] metadata = fromKvs.getMetadataForTable(tableRef);
        Preconditions.checkArgument(metadata != null && metadata.length != 0, "no metadata found for table %s", tableRef);
        metadataByTableName.put(tableRef, metadata);
    }
    processMessage("creating tables", KvsMigrationMessageLevel.INFO);
    toKvs.createTables(metadataByTableName);
    processMessage("setup complete", KvsMigrationMessageLevel.INFO);
}

Example 67 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class KeyValueServiceMigrator method migrateTables.

private void migrateTables(Set<TableReference> tables, TransactionManager readTxManager, TransactionManager txManager, KeyValueService writeKvs, long migrationTimestamp, ExecutorService executor, GeneralTaskCheckpointer checkpointer) {
    processMessage("Migrating tables at migrationTimestamp " + migrationTimestamp, KvsMigrationMessageLevel.INFO);
    for (TableReference table : tables) {
        KvsRangeMigrator rangeMigrator = new KvsRangeMigratorBuilder().srcTable(table).readBatchSize(getBatchSize(table)).readTxManager(readTxManager).txManager(txManager).writeKvs(writeKvs).migrationTimestamp(migrationTimestamp).checkpointer(checkpointer).build();
        TableMigratorBuilder builder = new TableMigratorBuilder().srcTable(table).partitions(PARTITIONS).partitioners(getPartitioners(fromKvs, table)).readBatchSize(getBatchSize(table)).executor(executor).checkpointer(checkpointer).progress(taskProgress).rangeMigrator(rangeMigrator);
        TableMigrator migrator = builder.build();
        migrator.migrate();
    }
}

Example 68 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class KeyValueServiceScrubberStore method queueCellsForScrubbing.

@Override
public void queueCellsForScrubbing(Multimap<Cell, TableReference> cellToTableRefs, long scrubTimestamp, int batchSize) {
    Map<Cell, byte[]> values = Maps.newHashMap();
    for (Map.Entry<Cell, Collection<TableReference>> entry : cellToTableRefs.asMap().entrySet()) {
        Cell cell = entry.getKey();
        for (TableReference tableRef : entry.getValue()) {
            byte[] tableBytes = EncodingUtils.encodeVarString(tableRef.getQualifiedName());
            byte[] col = EncodingUtils.add(tableBytes, cell.getColumnName());
            values.put(Cell.create(cell.getRowName(), col), EMPTY_CONTENTS);
        }
    }
    for (List<Entry<Cell, byte[]>> batch : Iterables.partition(values.entrySet(), batchSize)) {
        Map<Cell, byte[]> batchMap = Maps.newHashMap();
        for (Entry<Cell, byte[]> e : batch) {
            batchMap.put(e.getKey(), e.getValue());
        }
        keyValueService.put(AtlasDbConstants.SCRUB_TABLE, batchMap, scrubTimestamp);
    }
}

Example 69 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class KeyValueServiceScrubberStore method transformRows.

private SortedMap<Long, Multimap<TableReference, Cell>> transformRows(List<RowResult<Value>> input) {
    SortedMap<Long, Multimap<TableReference, Cell>> scrubTimestampToTableNameToCell = Maps.newTreeMap();
    for (RowResult<Value> rowResult : input) {
        byte[] row = rowResult.getRowName();
        for (Entry<byte[], Value> entry : rowResult.getColumns().entrySet()) {
            byte[] fullCol = entry.getKey();
            String table = EncodingUtils.decodeVarString(fullCol);
            byte[] col = Arrays.copyOfRange(fullCol, EncodingUtils.sizeOfVarString(table), fullCol.length);
            TableReference tableRef = TableReference.fromString(table);
            Cell cell = Cell.create(row, col);
            long timestamp = entry.getValue().getTimestamp();
            Multimap<TableReference, Cell> cells = scrubTimestampToTableNameToCell.get(timestamp);
            if (cells == null) {
                cells = ArrayListMultimap.create();
                scrubTimestampToTableNameToCell.put(timestamp, cells);
            }
            cells.put(tableRef, cell);
        }
    }
    return scrubTimestampToTableNameToCell;
}

Example 70 with TableReference

use of com.palantir.atlasdb.keyvalue.api.TableReference in project atlasdb by palantir.

the class Scrubber method scrubSomeCells.

/**
 * Scrubs some cells.
 *
 * @return number of cells read from _scrub table
 */
private int scrubSomeCells(SortedMap<Long, Multimap<TableReference, Cell>> scrubTimestampToTableNameToCell, TransactionManager txManager, long maxScrubTimestamp) {
    log.trace("Attempting to scrub cells: {}", scrubTimestampToTableNameToCell);
    if (log.isDebugEnabled()) {
        int numCells = 0;
        Set<TableReference> tables = Sets.newHashSet();
        for (Multimap<TableReference, Cell> v : scrubTimestampToTableNameToCell.values()) {
            tables.addAll(v.keySet());
            numCells += v.size();
        }
        log.debug("Attempting to scrub {} cells from tables {}", numCells, tables);
    }
    if (scrubTimestampToTableNameToCell.size() == 0) {
        // No cells left to scrub
        return 0;
    }
    int numCellsReadFromScrubTable = 0;
    List<Future<Void>> scrubFutures = Lists.newArrayList();
    Map<TableReference, Multimap<Cell, Long>> failedWrites = Maps.newHashMap();
    for (Map.Entry<Long, Multimap<TableReference, Cell>> entry : scrubTimestampToTableNameToCell.entrySet()) {
        final long scrubTimestamp = entry.getKey();
        final Multimap<TableReference, Cell> tableNameToCell = entry.getValue();
        numCellsReadFromScrubTable += tableNameToCell.size();
        // This is CRITICAL; don't scrub if the hard delete transaction didn't actually finish
        // (we still remove it from the _scrub table with the call to markCellsAsScrubbed though),
        // or else we could cause permanent data loss if the hard delete transaction failed after
        // queuing cells to scrub but before successfully committing
        long commitTimestamp = getCommitTimestampRollBackIfNecessary(scrubTimestamp, tableNameToCell);
        if (commitTimestamp == TransactionConstants.FAILED_COMMIT_TS) {
            for (Entry<TableReference, Collection<Cell>> cells : tableNameToCell.asMap().entrySet()) {
                Multimap<Cell, Long> failedCells = failedWrites.get(cells.getKey());
                if (failedCells == null) {
                    failedCells = ArrayListMultimap.create(cells.getValue().size(), 2);
                    failedWrites.put(cells.getKey(), failedCells);
                }
                for (Cell cell : cells.getValue()) {
                    failedCells.put(cell, scrubTimestamp);
                }
            }
        } else if (commitTimestamp < maxScrubTimestamp) {
            for (final List<Entry<TableReference, Cell>> batch : Iterables.partition(tableNameToCell.entries(), batchSizeSupplier.get())) {
                final Multimap<TableReference, Cell> batchMultimap = HashMultimap.create();
                for (Entry<TableReference, Cell> e : batch) {
                    batchMultimap.put(e.getKey(), e.getValue());
                }
                scrubFutures.add(exec.submit(() -> {
                    scrubCells(txManager, batchMultimap, scrubTimestamp, aggressiveScrub ? TransactionType.AGGRESSIVE_HARD_DELETE : TransactionType.HARD_DELETE);
                    return null;
                }));
            }
        }
    // else {
    // We cannot scrub this yet because not all transactions can read this value.
    // }
    }
    for (Future<Void> future : scrubFutures) {
        Futures.getUnchecked(future);
    }
    if (!failedWrites.isEmpty()) {
        scrubberStore.markCellsAsScrubbed(failedWrites, batchSizeSupplier.get());
    }
    log.trace("Finished scrubbing cells: {}", scrubTimestampToTableNameToCell);
    if (log.isDebugEnabled()) {
        Set<TableReference> tables = Sets.newHashSet();
        for (Multimap<TableReference, Cell> v : scrubTimestampToTableNameToCell.values()) {
            tables.addAll(v.keySet());
        }
        long minTimestamp = Collections.min(scrubTimestampToTableNameToCell.keySet());
        long maxTimestamp = Collections.max(scrubTimestampToTableNameToCell.keySet());
        log.debug("Finished scrubbing {} cells at {} timestamps ({}...{}) from tables {}", numCellsReadFromScrubTable, scrubTimestampToTableNameToCell.size(), minTimestamp, maxTimestamp, tables);
    }
    return numCellsReadFromScrubTable;
}