Examples with StreamMetadata com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata used on opensource projects

Example 46 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class StreamTestStreamStore method lookupStreamIdsByHash.

@Override
public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) {
    if (hashes.isEmpty()) {
        return ImmutableMap.of();
    }
    StreamTestStreamHashAidxTable idx = tables.getStreamTestStreamHashAidxTable(t);
    Set<StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes);
    Multimap<StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow, StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows);
    Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap();
    for (StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow r : m.keySet()) {
        for (StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumnValue v : m.get(r)) {
            Long streamId = v.getColumnName().getStreamId();
            Sha256Hash hash = r.getHash();
            if (hashForStreams.containsKey(streamId)) {
                AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId);
            }
            hashForStreams.put(streamId, hash);
        }
    }
    Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet());
    Map<Sha256Hash, Long> ret = Maps.newHashMap();
    for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) {
        if (e.getValue().getStatus() != Status.STORED) {
            continue;
        }
        Sha256Hash hash = hashForStreams.get(e.getKey());
        ret.put(hash, e.getKey());
    }
    return ret;
}

Example 47 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class StreamTestStreamStore method putHashIndexTask.

private void putHashIndexTask(Transaction t, Map<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> rowsToMetadata) {
    Multimap<StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow, StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumnValue> indexMap = HashMultimap.create();
    for (Entry<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) {
        StreamTestStreamMetadataTable.StreamTestStreamMetadataRow row = e.getKey();
        StreamMetadata metadata = e.getValue();
        Preconditions.checkArgument(metadata.getStatus() == Status.STORED, "Should only index successfully stored streams.");
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(metadata.getHash().toByteArray());
        }
        StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow hashRow = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow.of(hash);
        StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn column = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn.of(row.getId());
        StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumnValue columnValue = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumnValue.of(column, 0L);
        indexMap.put(hashRow, columnValue);
    }
    StreamTestStreamHashAidxTable hiTable = tables.getStreamTestStreamHashAidxTable(t);
    hiTable.put(indexMap);
}

Example 48 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class StreamTestWithHashMetadataCleanupTask method cellsCleanedUp.

@Override
public boolean cellsCleanedUp(Transaction t, Set<Cell> cells) {
    StreamTestWithHashStreamMetadataTable metaTable = tables.getStreamTestWithHashStreamMetadataTable(t);
    Collection<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow> rows = Lists.newArrayListWithCapacity(cells.size());
    for (Cell cell : cells) {
        rows.add(StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow.BYTES_HYDRATOR.hydrateFromBytes(cell.getRowName()));
    }
    Map<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> currentMetadata = metaTable.getMetadatas(rows);
    Set<Long> toDelete = Sets.newHashSet();
    for (Map.Entry<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> e : currentMetadata.entrySet()) {
        if (e.getValue().getStatus() != Status.STORED) {
            toDelete.add(e.getKey().getId());
        }
    }
    StreamTestWithHashStreamStore.of(tables).deleteStreams(t, toDelete);
    return false;
}

Example 49 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class StreamTestWithHashStreamStore method putHashIndexTask.

private void putHashIndexTask(Transaction t, Map<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> rowsToMetadata) {
    Multimap<StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow, StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumnValue> indexMap = HashMultimap.create();
    for (Entry<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) {
        StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow row = e.getKey();
        StreamMetadata metadata = e.getValue();
        Preconditions.checkArgument(metadata.getStatus() == Status.STORED, "Should only index successfully stored streams.");
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(metadata.getHash().toByteArray());
        }
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow hashRow = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow.of(hash);
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn column = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn.of(row.getId());
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumnValue columnValue = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumnValue.of(column, 0L);
        indexMap.put(hashRow, columnValue);
    }
    StreamTestWithHashStreamHashAidxTable hiTable = tables.getStreamTestWithHashStreamHashAidxTable(t);
    hiTable.put(indexMap);
}

Example 50 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class StreamTestWithHashStreamStore method makeStreamUsingTransaction.

private InputStream makeStreamUsingTransaction(Transaction parent, Long id, StreamMetadata metadata) {
    BiConsumer<Long, OutputStream> singleBlockLoader = (index, destination) -> loadSingleBlockToOutputStream(parent, id, index, destination);
    BlockGetter pageRefresher = new BlockLoader(singleBlockLoader, BLOCK_SIZE_IN_BYTES);
    long totalBlocks = getNumberOfBlocksFromMetadata(metadata);
    int blocksInMemory = getNumberOfBlocksThatFitInMemory();
    try {
        return BlockConsumingInputStream.create(pageRefresher, totalBlocks, blocksInMemory);
    } catch (IOException e) {
        throw Throwables.throwUncheckedException(e);
    }
}