Examples with StreamMetadata com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata used on opensource projects

Example 21 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class AbstractPersistentStreamStore method storeStream.

@Override
public Pair<Long, Sha256Hash> storeStream(InputStream stream) {
    // Store empty metadata before doing anything
    long id = storeEmptyMetadata();
    StreamMetadata metadata = storeBlocksAndGetFinalMetadata(null, id, stream);
    storeMetadataAndIndex(id, metadata);
    return Pair.create(id, new Sha256Hash(metadata.getHash().toByteArray()));
}

Example 22 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class ValueMetadataCleanupTask method cellsCleanedUp.

@Override
public boolean cellsCleanedUp(Transaction t, Set<Cell> cells) {
    ValueStreamMetadataTable metaTable = tables.getValueStreamMetadataTable(t);
    Collection<ValueStreamMetadataTable.ValueStreamMetadataRow> rows = Lists.newArrayListWithCapacity(cells.size());
    for (Cell cell : cells) {
        rows.add(ValueStreamMetadataTable.ValueStreamMetadataRow.of((Long) ValueType.VAR_LONG.convertToJava(cell.getRowName(), 0)));
    }
    Map<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> currentMetadata = metaTable.getMetadatas(rows);
    Set<Long> toDelete = Sets.newHashSet();
    for (Map.Entry<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> e : currentMetadata.entrySet()) {
        if (e.getValue().getStatus() != Status.STORED) {
            toDelete.add(e.getKey().getId());
        }
    }
    ValueStreamStore.of(tables).deleteStreams(t, toDelete);
    return false;
}

Example 23 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class ValueStreamStore method lookupStreamIdsByHash.

@Override
public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) {
    if (hashes.isEmpty()) {
        return ImmutableMap.of();
    }
    ValueStreamHashAidxTable idx = tables.getValueStreamHashAidxTable(t);
    Set<ValueStreamHashAidxTable.ValueStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes);
    Multimap<ValueStreamHashAidxTable.ValueStreamHashAidxRow, ValueStreamHashAidxTable.ValueStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows);
    Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap();
    for (ValueStreamHashAidxTable.ValueStreamHashAidxRow r : m.keySet()) {
        for (ValueStreamHashAidxTable.ValueStreamHashAidxColumnValue v : m.get(r)) {
            Long streamId = v.getColumnName().getStreamId();
            Sha256Hash hash = r.getHash();
            if (hashForStreams.containsKey(streamId)) {
                AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId);
            }
            hashForStreams.put(streamId, hash);
        }
    }
    Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet());
    Map<Sha256Hash, Long> ret = Maps.newHashMap();
    for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) {
        if (e.getValue().getStatus() != Status.STORED) {
            continue;
        }
        Sha256Hash hash = hashForStreams.get(e.getKey());
        ret.put(hash, e.getKey());
    }
    return ret;
}

Example 24 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class ValueStreamStore method putHashIndexTask.

private void putHashIndexTask(Transaction t, Map<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> rowsToMetadata) {
    Multimap<ValueStreamHashAidxTable.ValueStreamHashAidxRow, ValueStreamHashAidxTable.ValueStreamHashAidxColumnValue> indexMap = HashMultimap.create();
    for (Entry<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) {
        ValueStreamMetadataTable.ValueStreamMetadataRow row = e.getKey();
        StreamMetadata metadata = e.getValue();
        Preconditions.checkArgument(metadata.getStatus() == Status.STORED, "Should only index successfully stored streams.");
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(metadata.getHash().toByteArray());
        }
        ValueStreamHashAidxTable.ValueStreamHashAidxRow hashRow = ValueStreamHashAidxTable.ValueStreamHashAidxRow.of(hash);
        ValueStreamHashAidxTable.ValueStreamHashAidxColumn column = ValueStreamHashAidxTable.ValueStreamHashAidxColumn.of(row.getId());
        ValueStreamHashAidxTable.ValueStreamHashAidxColumnValue columnValue = ValueStreamHashAidxTable.ValueStreamHashAidxColumnValue.of(column, 0L);
        indexMap.put(hashRow, columnValue);
    }
    ValueStreamHashAidxTable hiTable = tables.getValueStreamHashAidxTable(t);
    hiTable.put(indexMap);
}

Example 25 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class ValueStreamStore method putMetadataAndHashIndexTask.

@Override
protected void putMetadataAndHashIndexTask(Transaction t, Map<Long, StreamMetadata> streamIdsToMetadata) {
    ValueStreamMetadataTable mdTable = tables.getValueStreamMetadataTable(t);
    Map<Long, StreamMetadata> prevMetadatas = getMetadata(t, streamIdsToMetadata.keySet());
    Map<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> rowsToStoredMetadata = Maps.newHashMap();
    Map<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> rowsToUnstoredMetadata = Maps.newHashMap();
    for (Entry<Long, StreamMetadata> e : streamIdsToMetadata.entrySet()) {
        long streamId = e.getKey();
        StreamMetadata metadata = e.getValue();
        StreamMetadata prevMetadata = prevMetadatas.get(streamId);
        if (metadata.getStatus() == Status.STORED) {
            if (prevMetadata == null || prevMetadata.getStatus() != Status.STORING) {
                // This can happen if we cleanup old streams.
                throw new TransactionFailedRetriableException("Cannot mark a stream as stored that isn't currently storing: " + prevMetadata);
            }
            rowsToStoredMetadata.put(ValueStreamMetadataTable.ValueStreamMetadataRow.of(streamId), metadata);
        } else if (metadata.getStatus() == Status.STORING) {
            // This will prevent two users trying to store the same id.
            if (prevMetadata != null) {
                throw new TransactionFailedRetriableException("Cannot reuse the same stream id: " + streamId);
            }
            rowsToUnstoredMetadata.put(ValueStreamMetadataTable.ValueStreamMetadataRow.of(streamId), metadata);
        }
    }
    putHashIndexTask(t, rowsToStoredMetadata);
    Map<ValueStreamMetadataTable.ValueStreamMetadataRow, StreamMetadata> rowsToMetadata = Maps.newHashMap();
    rowsToMetadata.putAll(rowsToStoredMetadata);
    rowsToMetadata.putAll(rowsToUnstoredMetadata);
    mdTable.putMetadata(rowsToMetadata);
}