use of com.palantir.util.crypto.Sha256Hash in project atlasdb by palantir.
the class StreamTest method testAddDelete.
@Test
public void testAddDelete() throws Exception {
final byte[] data = PtBytes.toBytes("streamed");
final long streamId = txManager.runTaskWithRetry((TransactionTask<Long, Exception>) t -> {
Sha256Hash hash = Sha256Hash.computeHash(data);
byte[] reference = "ref".getBytes();
return defaultStore.getByHashOrStoreStreamAndMarkAsUsed(t, hash, new ByteArrayInputStream(data), reference);
});
txManager.runTaskWithRetry((TransactionTask<Void, Exception>) t -> {
Optional<InputStream> inputStream = defaultStore.loadSingleStream(t, streamId);
assertTrue(inputStream.isPresent());
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
outputStream.write(inputStream.get());
assertArrayEquals(data, outputStream.toByteArray());
return null;
});
}
use of com.palantir.util.crypto.Sha256Hash in project atlasdb by palantir.
the class StreamTestMaxMemStreamStore method lookupStreamIdsByHash.
@Override
public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) {
if (hashes.isEmpty()) {
return ImmutableMap.of();
}
StreamTestMaxMemStreamHashAidxTable idx = tables.getStreamTestMaxMemStreamHashAidxTable(t);
Set<StreamTestMaxMemStreamHashAidxTable.StreamTestMaxMemStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes);
Multimap<StreamTestMaxMemStreamHashAidxTable.StreamTestMaxMemStreamHashAidxRow, StreamTestMaxMemStreamHashAidxTable.StreamTestMaxMemStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows);
Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap();
for (StreamTestMaxMemStreamHashAidxTable.StreamTestMaxMemStreamHashAidxRow r : m.keySet()) {
for (StreamTestMaxMemStreamHashAidxTable.StreamTestMaxMemStreamHashAidxColumnValue v : m.get(r)) {
Long streamId = v.getColumnName().getStreamId();
Sha256Hash hash = r.getHash();
if (hashForStreams.containsKey(streamId)) {
AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId);
}
hashForStreams.put(streamId, hash);
}
}
Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet());
Map<Sha256Hash, Long> ret = Maps.newHashMap();
for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) {
if (e.getValue().getStatus() != Status.STORED) {
continue;
}
Sha256Hash hash = hashForStreams.get(e.getKey());
ret.put(hash, e.getKey());
}
return ret;
}
use of com.palantir.util.crypto.Sha256Hash in project atlasdb by palantir.
the class StreamTestStreamStore method deleteStreams.
/**
* This should only be used from the cleanup tasks.
*/
void deleteStreams(Transaction t, final Set<Long> streamIds) {
if (streamIds.isEmpty()) {
return;
}
Set<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow> smRows = Sets.newHashSet();
Multimap<StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow, StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn> shToDelete = HashMultimap.create();
for (Long streamId : streamIds) {
smRows.add(StreamTestStreamMetadataTable.StreamTestStreamMetadataRow.of(streamId));
}
StreamTestStreamMetadataTable table = tables.getStreamTestStreamMetadataTable(t);
Map<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
Set<StreamTestStreamValueTable.StreamTestStreamValueRow> streamValueToDelete = Sets.newHashSet();
for (Entry<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
Long streamId = e.getKey().getId();
long blocks = getNumberOfBlocksFromMetadata(e.getValue());
for (long i = 0; i < blocks; i++) {
streamValueToDelete.add(StreamTestStreamValueTable.StreamTestStreamValueRow.of(streamId, i));
}
ByteString streamHash = e.getValue().getHash();
Sha256Hash hash = Sha256Hash.EMPTY;
if (streamHash != com.google.protobuf.ByteString.EMPTY) {
hash = new Sha256Hash(streamHash.toByteArray());
} else {
log.error("Empty hash for stream {}", streamId);
}
StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow hashRow = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow.of(hash);
StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn column = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn.of(streamId);
shToDelete.put(hashRow, column);
}
tables.getStreamTestStreamHashAidxTable(t).delete(shToDelete);
tables.getStreamTestStreamValueTable(t).delete(streamValueToDelete);
table.delete(smRows);
}
use of com.palantir.util.crypto.Sha256Hash in project atlasdb by palantir.
the class StreamTestWithHashStreamStore method deleteStreams.
/**
* This should only be used from the cleanup tasks.
*/
void deleteStreams(Transaction t, final Set<Long> streamIds) {
if (streamIds.isEmpty()) {
return;
}
Set<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow> smRows = Sets.newHashSet();
Multimap<StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow, StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn> shToDelete = HashMultimap.create();
for (Long streamId : streamIds) {
smRows.add(StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow.of(streamId));
}
StreamTestWithHashStreamMetadataTable table = tables.getStreamTestWithHashStreamMetadataTable(t);
Map<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
Set<StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow> streamValueToDelete = Sets.newHashSet();
for (Entry<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
Long streamId = e.getKey().getId();
long blocks = getNumberOfBlocksFromMetadata(e.getValue());
for (long i = 0; i < blocks; i++) {
streamValueToDelete.add(StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow.of(streamId, i));
}
ByteString streamHash = e.getValue().getHash();
Sha256Hash hash = Sha256Hash.EMPTY;
if (streamHash != com.google.protobuf.ByteString.EMPTY) {
hash = new Sha256Hash(streamHash.toByteArray());
} else {
log.error("Empty hash for stream {}", streamId);
}
StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow hashRow = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow.of(hash);
StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn column = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn.of(streamId);
shToDelete.put(hashRow, column);
}
tables.getStreamTestWithHashStreamHashAidxTable(t).delete(shToDelete);
tables.getStreamTestWithHashStreamValueTable(t).delete(streamValueToDelete);
table.delete(smRows);
}
use of com.palantir.util.crypto.Sha256Hash in project atlasdb by palantir.
the class UserPhotosStreamStore method lookupStreamIdsByHash.
@Override
public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) {
if (hashes.isEmpty()) {
return ImmutableMap.of();
}
UserPhotosStreamHashAidxTable idx = tables.getUserPhotosStreamHashAidxTable(t);
Set<UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes);
Multimap<UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow, UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows);
Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap();
for (UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow r : m.keySet()) {
for (UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxColumnValue v : m.get(r)) {
Long streamId = v.getColumnName().getStreamId();
Sha256Hash hash = r.getHash();
if (hashForStreams.containsKey(streamId)) {
AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId);
}
hashForStreams.put(streamId, hash);
}
}
Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet());
Map<Sha256Hash, Long> ret = Maps.newHashMap();
for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) {
if (e.getValue().getStatus() != Status.STORED) {
continue;
}
Sha256Hash hash = hashForStreams.get(e.getKey());
ret.put(hash, e.getKey());
}
return ret;
}
Aggregations