Examples with StreamMetadata com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata used on opensource projects

Example 56 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class TestHashComponentsStreamStore method touchMetadataWhileMarkingUsedForConflicts.

@Override
protected void touchMetadataWhileMarkingUsedForConflicts(Transaction t, Iterable<Long> ids) {
    TestHashComponentsStreamMetadataTable metaTable = tables.getTestHashComponentsStreamMetadataTable(t);
    Set<TestHashComponentsStreamMetadataTable.TestHashComponentsStreamMetadataRow> rows = Sets.newHashSet();
    for (Long id : ids) {
        rows.add(TestHashComponentsStreamMetadataTable.TestHashComponentsStreamMetadataRow.of(id));
    }
    Map<TestHashComponentsStreamMetadataTable.TestHashComponentsStreamMetadataRow, StreamMetadata> metadatas = metaTable.getMetadatas(rows);
    for (Map.Entry<TestHashComponentsStreamMetadataTable.TestHashComponentsStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        StreamMetadata metadata = e.getValue();
        Preconditions.checkState(metadata.getStatus() == Status.STORED, "Stream: %s has status: %s", e.getKey().getId(), metadata.getStatus());
        metaTable.putMetadata(e.getKey(), metadata);
    }
    SetView<TestHashComponentsStreamMetadataTable.TestHashComponentsStreamMetadataRow> missingRows = Sets.difference(rows, metadatas.keySet());
    if (!missingRows.isEmpty()) {
        throw new IllegalStateException("Missing metadata rows for:" + missingRows + " rows: " + rows + " metadata: " + metadatas + " txn timestamp: " + t.getTimestamp());
    }
}

Example 57 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class UserPhotosStreamStore method putMetadataAndHashIndexTask.

@Override
protected void putMetadataAndHashIndexTask(Transaction t, Map<Long, StreamMetadata> streamIdsToMetadata) {
    UserPhotosStreamMetadataTable mdTable = tables.getUserPhotosStreamMetadataTable(t);
    Map<Long, StreamMetadata> prevMetadatas = getMetadata(t, streamIdsToMetadata.keySet());
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> rowsToStoredMetadata = Maps.newHashMap();
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> rowsToUnstoredMetadata = Maps.newHashMap();
    for (Entry<Long, StreamMetadata> e : streamIdsToMetadata.entrySet()) {
        long streamId = e.getKey();
        StreamMetadata metadata = e.getValue();
        StreamMetadata prevMetadata = prevMetadatas.get(streamId);
        if (metadata.getStatus() == Status.STORED) {
            if (prevMetadata == null || prevMetadata.getStatus() != Status.STORING) {
                // This can happen if we cleanup old streams.
                throw new TransactionFailedRetriableException("Cannot mark a stream as stored that isn't currently storing: " + prevMetadata);
            }
            rowsToStoredMetadata.put(UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow.of(streamId), metadata);
        } else if (metadata.getStatus() == Status.STORING) {
            // This will prevent two users trying to store the same id.
            if (prevMetadata != null) {
                throw new TransactionFailedRetriableException("Cannot reuse the same stream id: " + streamId);
            }
            rowsToUnstoredMetadata.put(UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow.of(streamId), metadata);
        }
    }
    putHashIndexTask(t, rowsToStoredMetadata);
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> rowsToMetadata = Maps.newHashMap();
    rowsToMetadata.putAll(rowsToStoredMetadata);
    rowsToMetadata.putAll(rowsToUnstoredMetadata);
    mdTable.putMetadata(rowsToMetadata);
}

Example 58 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class UserPhotosMetadataCleanupTask method cellsCleanedUp.

@Override
public boolean cellsCleanedUp(Transaction t, Set<Cell> cells) {
    UserPhotosStreamMetadataTable metaTable = tables.getUserPhotosStreamMetadataTable(t);
    Collection<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow> rows = Lists.newArrayListWithCapacity(cells.size());
    for (Cell cell : cells) {
        rows.add(UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow.BYTES_HYDRATOR.hydrateFromBytes(cell.getRowName()));
    }
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> currentMetadata = metaTable.getMetadatas(rows);
    Set<Long> toDelete = Sets.newHashSet();
    for (Map.Entry<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> e : currentMetadata.entrySet()) {
        if (e.getValue().getStatus() != Status.STORED) {
            toDelete.add(e.getKey().getId());
        }
    }
    UserPhotosStreamStore.of(tables).deleteStreams(t, toDelete);
    return false;
}

Example 59 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class UserPhotosStreamStore method touchMetadataWhileMarkingUsedForConflicts.

@Override
protected void touchMetadataWhileMarkingUsedForConflicts(Transaction t, Iterable<Long> ids) {
    UserPhotosStreamMetadataTable metaTable = tables.getUserPhotosStreamMetadataTable(t);
    Set<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow> rows = Sets.newHashSet();
    for (Long id : ids) {
        rows.add(UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow.of(id));
    }
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> metadatas = metaTable.getMetadatas(rows);
    for (Map.Entry<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        StreamMetadata metadata = e.getValue();
        Preconditions.checkState(metadata.getStatus() == Status.STORED, "Stream: %s has status: %s", e.getKey().getId(), metadata.getStatus());
        metaTable.putMetadata(e.getKey(), metadata);
    }
    SetView<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow> missingRows = Sets.difference(rows, metadatas.keySet());
    if (!missingRows.isEmpty()) {
        throw new IllegalStateException("Missing metadata rows for:" + missingRows + " rows: " + rows + " metadata: " + metadatas + " txn timestamp: " + t.getTimestamp());
    }
}

Example 60 with StreamMetadata

use of com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata in project atlasdb by palantir.

the class UserPhotosStreamStore method deleteStreams.

/**
 * This should only be used from the cleanup tasks.
 */
void deleteStreams(Transaction t, final Set<Long> streamIds) {
    if (streamIds.isEmpty()) {
        return;
    }
    Set<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow> smRows = Sets.newHashSet();
    Multimap<UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow, UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxColumn> shToDelete = HashMultimap.create();
    for (Long streamId : streamIds) {
        smRows.add(UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow.of(streamId));
    }
    UserPhotosStreamMetadataTable table = tables.getUserPhotosStreamMetadataTable(t);
    Map<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
    Set<UserPhotosStreamValueTable.UserPhotosStreamValueRow> streamValueToDelete = Sets.newHashSet();
    for (Entry<UserPhotosStreamMetadataTable.UserPhotosStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        Long streamId = e.getKey().getId();
        long blocks = getNumberOfBlocksFromMetadata(e.getValue());
        for (long i = 0; i < blocks; i++) {
            streamValueToDelete.add(UserPhotosStreamValueTable.UserPhotosStreamValueRow.of(streamId, i));
        }
        ByteString streamHash = e.getValue().getHash();
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (streamHash != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(streamHash.toByteArray());
        } else {
            log.error("Empty hash for stream {}", streamId);
        }
        UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow hashRow = UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxRow.of(hash);
        UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxColumn column = UserPhotosStreamHashAidxTable.UserPhotosStreamHashAidxColumn.of(streamId);
        shToDelete.put(hashRow, column);
    }
    tables.getUserPhotosStreamHashAidxTable(t).delete(shToDelete);
    tables.getUserPhotosStreamValueTable(t).delete(streamValueToDelete);
    table.delete(smRows);
}