Examples with ByteString com.google.protobuf.ByteString used on opensource projects

Example 81 with ByteString

use of com.google.protobuf.ByteString in project atlasdb by palantir.

the class StreamTestStreamStore method deleteStreams.

/**
 * This should only be used from the cleanup tasks.
 */
void deleteStreams(Transaction t, final Set<Long> streamIds) {
    if (streamIds.isEmpty()) {
        return;
    }
    Set<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow> smRows = Sets.newHashSet();
    Multimap<StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow, StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn> shToDelete = HashMultimap.create();
    for (Long streamId : streamIds) {
        smRows.add(StreamTestStreamMetadataTable.StreamTestStreamMetadataRow.of(streamId));
    }
    StreamTestStreamMetadataTable table = tables.getStreamTestStreamMetadataTable(t);
    Map<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
    Set<StreamTestStreamValueTable.StreamTestStreamValueRow> streamValueToDelete = Sets.newHashSet();
    for (Entry<StreamTestStreamMetadataTable.StreamTestStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        Long streamId = e.getKey().getId();
        long blocks = getNumberOfBlocksFromMetadata(e.getValue());
        for (long i = 0; i < blocks; i++) {
            streamValueToDelete.add(StreamTestStreamValueTable.StreamTestStreamValueRow.of(streamId, i));
        }
        ByteString streamHash = e.getValue().getHash();
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (streamHash != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(streamHash.toByteArray());
        } else {
            log.error("Empty hash for stream {}", streamId);
        }
        StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow hashRow = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxRow.of(hash);
        StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn column = StreamTestStreamHashAidxTable.StreamTestStreamHashAidxColumn.of(streamId);
        shToDelete.put(hashRow, column);
    }
    tables.getStreamTestStreamHashAidxTable(t).delete(shToDelete);
    tables.getStreamTestStreamValueTable(t).delete(streamValueToDelete);
    table.delete(smRows);
}

Example 82 with ByteString

use of com.google.protobuf.ByteString in project atlasdb by palantir.

the class StreamTestWithHashStreamStore method deleteStreams.

/**
 * This should only be used from the cleanup tasks.
 */
void deleteStreams(Transaction t, final Set<Long> streamIds) {
    if (streamIds.isEmpty()) {
        return;
    }
    Set<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow> smRows = Sets.newHashSet();
    Multimap<StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow, StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn> shToDelete = HashMultimap.create();
    for (Long streamId : streamIds) {
        smRows.add(StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow.of(streamId));
    }
    StreamTestWithHashStreamMetadataTable table = tables.getStreamTestWithHashStreamMetadataTable(t);
    Map<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
    Set<StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow> streamValueToDelete = Sets.newHashSet();
    for (Entry<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        Long streamId = e.getKey().getId();
        long blocks = getNumberOfBlocksFromMetadata(e.getValue());
        for (long i = 0; i < blocks; i++) {
            streamValueToDelete.add(StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow.of(streamId, i));
        }
        ByteString streamHash = e.getValue().getHash();
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (streamHash != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(streamHash.toByteArray());
        } else {
            log.error("Empty hash for stream {}", streamId);
        }
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow hashRow = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow.of(hash);
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn column = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn.of(streamId);
        shToDelete.put(hashRow, column);
    }
    tables.getStreamTestWithHashStreamHashAidxTable(t).delete(shToDelete);
    tables.getStreamTestWithHashStreamValueTable(t).delete(streamValueToDelete);
    table.delete(smRows);
}

Example 83 with ByteString

use of com.google.protobuf.ByteString in project controller by opendaylight.

the class FollowerTest method testInitialSyncUpWithHandleInstallSnapshotFollowedByAppendEntries.

@Test
public void testInitialSyncUpWithHandleInstallSnapshotFollowedByAppendEntries() {
    logStart("testInitialSyncUpWithHandleInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    context.setCommitIndex(-1);
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int offset = 0;
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    int chunkIndex = 1;
    InstallSnapshot lastInstallSnapshot = null;
    for (int i = 0; i < totalChunks; i++) {
        byte[] chunkData = getNextChunk(bsSnapshot, offset, chunkSize);
        lastInstallSnapshot = new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, chunkIndex, totalChunks);
        follower.handleMessage(leaderActor, lastInstallSnapshot);
        offset = offset + 50;
        lastIncludedIndex++;
        chunkIndex++;
    }
    FollowerInitialSyncUpStatus syncStatus = MessageCollectorActor.expectFirstMatching(followerActor, FollowerInitialSyncUpStatus.class);
    assertFalse(syncStatus.isInitialSyncDone());
    // Clear all the messages
    MessageCollectorActor.clearMessages(followerActor);
    context.setLastApplied(101);
    context.setCommitIndex(101);
    setLastLogEntry(context, 1, 101, new MockRaftActorContext.MockPayload(""));
    List<ReplicatedLogEntry> entries = Arrays.asList(newReplicatedLogEntry(2, 101, "foo"));
    // The new commitIndex is 101
    AppendEntries appendEntries = new AppendEntries(2, "leader", 101, 1, entries, 102, 101, (short) 0);
    follower.handleMessage(leaderActor, appendEntries);
    syncStatus = MessageCollectorActor.expectFirstMatching(followerActor, FollowerInitialSyncUpStatus.class);
    assertTrue(syncStatus.isInitialSyncDone());
}

Example 84 with ByteString

use of com.google.protobuf.ByteString in project controller by opendaylight.

the class FollowerTest method testHandleInstallSnapshot.

/**
 * This test verifies that when InstallSnapshot is received by
 * the follower its applied correctly.
 */
@Test
public void testHandleInstallSnapshot() {
    logStart("testHandleInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    context.getTermInformation().update(1, "leader");
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int offset = 0;
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    int chunkIndex = 1;
    InstallSnapshot lastInstallSnapshot = null;
    for (int i = 0; i < totalChunks; i++) {
        byte[] chunkData = getNextChunk(bsSnapshot, offset, chunkSize);
        lastInstallSnapshot = new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, chunkIndex, totalChunks);
        follower.handleMessage(leaderActor, lastInstallSnapshot);
        offset = offset + 50;
        lastIncludedIndex++;
        chunkIndex++;
    }
    ApplySnapshot applySnapshot = MessageCollectorActor.expectFirstMatching(followerActor, ApplySnapshot.class);
    Snapshot snapshot = applySnapshot.getSnapshot();
    assertNotNull(lastInstallSnapshot);
    assertEquals("getLastIndex", lastInstallSnapshot.getLastIncludedIndex(), snapshot.getLastIndex());
    assertEquals("getLastIncludedTerm", lastInstallSnapshot.getLastIncludedTerm(), snapshot.getLastAppliedTerm());
    assertEquals("getLastAppliedIndex", lastInstallSnapshot.getLastIncludedIndex(), snapshot.getLastAppliedIndex());
    assertEquals("getLastTerm", lastInstallSnapshot.getLastIncludedTerm(), snapshot.getLastTerm());
    assertEquals("getState type", ByteState.class, snapshot.getState().getClass());
    Assert.assertArrayEquals("getState", bsSnapshot.toByteArray(), ((ByteState) snapshot.getState()).getBytes());
    assertEquals("getElectionTerm", 1, snapshot.getElectionTerm());
    assertEquals("getElectionVotedFor", "leader", snapshot.getElectionVotedFor());
    applySnapshot.getCallback().onSuccess();
    List<InstallSnapshotReply> replies = MessageCollectorActor.getAllMatching(leaderActor, InstallSnapshotReply.class);
    assertEquals("InstallSnapshotReply count", totalChunks, replies.size());
    chunkIndex = 1;
    for (InstallSnapshotReply reply : replies) {
        assertEquals("getChunkIndex", chunkIndex++, reply.getChunkIndex());
        assertEquals("getTerm", 1, reply.getTerm());
        assertEquals("isSuccess", true, reply.isSuccess());
        assertEquals("getFollowerId", context.getId(), reply.getFollowerId());
    }
    assertNull("Expected null SnapshotTracker", follower.getSnapshotTracker());
}

Example 85 with ByteString

use of com.google.protobuf.ByteString in project controller by opendaylight.

the class LeaderTest method testHandleInstallSnapshotReplyWithInvalidChunkIndex.

@Test
public void testHandleInstallSnapshotReplyWithInvalidChunkIndex() throws Exception {
    logStart("testHandleInstallSnapshotReplyWithInvalidChunkIndex");
    MockRaftActorContext actorContext = createActorContextWithFollower();
    final int commitIndex = 3;
    final int snapshotIndex = 2;
    final int snapshotTerm = 1;
    final int currentTerm = 2;
    actorContext.setConfigParams(new DefaultConfigParamsImpl() {

        @Override
        public int getSnapshotChunkSize() {
            return 50;
        }
    });
    actorContext.setCommitIndex(commitIndex);
    leader = new Leader(actorContext);
    leader.getFollower(FOLLOWER_ID).setMatchIndex(-1);
    leader.getFollower(FOLLOWER_ID).setNextIndex(0);
    Map<String, String> leadersSnapshot = new HashMap<>();
    leadersSnapshot.put("1", "A");
    leadersSnapshot.put("2", "B");
    leadersSnapshot.put("3", "C");
    // set the snapshot variables in replicatedlog
    actorContext.getReplicatedLog().setSnapshotIndex(snapshotIndex);
    actorContext.getReplicatedLog().setSnapshotTerm(snapshotTerm);
    actorContext.getTermInformation().update(currentTerm, leaderActor.path().toString());
    ByteString bs = toByteString(leadersSnapshot);
    Snapshot snapshot = Snapshot.create(ByteState.of(bs.toByteArray()), Collections.<ReplicatedLogEntry>emptyList(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null);
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
    leader.handleMessage(leaderActor, new SendInstallSnapshot(snapshot, ByteSource.wrap(bs.toByteArray())));
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    followerActor.underlyingActor().clear();
    leader.handleMessage(followerActor, new InstallSnapshotReply(actorContext.getTermInformation().getCurrentTerm(), FOLLOWER_ID, -1, false));
    Uninterruptibles.sleepUninterruptibly(actorContext.getConfigParams().getHeartBeatInterval().toMillis(), TimeUnit.MILLISECONDS);
    leader.handleMessage(leaderActor, SendHeartBeat.INSTANCE);
    installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
}