Examples with ByteString org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString used on opensource projects

Example 81 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project atlasdb by palantir.

the class StreamTestWithHashStreamStore method deleteStreams.

/**
 * This should only be used from the cleanup tasks.
 */
void deleteStreams(Transaction t, final Set<Long> streamIds) {
    if (streamIds.isEmpty()) {
        return;
    }
    Set<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow> smRows = Sets.newHashSet();
    Multimap<StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow, StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn> shToDelete = HashMultimap.create();
    for (Long streamId : streamIds) {
        smRows.add(StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow.of(streamId));
    }
    StreamTestWithHashStreamMetadataTable table = tables.getStreamTestWithHashStreamMetadataTable(t);
    Map<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows);
    Set<StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow> streamValueToDelete = Sets.newHashSet();
    for (Entry<StreamTestWithHashStreamMetadataTable.StreamTestWithHashStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) {
        Long streamId = e.getKey().getId();
        long blocks = getNumberOfBlocksFromMetadata(e.getValue());
        for (long i = 0; i < blocks; i++) {
            streamValueToDelete.add(StreamTestWithHashStreamValueTable.StreamTestWithHashStreamValueRow.of(streamId, i));
        }
        ByteString streamHash = e.getValue().getHash();
        Sha256Hash hash = Sha256Hash.EMPTY;
        if (streamHash != com.google.protobuf.ByteString.EMPTY) {
            hash = new Sha256Hash(streamHash.toByteArray());
        } else {
            log.error("Empty hash for stream {}", streamId);
        }
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow hashRow = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxRow.of(hash);
        StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn column = StreamTestWithHashStreamHashAidxTable.StreamTestWithHashStreamHashAidxColumn.of(streamId);
        shToDelete.put(hashRow, column);
    }
    tables.getStreamTestWithHashStreamHashAidxTable(t).delete(shToDelete);
    tables.getStreamTestWithHashStreamValueTable(t).delete(streamValueToDelete);
    table.delete(smRows);
}

Example 82 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project controller by opendaylight.

the class FollowerTest method testInitialSyncUpWithHandleInstallSnapshotFollowedByAppendEntries.

@Test
public void testInitialSyncUpWithHandleInstallSnapshotFollowedByAppendEntries() {
    logStart("testInitialSyncUpWithHandleInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    context.setCommitIndex(-1);
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int offset = 0;
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    int chunkIndex = 1;
    InstallSnapshot lastInstallSnapshot = null;
    for (int i = 0; i < totalChunks; i++) {
        byte[] chunkData = getNextChunk(bsSnapshot, offset, chunkSize);
        lastInstallSnapshot = new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, chunkIndex, totalChunks);
        follower.handleMessage(leaderActor, lastInstallSnapshot);
        offset = offset + 50;
        lastIncludedIndex++;
        chunkIndex++;
    }
    FollowerInitialSyncUpStatus syncStatus = MessageCollectorActor.expectFirstMatching(followerActor, FollowerInitialSyncUpStatus.class);
    assertFalse(syncStatus.isInitialSyncDone());
    // Clear all the messages
    MessageCollectorActor.clearMessages(followerActor);
    context.setLastApplied(101);
    context.setCommitIndex(101);
    setLastLogEntry(context, 1, 101, new MockRaftActorContext.MockPayload(""));
    List<ReplicatedLogEntry> entries = Arrays.asList(newReplicatedLogEntry(2, 101, "foo"));
    // The new commitIndex is 101
    AppendEntries appendEntries = new AppendEntries(2, "leader", 101, 1, entries, 102, 101, (short) 0);
    follower.handleMessage(leaderActor, appendEntries);
    syncStatus = MessageCollectorActor.expectFirstMatching(followerActor, FollowerInitialSyncUpStatus.class);
    assertTrue(syncStatus.isInitialSyncDone());
}

Example 83 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project controller by opendaylight.

the class FollowerTest method testHandleInstallSnapshot.

/**
 * This test verifies that when InstallSnapshot is received by
 * the follower its applied correctly.
 */
@Test
public void testHandleInstallSnapshot() {
    logStart("testHandleInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    context.getTermInformation().update(1, "leader");
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int offset = 0;
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    int chunkIndex = 1;
    InstallSnapshot lastInstallSnapshot = null;
    for (int i = 0; i < totalChunks; i++) {
        byte[] chunkData = getNextChunk(bsSnapshot, offset, chunkSize);
        lastInstallSnapshot = new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, chunkIndex, totalChunks);
        follower.handleMessage(leaderActor, lastInstallSnapshot);
        offset = offset + 50;
        lastIncludedIndex++;
        chunkIndex++;
    }
    ApplySnapshot applySnapshot = MessageCollectorActor.expectFirstMatching(followerActor, ApplySnapshot.class);
    Snapshot snapshot = applySnapshot.getSnapshot();
    assertNotNull(lastInstallSnapshot);
    assertEquals("getLastIndex", lastInstallSnapshot.getLastIncludedIndex(), snapshot.getLastIndex());
    assertEquals("getLastIncludedTerm", lastInstallSnapshot.getLastIncludedTerm(), snapshot.getLastAppliedTerm());
    assertEquals("getLastAppliedIndex", lastInstallSnapshot.getLastIncludedIndex(), snapshot.getLastAppliedIndex());
    assertEquals("getLastTerm", lastInstallSnapshot.getLastIncludedTerm(), snapshot.getLastTerm());
    assertEquals("getState type", ByteState.class, snapshot.getState().getClass());
    Assert.assertArrayEquals("getState", bsSnapshot.toByteArray(), ((ByteState) snapshot.getState()).getBytes());
    assertEquals("getElectionTerm", 1, snapshot.getElectionTerm());
    assertEquals("getElectionVotedFor", "leader", snapshot.getElectionVotedFor());
    applySnapshot.getCallback().onSuccess();
    List<InstallSnapshotReply> replies = MessageCollectorActor.getAllMatching(leaderActor, InstallSnapshotReply.class);
    assertEquals("InstallSnapshotReply count", totalChunks, replies.size());
    chunkIndex = 1;
    for (InstallSnapshotReply reply : replies) {
        assertEquals("getChunkIndex", chunkIndex++, reply.getChunkIndex());
        assertEquals("getTerm", 1, reply.getTerm());
        assertEquals("isSuccess", true, reply.isSuccess());
        assertEquals("getFollowerId", context.getId(), reply.getFollowerId());
    }
    assertNull("Expected null SnapshotTracker", follower.getSnapshotTracker());
}

Example 84 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project controller by opendaylight.

the class LeaderTest method testHandleInstallSnapshotReplyWithInvalidChunkIndex.

@Test
public void testHandleInstallSnapshotReplyWithInvalidChunkIndex() throws Exception {
    logStart("testHandleInstallSnapshotReplyWithInvalidChunkIndex");
    MockRaftActorContext actorContext = createActorContextWithFollower();
    final int commitIndex = 3;
    final int snapshotIndex = 2;
    final int snapshotTerm = 1;
    final int currentTerm = 2;
    actorContext.setConfigParams(new DefaultConfigParamsImpl() {

        @Override
        public int getSnapshotChunkSize() {
            return 50;
        }
    });
    actorContext.setCommitIndex(commitIndex);
    leader = new Leader(actorContext);
    leader.getFollower(FOLLOWER_ID).setMatchIndex(-1);
    leader.getFollower(FOLLOWER_ID).setNextIndex(0);
    Map<String, String> leadersSnapshot = new HashMap<>();
    leadersSnapshot.put("1", "A");
    leadersSnapshot.put("2", "B");
    leadersSnapshot.put("3", "C");
    // set the snapshot variables in replicatedlog
    actorContext.getReplicatedLog().setSnapshotIndex(snapshotIndex);
    actorContext.getReplicatedLog().setSnapshotTerm(snapshotTerm);
    actorContext.getTermInformation().update(currentTerm, leaderActor.path().toString());
    ByteString bs = toByteString(leadersSnapshot);
    Snapshot snapshot = Snapshot.create(ByteState.of(bs.toByteArray()), Collections.<ReplicatedLogEntry>emptyList(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null);
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
    leader.handleMessage(leaderActor, new SendInstallSnapshot(snapshot, ByteSource.wrap(bs.toByteArray())));
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    followerActor.underlyingActor().clear();
    leader.handleMessage(followerActor, new InstallSnapshotReply(actorContext.getTermInformation().getCurrentTerm(), FOLLOWER_ID, -1, false));
    Uninterruptibles.sleepUninterruptibly(actorContext.getConfigParams().getHeartBeatInterval().toMillis(), TimeUnit.MILLISECONDS);
    leader.handleMessage(leaderActor, SendHeartBeat.INSTANCE);
    installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
}

Example 85 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project controller by opendaylight.

the class LeaderTest method testSendAppendEntriesOnAnInProgressInstallSnapshot.

@Test
public void testSendAppendEntriesOnAnInProgressInstallSnapshot() throws Exception {
    logStart("testSendAppendEntriesOnAnInProgressInstallSnapshot");
    final MockRaftActorContext actorContext = createActorContextWithFollower();
    Map<String, String> leadersSnapshot = new HashMap<>();
    leadersSnapshot.put("1", "A");
    leadersSnapshot.put("2", "B");
    leadersSnapshot.put("3", "C");
    // clears leaders log
    actorContext.getReplicatedLog().removeFrom(0);
    final int commitIndex = 3;
    final int snapshotIndex = 2;
    final int snapshotTerm = 1;
    // set the snapshot variables in replicatedlog
    actorContext.getReplicatedLog().setSnapshotIndex(snapshotIndex);
    actorContext.getReplicatedLog().setSnapshotTerm(snapshotTerm);
    actorContext.setCommitIndex(commitIndex);
    // set follower timeout to 2 mins, helps during debugging
    actorContext.setConfigParams(new MockConfigParamsImpl(120000L, 10));
    leader = new Leader(actorContext);
    leader.getFollower(FOLLOWER_ID).setMatchIndex(-1);
    leader.getFollower(FOLLOWER_ID).setNextIndex(0);
    // update follower timestamp
    leader.markFollowerActive(FOLLOWER_ID);
    ByteString bs = toByteString(leadersSnapshot);
    leader.setSnapshotHolder(new SnapshotHolder(Snapshot.create(ByteState.of(bs.toByteArray()), Collections.<ReplicatedLogEntry>emptyList(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null), ByteSource.wrap(bs.toByteArray())));
    LeaderInstallSnapshotState fts = new LeaderInstallSnapshotState(actorContext.getConfigParams().getSnapshotChunkSize(), leader.logName());
    fts.setSnapshotBytes(ByteSource.wrap(bs.toByteArray()));
    leader.getFollower(FOLLOWER_ID).setLeaderInstallSnapshotState(fts);
    // send first chunk and no InstallSnapshotReply received yet
    fts.getNextChunk();
    fts.incrementChunkIndex();
    Uninterruptibles.sleepUninterruptibly(actorContext.getConfigParams().getHeartBeatInterval().toMillis(), TimeUnit.MILLISECONDS);
    leader.handleMessage(leaderActor, SendHeartBeat.INSTANCE);
    AppendEntries ae = MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    assertTrue("AppendEntries should be sent with empty entries", ae.getEntries().isEmpty());
    // InstallSnapshotReply received
    fts.markSendStatus(true);
    leader.handleMessage(leaderActor, SendHeartBeat.INSTANCE);
    InstallSnapshot is = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(commitIndex, is.getLastIncludedIndex());
}