Examples with MockPartitionId com.github.ambry.clustermap.MockPartitionId used on opensource projects

Example 11 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class CloudStorageManagerTest method scheduleNextForCompactionTest.

/**
 * Test {@code CloudStorageManager#scheduleNextForCompaction}
 */
@Test
public void scheduleNextForCompactionTest() throws IOException {
    CloudStorageManager cloudStorageManager = createNewCloudStorageManager();
    try {
        cloudStorageManager.scheduleNextForCompaction(new MockPartitionId());
        Assert.fail("CloudStorageManager scheduleNextForCompaction should throw unimplemented exception");
    } catch (UnsupportedOperationException e) {
    }
}

Example 12 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class CosmosDataAccessorTest method setup.

@Before
public void setup() {
    mockumentClient = mock(AsyncDocumentClient.class);
    byte dataCenterId = 66;
    short accountId = 101;
    short containerId = 5;
    PartitionId partitionId = new MockPartitionId();
    blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
    blobMetadata = new CloudBlobMetadata(blobId, System.currentTimeMillis(), Utils.Infinite_Time, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
    azureMetrics = new AzureMetrics(new MetricRegistry());
    VcrMetrics vcrMetrics = new VcrMetrics(new MetricRegistry());
    cosmosAccessor = new CosmosDataAccessor(mockumentClient, "ambry/metadata", "ambry/deletedContainer", vcrMetrics, azureMetrics);
}

Example 13 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class MessageInfoAndMetadataListSerDeTest method testSerDe.

@Test
public void testSerDe() throws Exception {
    MockClusterMap mockMap = new MockClusterMap();
    MockPartitionId partitionId = new MockPartitionId();
    short[] accountIds = { 100, 101, 102, 103 };
    short[] containerIds = { 10, 11, 12, 13 };
    boolean[] isDeletedVals = { false, true, false, true };
    boolean[] isTtlUpdatedVals = { true, false, false, true };
    boolean[] isUndeletedVals = { true, false, false, true };
    short[] lifeVersionVals = { 1, 2, 3, 4 };
    Long[] crcs = { null, 100L, Long.MIN_VALUE, Long.MAX_VALUE };
    StoreKey[] keys = { new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[0], containerIds[0], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[1], containerIds[1], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[2], containerIds[2], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[3], containerIds[3], partitionId, false, BlobId.BlobDataType.DATACHUNK) };
    long[] blobSizes = { 1024, 2048, 4096, 8192 };
    long[] times = { SystemTime.getInstance().milliseconds(), SystemTime.getInstance().milliseconds() - 1, SystemTime.getInstance().milliseconds() + TimeUnit.SECONDS.toMillis(5), Utils.Infinite_Time };
    MessageMetadata[] messageMetadata = new MessageMetadata[4];
    messageMetadata[0] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(100)));
    messageMetadata[1] = new MessageMetadata(null);
    messageMetadata[2] = null;
    messageMetadata[3] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(200)));
    List<MessageInfo> messageInfoList = new ArrayList<>(4);
    List<MessageMetadata> messageMetadataList = new ArrayList<>(4);
    for (int i = 0; i < 4; i++) {
        messageInfoList.add(new MessageInfo(keys[i], blobSizes[i], isDeletedVals[i], isTtlUpdatedVals[i], isUndeletedVals[i], times[i], crcs[i], accountIds[i], containerIds[i], times[i], lifeVersionVals[i]));
        messageMetadataList.add(messageMetadata[i]);
    }
    // Serialize and then deserialize
    MessageInfoAndMetadataListSerde messageInfoAndMetadataListSerde = new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, serDeVersion);
    ByteBuf buffer = Unpooled.buffer(messageInfoAndMetadataListSerde.getMessageInfoAndMetadataListSize());
    messageInfoAndMetadataListSerde.serializeMessageInfoAndMetadataList(buffer);
    if (serDeVersion >= MessageInfoAndMetadataListSerde.VERSION_5) {
        // should fail if the wrong version is provided
        try {
            MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, (short) (serDeVersion - 1));
            Assert.fail("Should have failed to deserialize");
        } catch (IllegalArgumentException e) {
        // expected. Nothing to do
        }
    }
    buffer.readerIndex(0);
    MessageInfoAndMetadataListSerde messageInfoAndMetadataList = MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, serDeVersion);
    // Verify
    List<MessageInfo> responseMessageInfoList = messageInfoAndMetadataList.getMessageInfoList();
    List<MessageMetadata> responseMessageMetadataList = messageInfoAndMetadataList.getMessageMetadataList();
    Assert.assertEquals(4, responseMessageInfoList.size());
    Assert.assertEquals(4, responseMessageMetadataList.size());
    for (int i = 0; i < 4; i++) {
        assertMessageInfoEquality(messageInfoList.get(i), responseMessageInfoList.get(i));
        assertMessageMetadataEquality(messageMetadataList.get(i), responseMessageMetadataList.get(i));
    }
}

Example 14 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class LeaderBasedReplicationTest method onRoutingTableUpdateCallbackTest.

/**
 * Test cluster map change callback in {@link ReplicationManager} for routing table updates.
 * Test setup: When creating partitions, have one replica in LEADER state and rest in STANDBY states on each data center and
 * later switch the states of replicas (LEADER to STANDBY and STANDBY to LEADER) on one of the DCs during the test
 * Test condition: When replication manager receives onRoutingTableUpdate() indication after the remote replica states were updated,
 * map of partition to peer leader replicas stored in replication manager should be updated correctly
 * @throws Exception
 */
@Test
public void onRoutingTableUpdateCallbackTest() throws Exception {
    Pair<StorageManager, ReplicationManager> managers = createStorageManagerAndReplicationManager(clusterMap, clusterMapConfig, mockHelixParticipant);
    StorageManager storageManager = managers.getFirst();
    MockReplicationManager replicationManager = (MockReplicationManager) managers.getSecond();
    replicationManager.start();
    // Trigger PartitionStateChangeListener callback to replication manager to notify that a local replica state has changed from STANDBY to LEADER
    List<? extends ReplicaId> replicaIds = clusterMap.getReplicaIds(replicationManager.dataNodeId);
    for (ReplicaId replicaId : replicaIds) {
        MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
        if (mockReplicaId.getReplicaState() == ReplicaState.LEADER) {
            MockPartitionId existingPartition = (MockPartitionId) mockReplicaId.getPartitionId();
            mockHelixParticipant.onPartitionBecomeLeaderFromStandby(existingPartition.toPathString());
            // verify that map of peerLeaderReplicasByPartition in PartitionLeaderInfo is updated correctly
            Set<ReplicaId> peerLeaderReplicasInPartitionLeaderInfo = replicationManager.leaderBasedReplicationAdmin.getLeaderPartitionToPeerLeaderReplicas().get(existingPartition.toPathString());
            Set<ReplicaId> peerLeaderReplicasInClusterMap = new HashSet<>(existingPartition.getReplicaIdsByState(ReplicaState.LEADER, null));
            peerLeaderReplicasInClusterMap.remove(mockReplicaId);
            assertThat("Mismatch in list of leader peer replicas stored by partition in replication manager with cluster map", peerLeaderReplicasInPartitionLeaderInfo, is(peerLeaderReplicasInClusterMap));
            // Switch the LEADER/STANDBY states for remote replicas on one of the remote data centers
            ReplicaId peerLeaderReplica = peerLeaderReplicasInClusterMap.iterator().next();
            ReplicaId peerStandByReplica = existingPartition.getReplicaIdsByState(ReplicaState.STANDBY, peerLeaderReplica.getDataNodeId().getDatacenterName()).get(0);
            existingPartition.setReplicaState(peerLeaderReplica, ReplicaState.STANDBY);
            existingPartition.setReplicaState(peerStandByReplica, ReplicaState.LEADER);
            // Trigger routing table change callback to replication manager
            ClusterMapChangeListener clusterMapChangeListener = clusterMap.getClusterMapChangeListener();
            clusterMapChangeListener.onRoutingTableChange();
            // verify that new remote leader is reflected in the peerLeaderReplicasByPartition map
            peerLeaderReplicasInPartitionLeaderInfo = replicationManager.leaderBasedReplicationAdmin.getLeaderPartitionToPeerLeaderReplicas().get(existingPartition.toPathString());
            peerLeaderReplicasInClusterMap = new HashSet<>(existingPartition.getReplicaIdsByState(ReplicaState.LEADER, null));
            peerLeaderReplicasInClusterMap.remove(mockReplicaId);
            assertThat("Mismatch in map of peer leader replicas stored by partition in replication manager with cluster map after routing table update", peerLeaderReplicasInPartitionLeaderInfo, is(peerLeaderReplicasInClusterMap));
        }
    }
    storageManager.shutdown();
}

Example 15 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class LeaderBasedReplicationTest method replicaThreadLeaderBasedReplicationTokenCatchUpForStandbyToLeaderTest.

/**
 * Test leader based replication to verify remote token is caught up for standby replicas and updated token is used
 * when their state transitions to leader.
 * @throws Exception
 */
@Test
public void replicaThreadLeaderBasedReplicationTokenCatchUpForStandbyToLeaderTest() throws Exception {
    /*
      Setup:
      we have 3 nodes that have replicas belonging to same partitions:
      a) localNode (local node that hosts partitions)
      b) remoteNodeInLocalDC (remote node in local data center that shares the partitions)
      c) remoteNodeInRemoteDC (remote node in remote data center that shares the partitions)

      Each node have few of its partitions as leaders and others are standby. They are randomly assigned during creation
      of replicas for mock partitions.
     */
    Map<DataNodeId, MockHost> hosts = new HashMap<>();
    hosts.put(remoteNodeInLocalDC, remoteHostInLocalDC);
    hosts.put(remoteNodeInRemoteDC, remoteHostInRemoteDC);
    int batchSize = 5;
    ConnectionPool mockConnectionPool = new MockConnectionPool(hosts, clusterMap, batchSize);
    Pair<StorageManager, ReplicationManager> managers = createStorageManagerAndReplicationManager(clusterMap, clusterMapConfig, mockHelixParticipant, mockConnectionPool);
    StorageManager storageManager = managers.getFirst();
    MockReplicationManager replicationManager = (MockReplicationManager) managers.getSecond();
    // set mock local stores on all remoteReplicaInfos which will used during replication.
    for (PartitionId partitionId : replicationManager.partitionToPartitionInfo.keySet()) {
        localHost.addStore(partitionId, null);
        Store localStore = localHost.getStore(partitionId);
        localStore.start();
        List<RemoteReplicaInfo> remoteReplicaInfos = replicationManager.partitionToPartitionInfo.get(partitionId).getRemoteReplicaInfos();
        remoteReplicaInfos.forEach(remoteReplicaInfo -> remoteReplicaInfo.setLocalStore(localStore));
    }
    // get remote replicas and replica thread for remote host on local datacenter
    ReplicaThread intraColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInLocalDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForLocalDC = intraColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInLocalDC);
    // get remote replicas and replica thread for remote host on remote datacenter
    ReplicaThread crossColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInRemoteDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForRemoteDC = crossColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInRemoteDC);
    // mock helix transition state from standby to leader for local leader partitions
    List<? extends ReplicaId> replicaIds = clusterMap.getReplicaIds(replicationManager.dataNodeId);
    for (ReplicaId replicaId : replicaIds) {
        MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
        if (mockReplicaId.getReplicaState() == ReplicaState.LEADER) {
            MockPartitionId mockPartitionId = (MockPartitionId) replicaId.getPartitionId();
            mockHelixParticipant.onPartitionBecomeLeaderFromStandby(mockPartitionId.toPathString());
        }
    }
    // Add put messages to all partitions on remoteHost1 and remoteHost2
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    for (PartitionId partitionId : partitionIds) {
        // add batchSize messages to the remoteHost1 and remote host 2 from which local host will replicate.
        addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHostInLocalDC, remoteHostInRemoteDC), batchSize + batchSize);
    }
    // Choose partitions that are leaders on both local and remote nodes
    Set<ReplicaId> leaderReplicasOnLocalAndRemoteNodes = new HashSet<>();
    // Track a standby replica which has leader partition on remote node. We will update the state of replica to leader after one cycle of replication
    // and verify that replication resumes from remote token.
    MockReplicaId localStandbyReplicaWithLeaderPartitionOnRemoteNode = null;
    List<? extends ReplicaId> localReplicas = clusterMap.getReplicaIds(replicationManager.dataNodeId);
    List<? extends ReplicaId> remoteReplicas = clusterMap.getReplicaIds(remoteNodeInRemoteDC);
    for (int i = 0; i < localReplicas.size(); i++) {
        MockReplicaId localReplica = (MockReplicaId) localReplicas.get(i);
        MockReplicaId remoteReplica = (MockReplicaId) remoteReplicas.get(i);
        if (localReplica.getReplicaState() == ReplicaState.LEADER && remoteReplica.getReplicaState() == ReplicaState.LEADER) {
            leaderReplicasOnLocalAndRemoteNodes.add(remoteReplicas.get(i));
        }
        if (localReplica.getReplicaState() == ReplicaState.STANDBY && remoteReplica.getReplicaState() == ReplicaState.LEADER && localStandbyReplicaWithLeaderPartitionOnRemoteNode == null) {
            localStandbyReplicaWithLeaderPartitionOnRemoteNode = localReplica;
        }
    }
    // replicate with remote node in remote DC
    crossColoReplicaThread.replicate();
    // missing messages are not fetched yet.
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        if (leaderReplicasOnLocalAndRemoteNodes.contains(remoteReplicaInfo.getReplicaId())) {
            assertEquals("remote token mismatch for leader replicas", ((MockFindToken) remoteReplicaInfo.getToken()).getIndex(), batchSize - 1);
        } else {
            assertEquals("remote token should not move forward for standby replicas until missing keys are fetched", ((MockFindToken) remoteReplicaInfo.getToken()).getIndex(), 0);
        }
    }
    // Replicate with remote node in local dc
    intraColoReplicaThread.replicate();
    // verify that remote token will be moved for all intra-colo replicas with token index = batchSize-1
    for (RemoteReplicaInfo replicaInfo : remoteReplicaInfosForLocalDC) {
        assertEquals("mismatch in remote token set for intra colo replicas", ((MockFindToken) replicaInfo.getToken()).getIndex(), batchSize - 1);
    }
    // process missing keys for cross colo replicas from previous metadata exchange
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        crossColoReplicaThread.processMissingKeysFromPreviousMetadataResponse(remoteReplicaInfo);
    }
    // as missing keys must now be obtained via intra-dc replication
    for (RemoteReplicaInfo replicaInfo : remoteReplicaInfosForRemoteDC) {
        assertEquals("mismatch in remote token set for inter colo replicas", ((MockFindToken) replicaInfo.getToken()).getIndex(), batchSize - 1);
    }
    // If we have a local standby replica with leader partition on remote node, change its state to leader
    if (localStandbyReplicaWithLeaderPartitionOnRemoteNode != null) {
        MockPartitionId mockPartitionId = (MockPartitionId) localStandbyReplicaWithLeaderPartitionOnRemoteNode.getPartitionId();
        mockHelixParticipant.onPartitionBecomeLeaderFromStandby(mockPartitionId.toPathString());
    }
    // Trigger replication again with remote node in remote DC
    crossColoReplicaThread.replicate();
    // leader localStandbyReplicaWithLeaderPartitionOnRemoteNode whose replication should be have resumed with remote token index = batchSize-1
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        if (leaderReplicasOnLocalAndRemoteNodes.contains(remoteReplicaInfo.getReplicaId()) || (remoteReplicaInfo.getLocalReplicaId().equals(localStandbyReplicaWithLeaderPartitionOnRemoteNode))) {
            assertEquals("remote token mismatch for leader replicas", ((MockFindToken) remoteReplicaInfo.getToken()).getIndex(), batchSize * 2 - 2);
        } else {
            assertEquals("remote token should not move forward for standby replicas until missing keys are fetched", ((MockFindToken) remoteReplicaInfo.getToken()).getIndex(), batchSize - 1);
        }
    }
    // Trigger replication again with remote node in local DC
    intraColoReplicaThread.replicate();
    // verify that remote token is moved forward for all intra-colo replicas.
    for (RemoteReplicaInfo replicaInfo : remoteReplicaInfosForLocalDC) {
        assertEquals("mismatch in remote token set for intra colo replicas", ((MockFindToken) replicaInfo.getToken()).getIndex(), batchSize * 2 - 2);
    }
    // process missing keys for cross colo replicas from previous metadata exchange
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        crossColoReplicaThread.processMissingKeysFromPreviousMetadataResponse(remoteReplicaInfo);
    }
    // via intra-dc replication.
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        assertEquals("mismatch in remote token set for intra colo replicas", ((MockFindToken) remoteReplicaInfo.getToken()).getIndex(), batchSize * 2 - 2);
    }
    storageManager.shutdown();
}