Examples with MockPartitionId com.github.ambry.clustermap.MockPartitionId used on opensource projects

Example 51 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class OperationTrackerTest method getOperationWithReplicaStateTest.

/**
 * Test GET operation is able to try on OFFLINE replicas if routerOperationTrackerIncludeDownReplicas is true.
 */
@Test
public void getOperationWithReplicaStateTest() {
    assumeTrue(replicasStateEnabled);
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1")));
    mockPartition = new MockPartitionId();
    for (ReplicaState state : EnumSet.of(ReplicaState.BOOTSTRAP, ReplicaState.STANDBY, ReplicaState.LEADER, ReplicaState.INACTIVE, ReplicaState.OFFLINE)) {
        populateReplicaList(1, state);
    }
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    // 1. include down replicas (OFFLINE replicas are eligible for GET)
    OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
    // make sure 4 requests fails and last one succeeds. (This is to verify operation tracker adds offline replica into replica pool as well)
    ReplicaId inflightReplica;
    for (int i = 0; i < 4; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        // verify that the first 4 replicas are not OFFLINE replica. (OFFLINE replica should be added to the end of queue)
        assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
        ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
        assertFalse("Operation should not complete", ot.isDone());
    }
    sendRequests(ot, 1, false);
    inflightReplica = inflightReplicas.poll();
    assertEquals("The last replica should be OFFLINE", ReplicaState.OFFLINE, mockPartition.replicaAndState.get(inflightReplica));
    ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should be done", ot.isDone());
    // 2. exclude down replicas
    repetitionTracker.clear();
    ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, false);
    for (int i = 0; i < 4; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        // verify that none of these replicas is OFFLINE replica.
        assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
        ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
        if (i < 3) {
            assertFalse("Operation should not complete", ot.isDone());
        } else {
            assertTrue("Operation should complete", ot.isDone());
        }
    }
}

Example 52 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class OperationTrackerTest method sendCrossColoRequestToDcWithMostReplicasTest.

/**
 * Test the case where originating dc name is null and operation tracker will choose dc with most replicas to send
 * cross-colo request.
 * local dc: one replica;
 * remote dc1: three replicas;
 * remote dc2: one replica;
 */
@Test
public void sendCrossColoRequestToDcWithMostReplicasTest() {
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    // set up one node per data center for testing
    MockDataNodeId localDcNode = new MockDataNodeId(portList, mountPaths, "dc-0");
    MockDataNodeId remoteDc1Node = new MockDataNodeId(portList, mountPaths, "dc-1");
    MockDataNodeId remoteDc2Node = new MockDataNodeId(portList, mountPaths, "dc-2");
    mockPartition = new MockPartitionId();
    localDcName = localDcNode.getDatacenterName();
    originatingDcName = null;
    chooseDcWithMostReplicas = true;
    mockClusterMap = new MockClusterMap(false, Arrays.asList(localDcNode, remoteDc1Node, remoteDc2Node), 1, Collections.singletonList(mockPartition), localDcName);
    populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(localDcNode));
    populateReplicaList(3, ReplicaState.STANDBY, Collections.singletonList(remoteDc1Node));
    populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(remoteDc2Node));
    OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
    // make local replica return Not_Found
    sendRequests(ot, 1, false);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.NOT_FOUND);
    ReplicaId inflightReplica;
    // next, operation tracker should send request to dc-1 as it has most replicas
    for (int i = 0; i < 3; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        assertEquals("The request should be sent to dc-1 with most replicas", "dc-1", inflightReplica.getDataNodeId().getDatacenterName());
        // we deliberately make all replicas in dc-1 return Not_Found to verify that operation won't terminate on not found
        ot.onResponse(inflightReplica, TrackedRequestFinalState.NOT_FOUND);
        assertFalse("Operation should not be done yet", ot.isDone());
    }
    // the last request should go to dc-2 and this time we make it succeed
    sendRequests(ot, 1, false);
    inflightReplica = inflightReplicas.poll();
    assertEquals("The request should be sent to dc-2", "dc-2", inflightReplica.getDataNodeId().getDatacenterName());
    ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should succeed", ot.hasSucceeded());
    assertTrue("Operation should be done", ot.isDone());
}

Example 53 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class OperationTrackerTest method downReplicasOrderingTest.

/**
 * Test to ensure that replicas that are down are also returned by the operation tracker, but they are
 * ordered after the healthy replicas.
 */
@Test
public void downReplicasOrderingTest() {
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = new ArrayList<>();
    datanodes.add(new MockDataNodeId(portList, mountPaths, "dc-0"));
    datanodes.add(new MockDataNodeId(portList, mountPaths, "dc-1"));
    mockPartition = new MockPartitionId();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), datanodes.get(0).getDatacenterName());
    int replicaCount = 6;
    populateReplicaList(replicaCount, ReplicaState.STANDBY);
    // Test scenarios with various number of replicas down
    for (int i = 0; i < replicaCount; i++) {
        testReplicaDown(replicaCount, i);
    }
}

Example 54 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class Http2NetworkClientTest method putGetTest.

@Test
public void putGetTest() throws Exception {
    MockClusterMap clusterMap = http2Cluster.getClusterMap();
    DataNodeId dataNodeId = http2Cluster.getGeneralDataNode();
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    SSLFactory sslFactory = new NettySslHttp2Factory(clientSSLConfig);
    Http2NetworkClient networkClient = new Http2NetworkClient(new Http2ClientMetrics(new MetricRegistry()), new Http2ClientConfig(new VerifiableProperties(new Properties())), sslFactory, eventLoopGroup);
    // Put a blob
    int blobSize = 1024 * 1024;
    byte[] usermetadata = new byte[1000];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", accountId, containerId, false);
    TestUtils.RANDOM.nextBytes(usermetadata);
    TestUtils.RANDOM.nextBytes(data);
    List<? extends PartitionId> partitionIds = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
    BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
    // put blob 1
    PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(usermetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
    RequestInfo request = new RequestInfo(dataNodeId.getHostname(), new Port(dataNodeId.getHttp2Port(), PortType.HTTP2), putRequest, clusterMap.getReplicaIds(dataNodeId).get(0), null);
    List<ResponseInfo> responseInfos = networkClient.sendAndPoll(Collections.singletonList(request), new HashSet<>(), 300);
    long startTime = SystemTime.getInstance().milliseconds();
    while (responseInfos.size() == 0) {
        responseInfos = networkClient.sendAndPoll(Collections.EMPTY_LIST, new HashSet<>(), 300);
        if (SystemTime.getInstance().milliseconds() - startTime >= 6000) {
            fail("Network Client no reponse and timeout.");
        }
        Thread.sleep(30);
    }
    assertEquals("Should be only one response", 1, responseInfos.size());
    DataInputStream dis = new NettyByteBufDataInputStream(responseInfos.get(0).content());
    PutResponse putResponse = PutResponse.readFrom(dis);
    assertEquals("No error expected.", ServerErrorCode.No_Error, putResponse.getError());
    // Get the blob
    // get blob properties
    ArrayList<BlobId> ids = new ArrayList<BlobId>();
    MockPartitionId partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
    ids.add(blobId1);
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partition, ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest = new GetRequest(1, "http2-clientid", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
    request = new RequestInfo(dataNodeId.getHostname(), new Port(dataNodeId.getHttp2Port(), PortType.HTTP2), getRequest, clusterMap.getReplicaIds(dataNodeId).get(0), null);
    responseInfos = networkClient.sendAndPoll(Collections.singletonList(request), new HashSet<>(), 300);
    startTime = SystemTime.getInstance().milliseconds();
    while (responseInfos.size() == 0) {
        responseInfos = networkClient.sendAndPoll(Collections.EMPTY_LIST, new HashSet<>(), 300);
        if (SystemTime.getInstance().milliseconds() - startTime >= 3000) {
            fail("Network Client no response and timeout.");
        }
        Thread.sleep(30);
    }
    assertEquals("Should be only one response", 1, responseInfos.size());
    dis = new NettyByteBufDataInputStream(responseInfos.get(0).content());
    GetResponse resp = GetResponse.readFrom(dis, clusterMap);
    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
    // verify BlobProperties
    BlobProperties propertyOutput = blobAll.getBlobInfo().getBlobProperties();
    assertEquals(blobSize, propertyOutput.getBlobSize());
    assertEquals("serviceid1", propertyOutput.getServiceId());
    assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
    assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
    // verify UserMetadata
    byte[] userMetadataOutput = blobAll.getBlobInfo().getUserMetadata();
    assertArrayEquals(usermetadata, userMetadataOutput);
    // verify content
    byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
    assertArrayEquals("Content mismatch.", data, actualBlobData);
}

Example 55 with MockPartitionId

use of com.github.ambry.clustermap.MockPartitionId in project ambry by linkedin.

the class UndeleteOperationTrackerTest method successWithSufficientEligibleHostsTest.

/**
 * Tests when not all hosts are eligible not enough hosts are.
 */
@Test
public void successWithSufficientEligibleHostsTest() {
    assumeTrue(replicasStateEnabled);
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1"), new MockDataNodeId(portList, mountPaths, "dc-2"), new MockDataNodeId(portList, mountPaths, "dc-3")));
    mockPartition = new MockPartitionId();
    populateReplicaList(8, ReplicaState.STANDBY);
    populateReplicaList(4, ReplicaState.INACTIVE);
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    UndeleteOperationTracker tracker = getOperationTracker(2);
    // Now we need to set all the response to be success
    for (int i = 0; i < 4; i++) {
        sendRequests(tracker, 2);
        assertFalse("Operation should not have failed", tracker.hasFailed());
        assertFalse("Operation should not have succeeded", tracker.hasSucceeded());
        assertFalse("Operation should not be done", tracker.isDone());
        for (int j = 0; j < 2; j++) {
            tracker.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
        }
    }
    assertFalse("Operation should not have failed", tracker.hasFailed());
    assertTrue("Operation should have succeeded", tracker.hasSucceeded());
    assertTrue("Operation should be done", tracker.isDone());
}