Examples with Port com.github.ambry.network.Port used on opensource projects

Example 16 with Port

use of com.github.ambry.network.Port in project ambry by linkedin.

the class OperationTrackerTest method deleteTtlUpdateWithReplicaStateTest.

/**
 * Test delete/ttlUpdate operation when replicasStateEnabled is enabled/disabled.
 * local dc: 2 STANDBY and 1 INACTIVE; remote dc: 2 STANDBY and 1 INACTIVE
 * 1. Issue 3 requests in parallel
 * 2. Make 2 requests fail
 * 3. Issue 1 requests (replicaState enabled tracker only has 4 eligible replicas)
 * 4. Make 1 succeed and 1 fail (replicaState enabled tracker should fail)
 * 5. Make remaining requests succeed, this only applies for tracker with replicaState disabled and operation should succeed.
 */
@Test
public void deleteTtlUpdateWithReplicaStateTest() {
    assumeTrue(operationTrackerType.equals(SIMPLE_OP_TRACKER));
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    // set up one node per data center for testing
    datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1")));
    mockPartition = new MockPartitionId();
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    // put two STANDBY replicas in each data center (note that "populateReplicaList" method alternatively distributes
    // the replica, so here we set 4 for two dc in total)
    populateReplicaList(4, ReplicaState.STANDBY);
    // put one INACTIVE in each data center
    populateReplicaList(2, ReplicaState.INACTIVE);
    // test both delete and Ttl Update cases
    for (RouterOperation operation : EnumSet.of(RouterOperation.DeleteOperation, RouterOperation.TtlUpdateOperation)) {
        repetitionTracker.clear();
        OperationTracker ot = getOperationTracker(true, 1, 2, operation, true);
        // issue delete/ttlUpdate requests to 2 local replica and 1 remote replica
        sendRequests(ot, 3, false);
        // make 2 requests fail and send requests again
        for (int i = 0; i < 2; ++i) {
            ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
        }
        // for replicaState enabled operation tracker, only 1 eligible replica left, so numRequestsExpected = 1
        sendRequests(ot, replicasStateEnabled ? 1 : 2, false);
        // make 1 requests fail and 1 request succeed then replicaState enabled operation tracker should fail
        ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
        ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
        if (replicasStateEnabled) {
            assertFalse("Operation should fail", ot.hasSucceeded());
        } else {
            // if replicasStateEnabled = false, operation tracker is able to succeed after 1 more request succeed
            ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
            assertTrue("Operation should succeed", ot.hasSucceeded());
        }
        assertTrue("Operation should be done", ot.isDone());
    }
}

Example 17 with Port

use of com.github.ambry.network.Port in project ambry by linkedin.

the class OperationTrackerTest method localPutWithReplicaStateTest.

/**
 * Test put operation in local dc when replicasStateEnabled is enabled/disabled.
 * Test steps:
 * Case1: Only 2 STANDBY replicas in local dc;
 *        Make 1 succeed and the other fail (both current and replicaState enabled tracker should fail)
 * Case2: 2 STANDBY, 1 INACTIVE replicas in local dc
 *        Make 1 fail and 2 succeed (replicaState enabled operation tracker should fail)
 * Case3: 1 LEADER, 4 STANDBY and 1 INACTIVE in local dc
 *        Make 3 succeed and 2 fail (replicaState enabled operation tracker should fail)
 */
@Test
public void localPutWithReplicaStateTest() {
    assumeTrue(operationTrackerType.equals(SIMPLE_OP_TRACKER));
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = Collections.singletonList(new MockDataNodeId(portList, mountPaths, "dc-0"));
    mockPartition = new MockPartitionId();
    // test that if there are only 2 eligible replicas, the success target should use routerConfig.routerPutSuccessTarget
    populateReplicaList(2, ReplicaState.STANDBY);
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    OperationTracker ot = getOperationTracker(true, 1, 2, RouterOperation.PutOperation, true);
    assertFalse("Operation should not have been done.", ot.isDone());
    sendRequests(ot, 2, false);
    // make one requests succeed, the other fail
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
    assertFalse("Operation should fail", ot.hasSucceeded());
    assertTrue("Operation should be done", ot.isDone());
    // add one more replica in INACTIVE state, now we have 2 STANDBY and 1 INACTIVE replicas
    populateReplicaList(1, ReplicaState.INACTIVE);
    repetitionTracker.clear();
    ot = getOperationTracker(true, 1, 3, RouterOperation.PutOperation, true);
    // issue PUT request
    sendRequests(ot, replicasStateEnabled ? 2 : 3, false);
    // make first request fail and rest requests succeed
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
    while (!inflightReplicas.isEmpty()) {
        ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
    }
    if (replicasStateEnabled) {
        assertFalse("Operation should fail because only 2 replicas eligible and 1 has failed", ot.hasSucceeded());
    } else {
        assertTrue("Operation should succeed when there are 2 success", ot.hasSucceeded());
    }
    // add three more replicas: one in LEADER state, the other two in STANDBY state
    populateReplicaList(1, ReplicaState.LEADER);
    populateReplicaList(2, ReplicaState.STANDBY);
    // now we have 6 replicas: 1 LEADER, 4 STANDBY and 1 INACTIVE. Number of eligible replicas = 1 + 4 = 5
    repetitionTracker.clear();
    ot = getOperationTracker(true, 1, 5, RouterOperation.PutOperation, true);
    // issue PUT request, parallelism should be 5 when replicaState is enabled.
    sendRequests(ot, 5, false);
    // remaining test is for replicaState enabled operation tracker
    if (replicasStateEnabled) {
        // make first 3 requests succeed
        for (int i = 0; i < 3; i++) {
            ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
            // success target should be 4 when replicaState is enabled for operation tracker, so operation is not done yet
            // even though it has succeeded on 3 replicas.
            assertFalse("Operation should not be done", ot.isDone());
        }
        // make last 2 requests fail, then operation should be done and result should be failure
        for (int i = 0; i < 2; i++) {
            ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
        }
        assertFalse("Operation should fail", ot.hasSucceeded());
        assertTrue("Operation should be done", ot.isDone());
    }
}

Example 18 with Port

use of com.github.ambry.network.Port in project ambry by linkedin.

the class OperationTrackerTest method initializeWithCloudDcs.

/**
 * Initialize 4 DCs, 2 disk datacenters, 2 cloud datacenters. Each disk datacenter has 3 replicas, and each cloud
 * datacenter has 1 replica.
 * @param makeCloudDcLocal {@code true} to make the local datacenter one of the cloud datacenters.
 */
private void initializeWithCloudDcs(boolean makeCloudDcLocal) {
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    mockPartition = new MockPartitionId();
    List<MockDataNodeId> diskNodes = Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1"));
    populateReplicaList(3 * diskNodes.size(), ReplicaState.STANDBY, diskNodes);
    List<MockDataNodeId> cloudNodes = Arrays.asList(new MockDataNodeId(portList, Collections.emptyList(), "cloud-dc-0"), new MockDataNodeId(portList, Collections.emptyList(), "cloud-dc-1"));
    // only one cloud replica per cloud dc.
    populateReplicaList(cloudNodes.size(), ReplicaState.STANDBY, cloudNodes);
    datanodes = new ArrayList<>();
    datanodes.addAll(diskNodes);
    datanodes.addAll(cloudNodes);
    localDcName = (makeCloudDcLocal ? cloudNodes : diskNodes).get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
}

Example 19 with Port

use of com.github.ambry.network.Port in project ambry by linkedin.

the class OperationTrackerTest method initialize.

/**
 * Initialize 4 DCs, each DC has 1 data node, which has 3 replicas.
 */
private void initialize() {
    int replicaCount = 12;
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1"), new MockDataNodeId(portList, mountPaths, "dc-2"), new MockDataNodeId(portList, mountPaths, "dc-3")));
    mockPartition = new MockPartitionId();
    populateReplicaList(replicaCount, ReplicaState.STANDBY);
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
}

Example 20 with Port

use of com.github.ambry.network.Port in project ambry by linkedin.

the class ServerTestUtil method endToEndCloudBackupTest.

/**
 * Tests blobs put to dataNode can be backed up by {@link com.github.ambry.cloud.VcrReplicationManager}.
 * @param cluster the {@link MockCluster} of dataNodes.
 * @param zkConnectString ZK endpoint to establish VCR cluster
 * @param vcrClusterName the name of VCR cluster
 * @param dataNode the datanode where blobs are originally put.
 * @param clientSSLConfig the {@link SSLConfig}.
 * @param clientSSLSocketFactory the {@link SSLSocketFactory}.
 * @param notificationSystem the {@link MockNotificationSystem} to track blobs event in {@link MockCluster}.
 * @param vcrSSLProps SSL related properties for VCR. Can be {@code null}.
 * @param doTtlUpdate Do ttlUpdate request if {@code true}.
 */
static void endToEndCloudBackupTest(MockCluster cluster, String zkConnectString, String vcrClusterName, DataNodeId dataNode, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, MockNotificationSystem notificationSystem, Properties vcrSSLProps, boolean doTtlUpdate) throws Exception {
    int blobBackupCount = 10;
    int blobSize = 100;
    int userMetaDataSize = 100;
    ClusterAgentsFactory clusterAgentsFactory = cluster.getClusterAgentsFactory();
    // Send blobs to DataNode
    byte[] userMetadata = new byte[userMetaDataSize];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long ttl = doTtlUpdate ? TimeUnit.DAYS.toMillis(1) : Utils.Infinite_Time;
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", null, null, false, ttl, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
    TestUtils.RANDOM.nextBytes(userMetadata);
    TestUtils.RANDOM.nextBytes(data);
    Port port;
    if (clientSSLConfig == null) {
        port = new Port(dataNode.getPort(), PortType.PLAINTEXT);
    } else {
        port = new Port(dataNode.getSSLPort(), PortType.SSL);
    }
    ConnectedChannel channel = getBlockingChannelBasedOnPortType(port, "localhost", clientSSLSocketFactory, clientSSLConfig);
    channel.connect();
    CountDownLatch latch = new CountDownLatch(1);
    DirectSender runnable = new DirectSender(cluster, channel, blobBackupCount, data, userMetadata, properties, null, latch);
    Thread threadToRun = new Thread(runnable);
    threadToRun.start();
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    // TODO: remove this temp fix after fixing race condition in MockCluster/MockNotificationSystem
    Thread.sleep(3000);
    List<BlobId> blobIds = runnable.getBlobIds();
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
        if (doTtlUpdate) {
            updateBlobTtl(channel, blobId, cluster.time.milliseconds());
        }
    }
    HelixControllerManager helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, cluster.getClusterMap());
    // Start the VCR and CloudBackupManager
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, vcrSSLProps);
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, clusterAgentsFactory.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), clusterAgentsFactory, notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // Waiting for backup done
    assertTrue("Did not backup all blobs in 2 minutes", latchBasedInMemoryCloudDestination.awaitUpload(2, TimeUnit.MINUTES));
    Map<String, CloudBlobMetadata> cloudBlobMetadataMap = latchBasedInMemoryCloudDestination.getBlobMetadata(blobIds);
    for (BlobId blobId : blobIds) {
        CloudBlobMetadata cloudBlobMetadata = cloudBlobMetadataMap.get(blobId.toString());
        assertNotNull("cloudBlobMetadata should not be null", cloudBlobMetadata);
        assertEquals("AccountId mismatch", accountId, cloudBlobMetadata.getAccountId());
        assertEquals("ContainerId mismatch", containerId, cloudBlobMetadata.getContainerId());
        assertEquals("Expiration time mismatch", Utils.Infinite_Time, cloudBlobMetadata.getExpirationTime());
    // TODO: verify other metadata and blob data
    }
    vcrServer.shutdown();
    helixControllerManager.syncStop();
}