use of com.github.ambry.network.Port in project ambry by linkedin.
the class AdaptiveOperationTrackerTest method nodeLevelAdaptiveTrackerTest.
/**
* Tests that adaptive tracker uses separate node-level histogram to determine if inflight requests are past due.
* @throws Exception
*/
@Test
public void nodeLevelAdaptiveTrackerTest() throws Exception {
// Mock a simple partition layout for this test: Partition1 has two replicas, one on LocalHost1 and the other on RemoteHost1;
// Similarly, Partition2 has two replicas, one on LocalHost2 and the other on RemoteHost1.
MockPartitionId mockPartition1 = new MockPartitionId(1L, MockClusterMap.DEFAULT_PARTITION_CLASS);
MockPartitionId mockPartition2 = new MockPartitionId(2L, MockClusterMap.DEFAULT_PARTITION_CLASS);
// create a new list mock datanodes instead of using the default class member
List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
List<String> mountPaths = Arrays.asList("mockMountPath0", "mockMountPath1", "mockMountPath2");
MockDataNodeId localHost1 = new MockDataNodeId("LocalHost1", portList, mountPaths, "dc-0");
MockDataNodeId localHost2 = new MockDataNodeId("LocalHost2", portList, mountPaths, "dc-0");
MockDataNodeId remoteHost1 = new MockDataNodeId("RemoteHost1", portList, mountPaths, "dc-1");
List<MockDataNodeId> datanodes = new ArrayList<>(Arrays.asList(localHost1, localHost2, remoteHost1));
// distribute replicas to nodes (Note that localDC name is still "dc-0" in current setup)
mockPartition1.replicaIds.add(new MockReplicaId(PORT, mockPartition1, localHost1, 1));
mockPartition2.replicaIds.add(new MockReplicaId(PORT, mockPartition2, localHost2, 2));
mockPartition1.replicaIds.add(new MockReplicaId(PORT, mockPartition1, remoteHost1, 1));
mockPartition2.replicaIds.add(new MockReplicaId(PORT, mockPartition2, remoteHost1, 2));
MockClusterMap clusterMap = new MockClusterMap(false, datanodes, 3, Arrays.asList(mockPartition1, mockPartition2), localDcName);
trackerScope = OperationTrackerScope.DataNode;
RouterConfig routerConfig = createRouterConfig(true, 1, 1, 6, null, true);
NonBlockingRouterMetrics originalMetrics = routerMetrics;
routerMetrics = new NonBlockingRouterMetrics(clusterMap, routerConfig);
Counter pastDueCount = routerMetrics.getBlobPastDueCount;
Map<Resource, CachedHistogram> localColoMap = routerMetrics.getBlobLocalDcResourceToLatency;
Map<Resource, CachedHistogram> crossColoMap = routerMetrics.getBlobCrossDcResourceToLatency;
// mock different latency distribution of local hosts and remote host
Histogram localHistogram1 = localColoMap.get(localHost1);
Histogram localHistogram2 = localColoMap.get(localHost2);
primeTracker(localHistogram1, routerConfig.routerOperationTrackerMinDataPointsRequired, new Pair<>(0L, 50L));
primeTracker(localHistogram2, routerConfig.routerOperationTrackerMinDataPointsRequired, new Pair<>(100L, 120L));
double localHostCutoff1 = localHistogram1.getSnapshot().getValue(QUANTILE);
double localHostCutoff2 = localHistogram2.getSnapshot().getValue(QUANTILE);
OperationTracker tracker1 = getOperationTracker(routerConfig, mockPartition1);
OperationTracker tracker2 = getOperationTracker(routerConfig, mockPartition2);
// issue first request for both partitions in local DC
sendRequests(tracker2, 1);
sendRequests(tracker1, 1);
// partition1: 0-1-0-0, partition2: 0-1-0-0
time.sleep((long) localHostCutoff1 + 1);
// partition1 should send 2nd request to RemoteNode1, partition2 won't because its 1st request isn't past due.
sendRequests(tracker1, 1);
sendRequests(tracker2, 0);
// partition1: 0-1-0-0(local), 0-1-0-0(remote); partition2: 0-1-0-0(local), 1-0-0-0(remote)
time.sleep((long) (localHostCutoff2 - localHostCutoff1) + 2);
sendRequests(tracker1, 0);
sendRequests(tracker2, 1);
// partition1: 0-1-0-0(local), 0-1-0-0(remote); partition2: 0-1-0-0(local), 0-1-0-0(remote)
assertFalse("Operation should not be done", tracker1.isDone() || tracker2.isDone());
// make local requests failed
tracker1.onResponse(partitionAndInflightReplicas.get(mockPartition1).poll(), TrackedRequestFinalState.TIMED_OUT);
tracker2.onResponse(partitionAndInflightReplicas.get(mockPartition2).poll(), TrackedRequestFinalState.FAILURE);
// make remote requests successful
tracker1.onResponse(partitionAndInflightReplicas.get(mockPartition1).poll(), TrackedRequestFinalState.SUCCESS);
tracker2.onResponse(partitionAndInflightReplicas.get(mockPartition2).poll(), TrackedRequestFinalState.SUCCESS);
assertTrue("Operation should have succeeded", tracker1.hasSucceeded() && tracker2.hasSucceeded());
// past due count should be 2 because requests to two local nodes didn't get response within threshold
assertEquals("Past due counter is not expected", 2, pastDueCount.getCount());
// number of data points in local colo histogram should be 1 because LocalHost2 finally responded FAILURE which would
// update the histogram. Note that request to LocalHost1 became TIMED_OUT in the end which should not be counted.
assertEquals("Mismatch in number of data points in local colo histogram", 1, routerMetrics.getBlobLocalDcLatencyMs.getCount());
// number of data points in cross colo histogram should be 2 because both requests to RemoteHost1 succeeded and histogram
// should be updated twice in this case.
assertEquals("Mismatch in number of data points in cross colo histogram", 2, routerMetrics.getBlobCrossDcLatencyMs.getCount());
// additional test: dynamically add 1 new partition and 2 new nodes. Each new node hosts a replica from new partition
MockDataNodeId newNode1 = clusterMap.createNewDataNodes(1, "dc-0").get(0);
MockDataNodeId newNode2 = clusterMap.createNewDataNodes(1, "dc-1").get(0);
MockPartitionId mockPartition3 = new MockPartitionId(3L, MockClusterMap.DEFAULT_PARTITION_CLASS);
mockPartition3.replicaIds.add(new MockReplicaId(PORT, mockPartition3, newNode1, 1));
mockPartition3.replicaIds.add(new MockReplicaId(PORT, mockPartition3, newNode2, 2));
OperationTracker tracker3 = getOperationTracker(routerConfig, mockPartition3);
// send 1st request
sendRequests(tracker3, 1);
// attempt to send 2nd one. This will trigger router metrics to create a histogram that associated with new node
// However, there is no 2nd request out because new created histogram doesn't of enough data points.
sendRequests(tracker3, 0);
// make the 1st request fail
tracker3.onResponse(partitionAndInflightReplicas.get(mockPartition3).poll(), TrackedRequestFinalState.FAILURE);
// 2nd request is sent
sendRequests(tracker3, 1);
// make the 2nd request succeed
tracker3.onResponse(partitionAndInflightReplicas.get(mockPartition3).poll(), TrackedRequestFinalState.SUCCESS);
assertTrue("Operation should have succeeded", tracker3.hasSucceeded());
// restore the tracer scope and routerMetrics
trackerScope = OperationTrackerScope.Datacenter;
routerMetrics = originalMetrics;
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class DeleteOperation method fetchRequests.
/**
* Fetch {@link DeleteRequest}s to send for the operation.
*/
private void fetchRequests(RequestRegistrationCallback<DeleteOperation> requestRegistrationCallback) {
Iterator<ReplicaId> replicaIterator = operationTracker.getReplicaIterator();
while (replicaIterator.hasNext()) {
ReplicaId replica = replicaIterator.next();
String hostname = replica.getDataNodeId().getHostname();
Port port = RouterUtils.getPortToConnectTo(replica, routerConfig.routerEnableHttp2NetworkClient);
DeleteRequest deleteRequest = createDeleteRequest();
deleteRequestInfos.put(deleteRequest.getCorrelationId(), new DeleteRequestInfo(time.milliseconds(), replica));
RequestInfo requestInfo = new RequestInfo(hostname, port, deleteRequest, replica, operationQuotaCharger);
requestRegistrationCallback.registerRequestToSend(this, requestInfo);
replicaIterator.remove();
if (RouterUtils.isRemoteReplica(routerConfig, replica)) {
logger.trace("Making request with correlationId {} to a remote replica {} in {} ", deleteRequest.getCorrelationId(), replica.getDataNodeId(), replica.getDataNodeId().getDatacenterName());
routerMetrics.crossColoRequestCount.inc();
} else {
logger.trace("Making request with correlationId {} to a local replica {} ", deleteRequest.getCorrelationId(), replica.getDataNodeId());
}
routerMetrics.getDataNodeBasedMetrics(replica.getDataNodeId()).deleteRequestRate.mark();
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class GetBlobInfoOperation method fetchRequests.
/**
* Fetch {@link GetRequest}s to send for the operation.
*/
private void fetchRequests(RequestRegistrationCallback<GetOperation> requestRegistrationCallback) {
Iterator<ReplicaId> replicaIterator = operationTracker.getReplicaIterator();
while (replicaIterator.hasNext()) {
ReplicaId replicaId = replicaIterator.next();
String hostname = replicaId.getDataNodeId().getHostname();
Port port = RouterUtils.getPortToConnectTo(replicaId, routerConfig.routerEnableHttp2NetworkClient);
GetRequest getRequest = createGetRequest(blobId, getOperationFlag(), options.getBlobOptions.getGetOption());
RequestInfo request = new RequestInfo(hostname, port, getRequest, replicaId, operationQuotaCharger);
int correlationId = getRequest.getCorrelationId();
correlationIdToGetRequestInfo.put(correlationId, new GetRequestInfo(replicaId, time.milliseconds()));
requestRegistrationCallback.registerRequestToSend(this, request);
replicaIterator.remove();
if (RouterUtils.isRemoteReplica(routerConfig, replicaId)) {
logger.trace("Making request with correlationId {} to a remote replica {} in {} ", correlationId, replicaId.getDataNodeId(), replicaId.getDataNodeId().getDatacenterName());
routerMetrics.crossColoRequestCount.inc();
} else {
logger.trace("Making request with correlationId {} to a local replica {} ", correlationId, replicaId.getDataNodeId());
}
routerMetrics.getDataNodeBasedMetrics(replicaId.getDataNodeId()).getBlobInfoRequestRate.mark();
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class TtlUpdateOperation method fetchRequests.
/**
* Fetch {@link TtlUpdateRequest}s to send for the operation.
* @param requestRegistrationCallback the {@link RequestRegistrationCallback} to use for addition of requests that
* need to be sent to the storage server
*/
private void fetchRequests(RequestRegistrationCallback<TtlUpdateOperation> requestRegistrationCallback) {
Iterator<ReplicaId> replicaIterator = operationTracker.getReplicaIterator();
while (replicaIterator.hasNext()) {
ReplicaId replica = replicaIterator.next();
String hostname = replica.getDataNodeId().getHostname();
Port port = RouterUtils.getPortToConnectTo(replica, routerConfig.routerEnableHttp2NetworkClient);
TtlUpdateRequest ttlUpdateRequest = createTtlUpdateRequest();
ttlUpdateRequestInfos.put(ttlUpdateRequest.getCorrelationId(), new TtlUpdateRequestInfo(time.milliseconds(), replica));
RequestInfo requestInfo = new RequestInfo(hostname, port, ttlUpdateRequest, replica, operationQuotaCharger);
requestRegistrationCallback.registerRequestToSend(this, requestInfo);
replicaIterator.remove();
if (RouterUtils.isRemoteReplica(routerConfig, replica)) {
LOGGER.trace("Making request with correlationId {} to a remote replica {} in {} ", ttlUpdateRequest.getCorrelationId(), replica.getDataNodeId(), replica.getDataNodeId().getDatacenterName());
routerMetrics.crossColoRequestCount.inc();
} else {
LOGGER.trace("Making request with correlationId {} to a local replica {} ", ttlUpdateRequest.getCorrelationId(), replica.getDataNodeId());
}
routerMetrics.getDataNodeBasedMetrics(replica.getDataNodeId()).ttlUpdateRequestRate.mark();
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class OperationTrackerTest method putOperationWithDynamicTargetTest.
/**
* Test put operation with both dynamic success target and get replica by state enabled.
* Test cases:
* Case 1: 1 LEADER and 2 STANDBY replicas (regular case)
* Case 2: 1 LEADER and 3 STANDBY replicas. One of them returns REQUEST_DISABLED error code. There are two combinations
* for remaining 3 replicas:
* (1) two success and one failure (whole operation should succeed)
* (2) one success and two failure (whole operation should fail)
* Case 3: 1 LEADER and 5 STANDBY replicas. Several combinations to discuss:
* (1) 3 REQUEST_DISABLED, 2 success and 1 failure (operation should succeed)
* (2) 2 REQUEST_DISABLED, 2 failure (operation should fail no matter what results are from remaining replicas)
* (3) 1 REQUEST_DISABLED, 1 failure, 4 success (operation should succeed)
* (4) 0 REQUEST_DISABLED, 2 failure, 4 success (operation should fail)
* Case 4: 1 LEADER, 4 STANDBY, 1 INACTIVE (this is to mock one replica has completed STANDBY -> INACTIVE)
* (1) 2 REQUEST_DISABLED, 1 failure and 2 success (operation should succeed)
* (2) 3 REQUEST_DISABLED, 1 failure (operation should fail no matter what result is from remaining replica)
*/
@Test
public void putOperationWithDynamicTargetTest() {
assumeTrue(operationTrackerType.equals(SIMPLE_OP_TRACKER) && replicasStateEnabled);
useDynamicTargetForPut = true;
List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
List<String> mountPaths = Collections.singletonList("mockMountPath");
datanodes = Collections.singletonList(new MockDataNodeId(portList, mountPaths, "dc-0"));
mockPartition = new MockPartitionId();
// Case 1
populateReplicaList(1, ReplicaState.LEADER);
populateReplicaList(2, ReplicaState.STANDBY);
localDcName = datanodes.get(0).getDatacenterName();
mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
OperationTracker ot = getOperationTracker(true, 1, 3, RouterOperation.PutOperation, true);
sendRequests(ot, 3, false);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
assertTrue("Operation should succeed", ot.hasSucceeded());
// Case 2.1
populateReplicaList(1, ReplicaState.STANDBY);
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 4, RouterOperation.PutOperation, true);
sendRequests(ot, 4, false);
// make 1 replica return REQUEST_DISABLED, then 1 failure and 2 success
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
assertTrue("Operation should succeed", ot.hasSucceeded());
// Case 2.2
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 4, RouterOperation.PutOperation, true);
sendRequests(ot, 4, false);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
}
assertFalse("Operation should fail", ot.hasSucceeded());
// Case 3.1
populateReplicaList(2, ReplicaState.STANDBY);
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 6, RouterOperation.PutOperation, true);
sendRequests(ot, 6, false);
for (int i = 0; i < 3; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
}
assertFalse("Operation should not be done yet", ot.isDone());
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
assertFalse("Operation should not be done yet", ot.isDone());
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
assertTrue("Operation should succeed", ot.hasSucceeded());
// Case 3.2
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 6, RouterOperation.PutOperation, true);
sendRequests(ot, 6, false);
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
}
assertFalse("Operation should fail", ot.hasSucceeded());
// Case 3.3
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 6, RouterOperation.PutOperation, true);
sendRequests(ot, 6, false);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
for (int i = 0; i < 4; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
assertTrue("Operation should succeed", ot.hasSucceeded());
// Case 3.4
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 6, RouterOperation.PutOperation, true);
sendRequests(ot, 6, false);
for (int i = 0; i < 6; ++i) {
if (i < 2) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
} else {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
}
assertFalse("Operation should fail", ot.hasSucceeded());
// Case 4
// re-populate replica list
mockPartition.cleanUp();
repetitionTracker.clear();
populateReplicaList(1, ReplicaState.LEADER);
populateReplicaList(4, ReplicaState.STANDBY);
populateReplicaList(1, ReplicaState.INACTIVE);
ot = getOperationTracker(true, 1, 5, RouterOperation.PutOperation, true);
sendRequests(ot, 5, false);
// Case 4.1
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
}
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
assertFalse("Operation should not be done yet", ot.isDone());
for (int i = 0; i < 2; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
}
assertTrue("Operation should succeed", ot.hasSucceeded());
// Case 4.2
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 5, RouterOperation.PutOperation, true);
sendRequests(ot, 5, false);
for (int i = 0; i < 3; ++i) {
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.REQUEST_DISABLED);
}
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
assertTrue("Operation should be done", ot.isDone());
assertFalse("Operation should fail", ot.hasSucceeded());
}
Aggregations