use of com.github.ambry.network.Port in project ambry by linkedin.
the class OperationTrackerTest method getOperationWithReplicaStateTest.
/**
* Test GET operation is able to try on OFFLINE replicas if routerOperationTrackerIncludeDownReplicas is true.
*/
@Test
public void getOperationWithReplicaStateTest() {
assumeTrue(replicasStateEnabled);
List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
List<String> mountPaths = Collections.singletonList("mockMountPath");
datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1")));
mockPartition = new MockPartitionId();
for (ReplicaState state : EnumSet.of(ReplicaState.BOOTSTRAP, ReplicaState.STANDBY, ReplicaState.LEADER, ReplicaState.INACTIVE, ReplicaState.OFFLINE)) {
populateReplicaList(1, state);
}
localDcName = datanodes.get(0).getDatacenterName();
mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
// 1. include down replicas (OFFLINE replicas are eligible for GET)
OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
// make sure 4 requests fails and last one succeeds. (This is to verify operation tracker adds offline replica into replica pool as well)
ReplicaId inflightReplica;
for (int i = 0; i < 4; ++i) {
sendRequests(ot, 1, false);
inflightReplica = inflightReplicas.poll();
// verify that the first 4 replicas are not OFFLINE replica. (OFFLINE replica should be added to the end of queue)
assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
assertFalse("Operation should not complete", ot.isDone());
}
sendRequests(ot, 1, false);
inflightReplica = inflightReplicas.poll();
assertEquals("The last replica should be OFFLINE", ReplicaState.OFFLINE, mockPartition.replicaAndState.get(inflightReplica));
ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
assertTrue("Operation should be done", ot.isDone());
// 2. exclude down replicas
repetitionTracker.clear();
ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, false);
for (int i = 0; i < 4; ++i) {
sendRequests(ot, 1, false);
inflightReplica = inflightReplicas.poll();
// verify that none of these replicas is OFFLINE replica.
assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
if (i < 3) {
assertFalse("Operation should not complete", ot.isDone());
} else {
assertTrue("Operation should complete", ot.isDone());
}
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class OperationTrackerTest method sendCrossColoRequestToDcWithMostReplicasTest.
/**
* Test the case where originating dc name is null and operation tracker will choose dc with most replicas to send
* cross-colo request.
* local dc: one replica;
* remote dc1: three replicas;
* remote dc2: one replica;
*/
@Test
public void sendCrossColoRequestToDcWithMostReplicasTest() {
List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
List<String> mountPaths = Collections.singletonList("mockMountPath");
// set up one node per data center for testing
MockDataNodeId localDcNode = new MockDataNodeId(portList, mountPaths, "dc-0");
MockDataNodeId remoteDc1Node = new MockDataNodeId(portList, mountPaths, "dc-1");
MockDataNodeId remoteDc2Node = new MockDataNodeId(portList, mountPaths, "dc-2");
mockPartition = new MockPartitionId();
localDcName = localDcNode.getDatacenterName();
originatingDcName = null;
chooseDcWithMostReplicas = true;
mockClusterMap = new MockClusterMap(false, Arrays.asList(localDcNode, remoteDc1Node, remoteDc2Node), 1, Collections.singletonList(mockPartition), localDcName);
populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(localDcNode));
populateReplicaList(3, ReplicaState.STANDBY, Collections.singletonList(remoteDc1Node));
populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(remoteDc2Node));
OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
// make local replica return Not_Found
sendRequests(ot, 1, false);
ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.NOT_FOUND);
ReplicaId inflightReplica;
// next, operation tracker should send request to dc-1 as it has most replicas
for (int i = 0; i < 3; ++i) {
sendRequests(ot, 1, false);
inflightReplica = inflightReplicas.poll();
assertEquals("The request should be sent to dc-1 with most replicas", "dc-1", inflightReplica.getDataNodeId().getDatacenterName());
// we deliberately make all replicas in dc-1 return Not_Found to verify that operation won't terminate on not found
ot.onResponse(inflightReplica, TrackedRequestFinalState.NOT_FOUND);
assertFalse("Operation should not be done yet", ot.isDone());
}
// the last request should go to dc-2 and this time we make it succeed
sendRequests(ot, 1, false);
inflightReplica = inflightReplicas.poll();
assertEquals("The request should be sent to dc-2", "dc-2", inflightReplica.getDataNodeId().getDatacenterName());
ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
assertTrue("Operation should succeed", ot.hasSucceeded());
assertTrue("Operation should be done", ot.isDone());
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class OperationTrackerTest method downReplicasOrderingTest.
/**
* Test to ensure that replicas that are down are also returned by the operation tracker, but they are
* ordered after the healthy replicas.
*/
@Test
public void downReplicasOrderingTest() {
List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
List<String> mountPaths = Collections.singletonList("mockMountPath");
datanodes = new ArrayList<>();
datanodes.add(new MockDataNodeId(portList, mountPaths, "dc-0"));
datanodes.add(new MockDataNodeId(portList, mountPaths, "dc-1"));
mockPartition = new MockPartitionId();
mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), datanodes.get(0).getDatacenterName());
int replicaCount = 6;
populateReplicaList(replicaCount, ReplicaState.STANDBY);
// Test scenarios with various number of replicas down
for (int i = 0; i < replicaCount; i++) {
testReplicaDown(replicaCount, i);
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class StatsManagerTest method testStatsManagerWithProblematicStores.
/**
* Test to verify the behavior when dealing with {@link Store} that is null and when {@link StoreException} is thrown.
* @throws Exception
*/
@Test
public void testStatsManagerWithProblematicStores() throws Exception {
DataNodeId dataNodeId = new MockDataNodeId(Collections.singletonList(new Port(6667, PortType.PLAINTEXT)), Collections.singletonList("/tmp"), "DC1");
Map<PartitionId, Store> problematicStoreMap = new HashMap<>();
PartitionId partitionId1 = new MockPartitionId(1, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
PartitionId partitionId2 = new MockPartitionId(2, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
problematicStoreMap.put(partitionId1, null);
Store exceptionStore = new MockStore(new MockStoreStats(new HashMap<>(), true));
problematicStoreMap.put(partitionId2, exceptionStore);
StatsManager testStatsManager = new StatsManager(new MockStorageManager(problematicStoreMap, dataNodeId), Arrays.asList(partitionId1.getReplicaIds().get(0), partitionId2.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
List<PartitionId> unreachablePartitions = new ArrayList<>();
Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> hostAccountStorageStatsMap = new HashMap<>();
for (PartitionId partitionId : problematicStoreMap.keySet()) {
testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
}
assertEquals("Aggregated map should not contain any value", 0L, hostAccountStorageStatsMap.size());
assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
StatsManager.AccountStatsPublisher publisher = testStatsManager.new AccountStatsPublisher(accountStatsStore);
publisher.run();
HostAccountStorageStatsWrapper statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
List<String> unreachableStores = statsWrapper.getHeader().getUnreachableStores();
assertTrue("The unreachable store list should contain Partition1 and Partition2", unreachableStores.containsAll(Arrays.asList(partitionId1.toPathString(), partitionId2.toPathString())));
// test for the scenario where some stores are healthy and some are bad
Map<PartitionId, Store> mixedStoreMap = new HashMap<>(storeMap);
unreachablePartitions.clear();
PartitionId partitionId3 = new MockPartitionId(3, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
PartitionId partitionId4 = new MockPartitionId(4, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
mixedStoreMap.put(partitionId3, null);
mixedStoreMap.put(partitionId4, exceptionStore);
testStatsManager = new StatsManager(new MockStorageManager(mixedStoreMap, dataNodeId), Arrays.asList(partitionId3.getReplicaIds().get(0), partitionId4.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
hostAccountStorageStatsMap.clear();
for (PartitionId partitionId : mixedStoreMap.keySet()) {
testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
}
assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
// test fetchSnapshot method in StatsManager
unreachablePartitions.clear();
// partition 0, 1, 2 are healthy stores, partition 3, 4 are bad ones.
for (PartitionId partitionId : mixedStoreMap.keySet()) {
Map<Short, Map<Short, ContainerStorageStats>> containerStatsMapForPartition = hostAccountStorageStatsMap.get(partitionId.getId());
if (partitionId.getId() < 3) {
assertEquals("Actual map does not match with expected snapshot with partition id " + partitionId.toPathString(), hostAccountStorageStats.getStorageStats().get(partitionId.getId()), containerStatsMapForPartition);
}
}
}
use of com.github.ambry.network.Port in project ambry by linkedin.
the class ServerTestUtil method endToEndReplicationWithMultiNodeMultiPartitionMultiDCTest.
static void endToEndReplicationWithMultiNodeMultiPartitionMultiDCTest(String sourceDatacenter, String sslEnabledDatacenters, PortType portType, MockCluster cluster, MockNotificationSystem notificationSystem, Properties routerProps) throws Exception {
Properties props = new Properties();
props.setProperty("router.hostname", "localhost");
props.setProperty("router.datacenter.name", sourceDatacenter);
props.setProperty("router.put.request.parallelism", "1");
props.setProperty("router.put.success.target", "1");
props.setProperty("clustermap.cluster.name", "test");
props.setProperty("clustermap.datacenter.name", sourceDatacenter);
props.setProperty("clustermap.host.name", "localhost");
props.setProperty("kms.default.container.key", TestUtils.getRandomKey(32));
props.putAll(routerProps);
VerifiableProperties verifiableProperties = new VerifiableProperties(props);
AccountService accountService = new InMemAccountService(false, true);
Router router = new NonBlockingRouterFactory(verifiableProperties, cluster.getClusterMap(), notificationSystem, getSSLFactoryIfRequired(verifiableProperties), accountService).getRouter();
int numberOfRequestsToSend = 15;
int numberOfVerifierThreads = 3;
final LinkedBlockingQueue<Payload> payloadQueue = new LinkedBlockingQueue<Payload>();
final AtomicReference<Exception> exceptionRef = new AtomicReference<>(null);
final CountDownLatch callbackLatch = new CountDownLatch(numberOfRequestsToSend);
List<Future<String>> putFutures = new ArrayList<>(numberOfRequestsToSend);
short accountId = Utils.getRandomShort(TestUtils.RANDOM);
short containerId = Utils.getRandomShort(TestUtils.RANDOM);
for (int i = 0; i < numberOfRequestsToSend; i++) {
int size = new Random().nextInt(5000);
final BlobProperties properties = new BlobProperties(size, "service1", "owner id check", "image/jpeg", false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
final byte[] metadata = new byte[new Random().nextInt(1000)];
final byte[] blob = new byte[size];
TestUtils.RANDOM.nextBytes(metadata);
TestUtils.RANDOM.nextBytes(blob);
Future<String> future = router.putBlob(properties, metadata, new ByteBufferReadableStreamChannel(ByteBuffer.wrap(blob)), new PutBlobOptionsBuilder().build(), new Callback<String>() {
@Override
public void onCompletion(String result, Exception exception) {
if (exception == null) {
payloadQueue.add(new Payload(properties, metadata, blob, result));
} else {
exceptionRef.set(exception);
}
callbackLatch.countDown();
}
}, QUOTA_CHARGE_EVENT_LISTENER);
putFutures.add(future);
}
for (Future<String> future : putFutures) {
future.get(20, TimeUnit.SECONDS);
}
assertTrue("Did not receive all callbacks in time", callbackLatch.await(1, TimeUnit.SECONDS));
if (exceptionRef.get() != null) {
throw exceptionRef.get();
}
// put away for future use
Payload payload1 = payloadQueue.peek();
MockClusterMap clusterMap = cluster.getClusterMap();
BlobId blobId1 = new BlobId(payload1.blobId, clusterMap);
assertEquals("Did not put expected number of blobs", numberOfRequestsToSend, payloadQueue.size());
Properties sslProps = new Properties();
sslProps.putAll(routerProps);
sslProps.setProperty("clustermap.ssl.enabled.datacenters", sslEnabledDatacenters);
sslProps.setProperty("clustermap.cluster.name", "test");
sslProps.setProperty("clustermap.datacenter.name", sourceDatacenter);
sslProps.setProperty("clustermap.host.name", "localhost");
sslProps.setProperty("connectionpool.read.timeout.ms", "15000");
VerifiableProperties vProps = new VerifiableProperties(sslProps);
ConnectionPool connectionPool = new BlockingChannelConnectionPool(new ConnectionPoolConfig(vProps), new SSLConfig(vProps), new ClusterMapConfig(vProps), new MetricRegistry());
CountDownLatch verifierLatch = new CountDownLatch(numberOfVerifierThreads);
AtomicInteger totalRequests = new AtomicInteger(numberOfRequestsToSend);
AtomicInteger verifiedRequests = new AtomicInteger(0);
AtomicBoolean cancelTest = new AtomicBoolean(false);
for (int i = 0; i < numberOfVerifierThreads; i++) {
Thread thread = new Thread(new Verifier(payloadQueue, verifierLatch, totalRequests, verifiedRequests, cluster.getClusterMap(), cancelTest, portType, connectionPool, notificationSystem, cluster.time));
thread.start();
}
assertTrue("Did not verify in 2 minutes", verifierLatch.await(2, TimeUnit.MINUTES));
assertEquals(totalRequests.get(), verifiedRequests.get());
BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
MockDataNodeId dataNodeId = clusterMap.getDataNodes().get(0);
Port port = new Port(portType == PortType.PLAINTEXT ? dataNodeId.getPort() : dataNodeId.getSSLPort(), portType);
ConnectedChannel channel = connectionPool.checkOutConnection("localhost", port, 10000);
PartitionId partitionId = blobId1.getPartition();
// stop the store via AdminRequest
System.out.println("Begin to stop a BlobStore");
AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionId, 1, "clientid2");
BlobStoreControlAdminRequest controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StopStore, adminRequest);
DataInputStream stream = channel.sendAndReceive(controlRequest).getInputStream();
AdminResponse adminResponse = AdminResponse.readFrom(stream);
releaseNettyBufUnderneathStream(stream);
assertEquals("Stop store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
// put a blob on a stopped store, which should fail
byte[] userMetadata = new byte[1000];
byte[] data = new byte[3187];
BlobProperties properties = new BlobProperties(3187, "serviceid1", accountId, containerId, false, cluster.time.milliseconds());
BlobId blobId2 = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), accountId, containerId, partitionId, false, BlobId.BlobDataType.DATACHUNK);
PutRequest putRequest2 = new PutRequest(1, "clientId2", blobId2, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
DataInputStream putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
PutResponse response2 = PutResponse.readFrom(putResponseStream);
releaseNettyBufUnderneathStream(putResponseStream);
assertEquals("Put blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, response2.getError());
// get a blob properties on a stopped store, which should fail
ArrayList<BlobId> ids = new ArrayList<>();
ids.add(blobId1);
ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<>();
PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
partitionRequestInfoList.add(partitionRequestInfo);
GetRequest getRequest1 = new GetRequest(1, "clientId1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
stream = channel.sendAndReceive(getRequest1).getInputStream();
GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
assertEquals("Get blob properties on stopped store should fail", ServerErrorCode.Replica_Unavailable, resp1.getPartitionResponseInfoList().get(0).getErrorCode());
releaseNettyBufUnderneathStream(stream);
// delete a blob on a stopped store, which should fail
DeleteRequest deleteRequest = new DeleteRequest(1, "clientId1", blobId1, System.currentTimeMillis());
stream = channel.sendAndReceive(deleteRequest).getInputStream();
DeleteResponse deleteResponse = DeleteResponse.readFrom(stream);
releaseNettyBufUnderneathStream(stream);
assertEquals("Delete blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, deleteResponse.getError());
// start the store via AdminRequest
System.out.println("Begin to restart the BlobStore");
adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionId, 1, "clientId");
controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StartStore, adminRequest);
stream = channel.sendAndReceive(controlRequest).getInputStream();
adminResponse = AdminResponse.readFrom(stream);
releaseNettyBufUnderneathStream(stream);
assertEquals("Start store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
List<? extends ReplicaId> replicaIds = partitionId.getReplicaIds();
for (ReplicaId replicaId : replicaIds) {
// forcibly mark replicas and disks as up.
MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
mockReplicaId.markReplicaDownStatus(false);
((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, false);
}
// put a blob on a restarted store , which should succeed
putRequest2 = new PutRequest(1, "clientId2", blobId2, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
response2 = PutResponse.readFrom(putResponseStream);
releaseNettyBufUnderneathStream(putResponseStream);
assertEquals("Put blob on restarted store should succeed", ServerErrorCode.No_Error, response2.getError());
// verify the put blob has been replicated successfully.
notificationSystem.awaitBlobCreations(blobId2.getID());
// get a blob on a restarted store , which should succeed
ids = new ArrayList<BlobId>();
ids.add(blobId2);
partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
partitionRequestInfoList.add(partitionRequestInfo);
GetRequest getRequest2 = new GetRequest(1, "clientId2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
stream = channel.sendAndReceive(getRequest2).getInputStream();
GetResponse resp2 = GetResponse.readFrom(stream, clusterMap);
InputStream responseStream = resp2.getInputStream();
BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
assertArrayEquals("Content mismatch.", data, actualBlobData);
releaseNettyBufUnderneathStream(stream);
// delete a blob on a restarted store , which should succeed
deleteRequest = new DeleteRequest(1, "clientId2", blobId2, System.currentTimeMillis());
stream = channel.sendAndReceive(deleteRequest).getInputStream();
deleteResponse = DeleteResponse.readFrom(stream);
releaseNettyBufUnderneathStream(stream);
assertEquals("Delete blob on restarted store should succeed", ServerErrorCode.No_Error, deleteResponse.getError());
router.close();
connectionPool.shutdown();
}
Aggregations