use of com.github.ambry.account.InMemAccountService in project ambry by linkedin.
the class CloudToStoreReplicationManagerTest method cloudReplicaAdditionTest.
/**
* Test both success and failure cases when adding cloud replica
* @throws Exception
*/
@Test
public void cloudReplicaAdditionTest() throws Exception {
StorageManager storageManager = new StorageManager(storeConfig, new DiskManagerConfig(verifiableProperties), Utils.newScheduler(1, true), clusterMap.getMetricRegistry(), null, clusterMap, currentNode, null, Collections.singletonList(mockHelixParticipant), new MockTime(), null, new InMemAccountService(false, false));
CloudToStoreReplicationManager cloudToStoreReplicationManager = new CloudToStoreReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, mockScheduler, currentNode, null, clusterMap.getMetricRegistry(), null, storeKeyConverterFactory, serverConfig.serverMessageTransformer, mockClusterSpectator, mockHelixParticipant);
storageManager.start();
cloudToStoreReplicationManager.start();
mockClusterSpectator.spectate();
// 1. test adding cloud replica that is not present locally
mockHelixParticipant.onPartitionBecomeLeaderFromStandby(NEW_PARTITION_NAME);
assertNull("Cloud replica thread should not be created", TestUtils.getThreadByThisName(REPLICA_THREAD_PREFIX));
// create a new partition and add corresponding store in storage manager
PartitionId newPartition = new MockPartitionId(Long.parseLong(NEW_PARTITION_NAME), MockClusterMap.DEFAULT_PARTITION_CLASS, clusterMap.getDataNodes(), 0);
ReplicaId replicaToAdd = newPartition.getReplicaIds().get(0);
assertTrue("Adding new store should succeed", storageManager.addBlobStore(replicaToAdd));
// 2. we deliberately shut down the store to induce failure when adding cloud replica
storageManager.shutdownBlobStore(newPartition);
mockHelixParticipant.onPartitionBecomeLeaderFromStandby(NEW_PARTITION_NAME);
assertNull("Cloud replica thread should not be created", TestUtils.getThreadByThisName(REPLICA_THREAD_PREFIX));
storageManager.startBlobStore(newPartition);
// 3. mock success case
mockHelixParticipant.onPartitionBecomeLeaderFromStandby(NEW_PARTITION_NAME);
assertNotNull("Cloud replica thread should be created for DC1", TestUtils.getThreadByThisName(REPLICA_THREAD_PREFIX));
cloudToStoreReplicationManager.shutdown();
storageManager.shutdown();
}
use of com.github.ambry.account.InMemAccountService in project ambry by linkedin.
the class ReplicationTest method addAndRemoveReplicaTest.
/**
* Test dynamically add/remove replica in {@link ReplicationManager}
* @throws Exception
*/
@Test
public void addAndRemoveReplicaTest() throws Exception {
MockClusterMap clusterMap = new MockClusterMap();
ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
StoreConfig storeConfig = new StoreConfig(verifiableProperties);
DataNodeId dataNodeId = clusterMap.getDataNodeIds().get(0);
MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
storeKeyConverterFactory.setConversionMap(new HashMap<>());
StorageManager storageManager = new StorageManager(storeConfig, new DiskManagerConfig(verifiableProperties), Utils.newScheduler(1, true), new MetricRegistry(), null, clusterMap, dataNodeId, null, null, new MockTime(), null, new InMemAccountService(false, false));
storageManager.start();
MockReplicationManager replicationManager = new MockReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, clusterMap, dataNodeId, storeKeyConverterFactory, null);
ReplicaId replicaToTest = clusterMap.getReplicaIds(dataNodeId).get(0);
// Attempting to add replica that already exists should fail
assertFalse("Adding an existing replica should fail", replicationManager.addReplica(replicaToTest));
// Create a brand new replica that sits on one of the disk of datanode, add it into replication manager
PartitionId newPartition = clusterMap.createNewPartition(clusterMap.getDataNodes());
for (ReplicaId replicaId : newPartition.getReplicaIds()) {
if (replicaId.getDataNodeId() == dataNodeId) {
replicaToTest = replicaId;
break;
}
}
// Before adding replica, partitionToPartitionInfo and mountPathToPartitionInfos should not contain new partition
assertFalse("partitionToPartitionInfo should not contain new partition", replicationManager.getPartitionToPartitionInfoMap().containsKey(newPartition));
for (PartitionInfo partitionInfo : replicationManager.getMountPathToPartitionInfosMap().get(replicaToTest.getMountPath())) {
assertNotSame("mountPathToPartitionInfos should not contain new partition", partitionInfo.getPartitionId(), newPartition);
}
// Add new replica to replication manager
assertTrue("Adding new replica to replication manager should succeed", replicationManager.addReplica(replicaToTest));
// After adding replica, partitionToPartitionInfo and mountPathToPartitionInfos should contain new partition
assertTrue("partitionToPartitionInfo should contain new partition", replicationManager.getPartitionToPartitionInfoMap().containsKey(newPartition));
Optional<PartitionInfo> newPartitionInfo = replicationManager.getMountPathToPartitionInfosMap().get(replicaToTest.getMountPath()).stream().filter(partitionInfo -> partitionInfo.getPartitionId() == newPartition).findAny();
assertTrue("mountPathToPartitionInfos should contain new partition info", newPartitionInfo.isPresent());
// Verify that all remoteReplicaInfos of new added replica have assigned thread
for (RemoteReplicaInfo remoteReplicaInfo : newPartitionInfo.get().getRemoteReplicaInfos()) {
assertNotNull("The remote replica should be assigned to one replica thread", remoteReplicaInfo.getReplicaThread());
}
// Remove replica
assertTrue("Remove replica from replication manager should succeed", replicationManager.removeReplica(replicaToTest));
// Verify replica is removed, so partitionToPartitionInfo and mountPathToPartitionInfos should not contain new partition
assertFalse("partitionToPartitionInfo should not contain new partition", replicationManager.getPartitionToPartitionInfoMap().containsKey(newPartition));
for (PartitionInfo partitionInfo : replicationManager.getMountPathToPartitionInfosMap().get(replicaToTest.getMountPath())) {
assertNotSame("mountPathToPartitionInfos should not contain new partition", partitionInfo.getPartitionId(), newPartition);
}
// Verify that none of remoteReplicaInfo should have assigned thread
for (RemoteReplicaInfo remoteReplicaInfo : newPartitionInfo.get().getRemoteReplicaInfos()) {
assertNull("The remote replica should be assigned to one replica thread", remoteReplicaInfo.getReplicaThread());
}
// Remove the same replica that doesn't exist should be no-op
ReplicationManager mockManager = Mockito.spy(replicationManager);
assertFalse("Remove non-existent replica should return false", replicationManager.removeReplica(replicaToTest));
verify(mockManager, never()).removeRemoteReplicaInfoFromReplicaThread(anyList());
storageManager.shutdown();
}
use of com.github.ambry.account.InMemAccountService in project ambry by linkedin.
the class ChunkFillTest method fillChunksAndAssertSuccess.
/**
* Create a {@link PutOperation} and pass in a channel with the blobSize set by the caller; and test the chunk
* filling flow for puts.
* Note that this test is for the chunk filling flow, not for the ChunkFiller thread (which never gets exercised,
* as we do not even instantiate the {@link PutManager})
*/
private void fillChunksAndAssertSuccess() throws Exception {
VerifiableProperties vProps = getNonBlockingRouterProperties();
MockClusterMap mockClusterMap = new MockClusterMap();
RouterConfig routerConfig = new RouterConfig(vProps);
routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
ResponseHandler responseHandler = new ResponseHandler(mockClusterMap);
short accountId = Utils.getRandomShort(random);
short containerId = Utils.getRandomShort(random);
BlobProperties putBlobProperties = new BlobProperties(blobSize, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, accountId, containerId, testEncryption, null, null, null);
Random random = new Random();
byte[] putUserMetadata = new byte[10];
random.nextBytes(putUserMetadata);
putContent = new byte[blobSize];
random.nextBytes(putContent);
final ReadableStreamChannel putChannel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(putContent));
FutureResult<String> futureResult = new FutureResult<String>();
MockTime time = new MockTime();
MockNetworkClientFactory networkClientFactory = new MockNetworkClientFactory(vProps, null, 0, 0, 0, null, time);
if (testEncryption) {
kms = new MockKeyManagementService(new KMSConfig(vProps), TestUtils.getRandomKey(SingleKeyManagementServiceTest.DEFAULT_KEY_SIZE_CHARS));
cryptoService = new MockCryptoService(new CryptoServiceConfig(vProps));
cryptoJobHandler = new CryptoJobHandler(CryptoJobHandlerTest.DEFAULT_THREAD_COUNT);
}
MockRouterCallback routerCallback = new MockRouterCallback(networkClientFactory.getNetworkClient(), Collections.EMPTY_LIST);
PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), putUserMetadata, putChannel, PutBlobOptions.DEFAULT, futureResult, null, routerCallback, null, kms, cryptoService, cryptoJobHandler, time, putBlobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
op.startOperation();
numChunks = RouterUtils.getNumChunksForBlobAndChunkSize(blobSize, chunkSize);
compositeBuffers = new ByteBuf[numChunks];
compositeEncryptionKeys = new ByteBuffer[numChunks];
compositeBlobIds = new BlobId[numChunks];
final AtomicReference<Exception> operationException = new AtomicReference<Exception>(null);
int chunksLeftToBeFilled = numChunks;
do {
if (testEncryption) {
int chunksPerBatch = Math.min(routerConfig.routerMaxInMemPutChunks, chunksLeftToBeFilled);
CountDownLatch onPollLatch = new CountDownLatch(chunksPerBatch);
routerCallback.setOnPollLatch(onPollLatch);
op.fillChunks();
Assert.assertTrue("Latch should have been zeroed out", onPollLatch.await(1000, TimeUnit.MILLISECONDS));
chunksLeftToBeFilled -= chunksPerBatch;
} else {
op.fillChunks();
}
// since the channel is ByteBuffer based.
for (PutOperation.PutChunk putChunk : op.putChunks) {
if (putChunk.isFree()) {
continue;
}
Assert.assertEquals("Chunk should be ready.", PutOperation.ChunkState.Ready, putChunk.getState());
ByteBuf buf = putChunk.buf.retainedDuplicate();
totalSizeWritten += buf.readableBytes();
compositeBuffers[putChunk.getChunkIndex()] = buf;
if (testEncryption) {
compositeEncryptionKeys[putChunk.getChunkIndex()] = putChunk.encryptedPerBlobKey.duplicate();
compositeBlobIds[putChunk.getChunkIndex()] = putChunk.chunkBlobId;
}
putChunk.clear();
}
} while (!op.isChunkFillingDone());
if (!testEncryption) {
Assert.assertEquals("total size written out should match the blob size", blobSize, totalSizeWritten);
}
// for encrypted path, size will be implicitly tested via assertDataIdentity
Exception exception = operationException.get();
if (exception != null) {
throw exception;
}
assertDataIdentity(mockClusterMap);
}
use of com.github.ambry.account.InMemAccountService in project ambry by linkedin.
the class DeleteManagerTest method testVariousServerErrorCodesForThreeParallelism.
/**
* The parallelism is set to 3 not 9.
*
* Test the case where servers return different {@link ServerErrorCode}, and the {@link DeleteOperation}
* is able to resolve and conclude the correct {@link RouterErrorCode}. The {link ServerErrorCode} tested
* are those could be mapped to {@link RouterErrorCode#AmbryUnavailable}. The order of received responses
* is the same as defined in {@code serverErrorCodes}.
*/
@Test
public void testVariousServerErrorCodesForThreeParallelism() throws Exception {
assertCloseCleanup(router);
Properties props = getNonBlockingRouterProperties();
props.setProperty("router.delete.request.parallelism", "3");
VerifiableProperties vProps = new VerifiableProperties(props);
RouterConfig routerConfig = new RouterConfig(vProps);
router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(clusterMap, routerConfig), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, serverLayout, mockTime), new LoggingNotificationSystem(), clusterMap, null, null, null, new InMemAccountService(false, true), mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
ServerErrorCode[] serverErrorCodes = new ServerErrorCode[9];
serverErrorCodes[0] = ServerErrorCode.Blob_Not_Found;
serverErrorCodes[1] = ServerErrorCode.Data_Corrupt;
serverErrorCodes[2] = ServerErrorCode.IO_Error;
serverErrorCodes[3] = ServerErrorCode.Partition_Unknown;
serverErrorCodes[4] = ServerErrorCode.Disk_Unavailable;
serverErrorCodes[5] = ServerErrorCode.No_Error;
serverErrorCodes[6] = ServerErrorCode.Data_Corrupt;
serverErrorCodes[7] = ServerErrorCode.Unknown_Error;
serverErrorCodes[8] = ServerErrorCode.Disk_Unavailable;
testWithErrorCodes(serverErrorCodes, partition, serverLayout, RouterErrorCode.AmbryUnavailable, deleteErrorCodeChecker);
}
use of com.github.ambry.account.InMemAccountService in project ambry by linkedin.
the class DeleteManagerTest method init.
/**
* Initializes ClusterMap, Router, mock servers, and an {@code BlobId} to be deleted.
*/
@Before
public void init() throws Exception {
VerifiableProperties vProps = new VerifiableProperties(getNonBlockingRouterProperties());
mockTime = new MockTime();
mockSelectorState = new AtomicReference<MockSelectorState>(MockSelectorState.Good);
clusterMap = new MockClusterMap();
serverLayout = new MockServerLayout(clusterMap);
RouterConfig routerConfig = new RouterConfig(vProps);
router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(clusterMap, routerConfig), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, serverLayout, mockTime), new LoggingNotificationSystem(), clusterMap, null, null, null, new InMemAccountService(false, true), mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
List<PartitionId> mockPartitions = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
partition = mockPartitions.get(ThreadLocalRandom.current().nextInt(mockPartitions.size()));
blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), partition, false, BlobId.BlobDataType.DATACHUNK);
blobIdString = blobId.getID();
}
Aggregations