use of com.github.ambry.store.Store in project ambry by linkedin.
the class StatsManagerTest method testStatsManagerWithProblematicStores.
/**
* Test to verify the behavior when dealing with {@link Store} that is null and when {@link StoreException} is thrown.
* @throws Exception
*/
@Test
public void testStatsManagerWithProblematicStores() throws Exception {
DataNodeId dataNodeId = new MockDataNodeId(Collections.singletonList(new Port(6667, PortType.PLAINTEXT)), Collections.singletonList("/tmp"), "DC1");
Map<PartitionId, Store> problematicStoreMap = new HashMap<>();
PartitionId partitionId1 = new MockPartitionId(1, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
PartitionId partitionId2 = new MockPartitionId(2, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
problematicStoreMap.put(partitionId1, null);
Store exceptionStore = new MockStore(new MockStoreStats(new HashMap<>(), true));
problematicStoreMap.put(partitionId2, exceptionStore);
StatsManager testStatsManager = new StatsManager(new MockStorageManager(problematicStoreMap, dataNodeId), Arrays.asList(partitionId1.getReplicaIds().get(0), partitionId2.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
List<PartitionId> unreachablePartitions = new ArrayList<>();
Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> hostAccountStorageStatsMap = new HashMap<>();
for (PartitionId partitionId : problematicStoreMap.keySet()) {
testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
}
assertEquals("Aggregated map should not contain any value", 0L, hostAccountStorageStatsMap.size());
assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
StatsManager.AccountStatsPublisher publisher = testStatsManager.new AccountStatsPublisher(accountStatsStore);
publisher.run();
HostAccountStorageStatsWrapper statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
List<String> unreachableStores = statsWrapper.getHeader().getUnreachableStores();
assertTrue("The unreachable store list should contain Partition1 and Partition2", unreachableStores.containsAll(Arrays.asList(partitionId1.toPathString(), partitionId2.toPathString())));
// test for the scenario where some stores are healthy and some are bad
Map<PartitionId, Store> mixedStoreMap = new HashMap<>(storeMap);
unreachablePartitions.clear();
PartitionId partitionId3 = new MockPartitionId(3, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
PartitionId partitionId4 = new MockPartitionId(4, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
mixedStoreMap.put(partitionId3, null);
mixedStoreMap.put(partitionId4, exceptionStore);
testStatsManager = new StatsManager(new MockStorageManager(mixedStoreMap, dataNodeId), Arrays.asList(partitionId3.getReplicaIds().get(0), partitionId4.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
hostAccountStorageStatsMap.clear();
for (PartitionId partitionId : mixedStoreMap.keySet()) {
testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
}
assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
// test fetchSnapshot method in StatsManager
unreachablePartitions.clear();
// partition 0, 1, 2 are healthy stores, partition 3, 4 are bad ones.
for (PartitionId partitionId : mixedStoreMap.keySet()) {
Map<Short, Map<Short, ContainerStorageStats>> containerStatsMapForPartition = hostAccountStorageStatsMap.get(partitionId.getId());
if (partitionId.getId() < 3) {
assertEquals("Actual map does not match with expected snapshot with partition id " + partitionId.toPathString(), hostAccountStorageStats.getStorageStats().get(partitionId.getId()), containerStatsMapForPartition);
}
}
}
use of com.github.ambry.store.Store in project ambry by linkedin.
the class StatsManagerTest method testAddAndRemoveReplica.
/**
* Test to verify the {@link StatsManager} behaves correctly when dynamically adding/removing {@link ReplicaId}.
* @throws Exception
*/
@Test
public void testAddAndRemoveReplica() throws Exception {
// setup testing environment
Map<PartitionId, Store> testStoreMap = new HashMap<>();
List<ReplicaId> testReplicas = new ArrayList<>();
DataNodeId dataNodeId = new MockDataNodeId(Collections.singletonList(new Port(6667, PortType.PLAINTEXT)), Collections.singletonList("/tmp"), "DC1");
for (int i = 0; i < 3; i++) {
PartitionId partitionId = new MockPartitionId(i, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
testStoreMap.put(partitionId, new MockStore(new MockStoreStats(hostAccountStorageStats.getStorageStats().get(i), false)));
testReplicas.add(partitionId.getReplicaIds().get(0));
}
StorageManager mockStorageManager = new MockStorageManager(testStoreMap, dataNodeId);
StatsManager testStatsManager = new StatsManager(mockStorageManager, testReplicas, new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
// verify that adding an existing store to StatsManager should fail
assertFalse("Adding a store which already exists should fail", testStatsManager.addReplica(testReplicas.get(0)));
PartitionId partitionId3 = new MockPartitionId(3, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
testStoreMap.put(partitionId3, new MockStore(new MockStoreStats(hostAccountStorageStats.getStorageStats().get(0), false)));
// verify that partitionId3 is not in stats report before adding to statsManager
StatsManager.AccountStatsPublisher publisher = testStatsManager.new AccountStatsPublisher(accountStatsStore);
publisher.run();
HostAccountStorageStatsWrapper statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
assertFalse("Partition3 should not present in stats report", statsWrapper.getStats().getStorageStats().containsKey(partitionId3.getId()));
// verify that after adding into statsManager, PartitionId3 is in stats report
testStatsManager.addReplica(partitionId3.getReplicaIds().get(0));
publisher.run();
statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
assertTrue("Partition3 should present in stats report", statsWrapper.getStats().getStorageStats().containsKey(partitionId3.getId()));
// verify that after removing PartitionId0 (corresponding to the first replica in replicas list), PartitionId0 is not in the stats report
PartitionId partitionId0 = testReplicas.get(0).getPartitionId();
assertTrue("Partition0 should present in stats report before removal", statsWrapper.getStats().getStorageStats().containsKey(partitionId0.getId()));
testStoreMap.remove(testReplicas.get(0).getPartitionId());
testStatsManager.removeReplica(testReplicas.get(0));
publisher.run();
statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
assertFalse("Partition0 should not present in stats report after removal", statsWrapper.getStats().getStorageStats().containsKey(partitionId0.getId()));
// verify that removing the PartitionId0 should fail because it no longer exists in StatsManager
assertFalse(testStatsManager.removeReplica(testReplicas.get(0)));
// concurrent remove test
CountDownLatch getStatsCountdown1 = new CountDownLatch(1);
CountDownLatch waitRemoveCountdown = new CountDownLatch(1);
((MockStorageManager) mockStorageManager).waitOperationCountdown = waitRemoveCountdown;
((MockStorageManager) mockStorageManager).firstCall = true;
((MockStorageManager) mockStorageManager).unreachablePartitions.clear();
for (Store store : testStoreMap.values()) {
((MockStore) store).getStatsCountdown = getStatsCountdown1;
((MockStore) store).isCollected = false;
}
List<PartitionId> partitionRemoved = new ArrayList<>();
Utils.newThread(() -> {
// wait until at least one store has been collected (this ensures stats aggregation using old snapshot of map)
try {
getStatsCountdown1.await();
} catch (InterruptedException e) {
throw new IllegalStateException("CountDown await was interrupted", e);
}
// find one store which hasn't been collected
ReplicaId replicaToRemove = null;
for (Map.Entry<PartitionId, Store> partitionToStore : testStoreMap.entrySet()) {
MockStore store = (MockStore) partitionToStore.getValue();
if (!store.isCollected) {
replicaToRemove = partitionToStore.getKey().getReplicaIds().get(0);
break;
}
}
if (replicaToRemove != null) {
testStatsManager.removeReplica(replicaToRemove);
testStoreMap.remove(replicaToRemove.getPartitionId());
partitionRemoved.add(replicaToRemove.getPartitionId());
// count down to allow stats aggregation to proceed
waitRemoveCountdown.countDown();
}
}, false).start();
publisher.run();
statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
// verify that the removed store is indeed unreachable during stats aggregation
assertTrue("The removed partition should be unreachable during aggregation", ((MockStorageManager) mockStorageManager).unreachablePartitions.contains(partitionRemoved.get(0)));
// verify unreachable store list doesn't contain the store which is removed.
List<String> unreachableStores = statsWrapper.getHeader().getUnreachableStores();
assertFalse("The removed partition should not present in unreachable list", unreachableStores.contains(partitionRemoved.get(0).toPathString()));
// concurrent add test
CountDownLatch getStatsCountdown2 = new CountDownLatch(1);
CountDownLatch waitAddCountdown = new CountDownLatch(1);
((MockStorageManager) mockStorageManager).waitOperationCountdown = waitAddCountdown;
((MockStorageManager) mockStorageManager).firstCall = true;
((MockStorageManager) mockStorageManager).unreachablePartitions.clear();
for (Store store : testStoreMap.values()) {
((MockStore) store).getStatsCountdown = getStatsCountdown2;
((MockStore) store).isCollected = false;
}
PartitionId partitionId4 = new MockPartitionId(4, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
Utils.newThread(() -> {
// wait until at least one store has been collected (this ensures stats aggregation using old snapshot of map)
try {
getStatsCountdown2.await();
} catch (InterruptedException e) {
throw new IllegalStateException("CountDown await was interrupted", e);
}
testStatsManager.addReplica(partitionId4.getReplicaIds().get(0));
testStoreMap.put(partitionId4, new MockStore(new MockStoreStats(hostAccountStorageStats.getStorageStats().get(0), false)));
// count down to allow stats aggregation to proceed
waitAddCountdown.countDown();
}, false).start();
publisher.run();
statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
// verify that new added PartitionId4 is not in report for this round of aggregation
assertFalse("Partition4 should not present in stats report", statsWrapper.getStats().getStorageStats().containsKey(partitionId4.getId()));
// verify that new added PartitionId4 will be collected for next round of aggregation
publisher.run();
statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
assertTrue("Partition4 should present in stats report", statsWrapper.getStats().getStorageStats().containsKey(partitionId4.getId()));
}
use of com.github.ambry.store.Store in project ambry by linkedin.
the class AmbryRequests method handleDeleteRequest.
public void handleDeleteRequest(Request request) throws IOException, InterruptedException {
DeleteRequest deleteRequest = DeleteRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
metrics.deleteBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.deleteBlobRequestRate.mark();
long startTime = SystemTime.getInstance().milliseconds();
DeleteResponse response = null;
try {
ServerErrorCode error = validateRequest(deleteRequest.getBlobId().getPartition(), RequestOrResponseType.DeleteRequest);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating delete request failed with error {} for request {}", error, deleteRequest);
response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), error);
} else {
MessageFormatInputStream stream = new DeleteMessageFormatInputStream(deleteRequest.getBlobId(), deleteRequest.getAccountId(), deleteRequest.getContainerId(), deleteRequest.getDeletionTimeInMs());
MessageInfo info = new MessageInfo(deleteRequest.getBlobId(), stream.getSize(), deleteRequest.getAccountId(), deleteRequest.getContainerId(), deleteRequest.getDeletionTimeInMs());
ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
infoList.add(info);
MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
Store storeToDelete = storageManager.getStore(deleteRequest.getBlobId().getPartition());
storeToDelete.delete(writeset);
response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ServerErrorCode.No_Error);
if (notification != null) {
notification.onBlobReplicaDeleted(currentNode.getHostname(), currentNode.getPort(), deleteRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
}
}
} catch (StoreException e) {
boolean logInErrorLevel = false;
if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
metrics.idNotFoundError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
metrics.ttlExpiredError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
metrics.idDeletedError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
metrics.deleteAuthorizationFailure.inc();
} else {
logInErrorLevel = true;
metrics.unExpectedStoreDeleteError.inc();
}
if (logInErrorLevel) {
logger.error("Store exception on a delete with error code {} for request {}", e.getErrorCode(), deleteRequest, e);
} else {
logger.trace("Store exception on a delete with error code {} for request {}", e.getErrorCode(), deleteRequest, e);
}
response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
} catch (Exception e) {
logger.error("Unknown exception for delete request " + deleteRequest, e);
response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ServerErrorCode.Unknown_Error);
metrics.unExpectedStoreDeleteError.inc();
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", deleteRequest, response, processingTime);
metrics.deleteBlobProcessingTimeInMs.update(processingTime);
}
requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.deleteBlobResponseQueueTimeInMs, metrics.deleteBlobSendTimeInMs, metrics.deleteBlobTotalTimeInMs, null, null, totalTimeSpent));
}
use of com.github.ambry.store.Store in project ambry by linkedin.
the class AmbryRequests method handlePutRequest.
public void handlePutRequest(Request request) throws IOException, InterruptedException {
PutRequest.ReceivedPutRequest receivedRequest = PutRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.putBlobRequestRate.mark();
long startTime = SystemTime.getInstance().milliseconds();
PutResponse response = null;
try {
ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
} else {
MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
MessageInfo info = new MessageInfo(receivedRequest.getBlobId(), stream.getSize(), false, Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), receivedRequest.getBlobProperties().getTimeToLiveInSeconds()), receivedRequest.getCrc(), receivedRequest.getBlobProperties().getAccountId(), receivedRequest.getBlobProperties().getContainerId(), receivedRequest.getBlobProperties().getCreationTimeInMs());
ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
infoList.add(info);
MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
Store storeToPut = storageManager.getStore(receivedRequest.getBlobId().getPartition());
storeToPut.put(writeset);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
if (notification != null) {
notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
}
}
} catch (StoreException e) {
logger.error("Store exception on a put with error code " + e.getErrorCode() + " for request " + receivedRequest, e);
if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
metrics.idAlreadyExistError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.IOError) {
metrics.storeIOError.inc();
} else {
metrics.unExpectedStorePutError.inc();
}
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
} catch (Exception e) {
logger.error("Unknown exception on a put for request " + receivedRequest, e);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
metrics.putBlobProcessingTimeInMs.update(processingTime);
metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
}
sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}
use of com.github.ambry.store.Store in project ambry by linkedin.
the class StatsManager method fetchSnapshot.
/**
* Fetch the {@link StatsSnapshot} for the given {@link PartitionId}.
* @param partitionId the {@link PartitionId} to try to fetch the {@link StatsSnapshot} from
* @param unreachableStores a list of partitionIds to keep track of the unreachable stores (partitions)
* @return
*/
StatsSnapshot fetchSnapshot(PartitionId partitionId, List<String> unreachableStores) {
StatsSnapshot statsSnapshot = null;
Store store = storageManager.getStore(partitionId);
if (store == null) {
unreachableStores.add(partitionId.toString());
} else {
try {
statsSnapshot = store.getStoreStats().getStatsSnapshot(time.milliseconds());
} catch (StoreException e) {
logger.error("StoreException on fetching stats snapshot for store {}", store, e);
unreachableStores.add(partitionId.toString());
}
}
return statsSnapshot;
}
Aggregations