Examples with Container com.github.ambry.account.Container used on opensource projects

Example 36 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class StorageQuotaEnforcer method charge.

/**
 * Add given {@code usage} to the current storage usage of account/container carried in {@code restRequest} even
 * if the result exceeds quota for the target account/container. If there is no account and container found in
 * {@code restRequest}, then this is a no-op. If there is no quota found for the account/container, then this is
 * a no-op. A {@link Pair} whose first element is quota and second element is the storage usage after charge.
 * @param restRequest the {@link RestRequest} that carries account and container in the header.
 * @param usage the usage to charge
 * @return A {@link Pair} whose first element is quota and second element is the storage usage after charge.
 */
Pair<Long, Long> charge(RestRequest restRequest, long usage) {
    long quotaValue = -1L;
    long usageAfterCharge = 0L;
    try {
        Account account = RestUtils.getAccountFromArgs(restRequest.getArgs());
        Container container = RestUtils.getContainerFromArgs(restRequest.getArgs());
        QuotaResource quotaResource = account.getQuotaResourceType() == QuotaResourceType.ACCOUNT ? QuotaResource.fromAccount(account) : QuotaResource.fromContainer(container);
        quotaValue = getQuotaValueForResource(quotaResource);
        if (quotaValue != -1L) {
            AtomicLong existingUsage = new AtomicLong();
            storageUsages.compute(quotaResource, (k, v) -> {
                existingUsage.set(v == null ? 0 : v);
                if (v == null) {
                    return usage;
                }
                return v + usage;
            });
            usageAfterCharge = existingUsage.addAndGet(usage);
        }
    } catch (Exception e) {
        logger.error("Failed to charge for RestRequest {}", restRequest, e);
    }
    return new Pair<>(quotaValue, usageAfterCharge);
}

Example 37 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class ReplicationTest method blockDeprecatedContainerReplicationTest.

/**
 * Tests if deprecated containers have been blocked during replication.
 */
@Test
public void blockDeprecatedContainerReplicationTest() throws Exception {
    Properties properties = new Properties();
    properties.setProperty("replication.container.deletion.enabled", "true");
    replicationConfig = new ReplicationConfig(new VerifiableProperties(properties));
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    storeKeyConverter.setConversionMap(conversionMap);
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        BlobId b0 = generateRandomBlobId(partitionId);
        conversionMap.put(b0, b0);
        BlobId b1 = generateRandomBlobId(partitionId);
        conversionMap.put(b1, b1);
        // add 2 messages to both hosts.
        storeKeyConverter.setConversionMap(conversionMap);
        storeKeyConverter.convert(conversionMap.keySet());
        // addPutMessagesToReplicasOfPartition(Arrays.asList(b0), Arrays.asList(localHost, remoteHost));
        // add 3 messages to the remote host only
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0, b1), Collections.singletonList(remoteHost));
    }
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    int batchSize = 4;
    ReplicationMetrics replicationMetrics = new ReplicationMetrics(new MetricRegistry(), clusterMap.getReplicaIds(localHost.dataNodeId));
    replicationMetrics.populateSingleColoMetrics(remoteHost.dataNodeId.getDatacenterName());
    List<RemoteReplicaInfo> remoteReplicaInfoList = localHost.getRemoteReplicaInfos(remoteHost, null);
    Map<DataNodeId, List<RemoteReplicaInfo>> replicasToReplicate = Collections.singletonMap(remoteHost.dataNodeId, remoteReplicaInfoList);
    storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
    Map<DataNodeId, MockHost> hosts = new HashMap<>();
    hosts.put(remoteHost.dataNodeId, remoteHost);
    MockConnectionPool connectionPool = new MockConnectionPool(hosts, clusterMap, batchSize);
    Predicate<MessageInfo> skipPredicate = new ReplicationSkipPredicate(accountService, replicationConfig);
    ReplicaThread replicaThread = new ReplicaThread("threadtest", new MockFindTokenHelper(storeKeyFactory, replicationConfig), clusterMap, new AtomicInteger(0), localHost.dataNodeId, connectionPool, replicationConfig, replicationMetrics, null, storeKeyConverter, transformer, clusterMap.getMetricRegistry(), false, localHost.dataNodeId.getDatacenterName(), new ResponseHandler(clusterMap), time, null, skipPredicate);
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfoList) {
        replicaThread.addRemoteReplicaInfo(remoteReplicaInfo);
    }
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasToReplicate.get(remoteHost.dataNodeId);
    DataNodeId remoteNode = remoteReplicaInfos.get(0).getReplicaId().getDataNodeId();
    ReplicaMetadataResponse response = replicaThread.getReplicaMetadataResponse(remoteReplicaInfos, new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteNode);
    // case1 DELETE_IN_PROGRESS container with retention time qualified.
    for (int i = 0; i < 2; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getDeleteTriggerTime()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(replicationConfig.replicationContainerDeletionRetentionDays + 1));
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.DELETE_IN_PROGRESS);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All DELETE_IN_PROGRESS blobs qualified with retention time should be skipped during replication", 0, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case2 DELETE_IN_PROGRESS container with retention time not qualified.
    for (int i = 2; i < 4; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.DELETE_IN_PROGRESS);
            Mockito.when(container.getDeleteTriggerTime()).thenReturn(System.currentTimeMillis());
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All DELETE_IN_PROGRESS blobs not qualified with retention time should not be skipped during replication", 2, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case3 INACTIVE container
    for (int i = 4; i < 6; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.INACTIVE);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All INACTIVE blobs should be skipped during replication", 0, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case 4 ACTIVE Container
    for (int i = 6; i < 8; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.ACTIVE);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All non-deprecated blobs should not be skipped during replication", 2, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
}

Example 38 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class DeleteManager method onComplete.

/**
 * Called when the delete operation is completed. The {@code DeleteManager} also finishes the delete operation
 * by performing the callback and notification.
 * @param op The {@link DeleteOperation} that has completed.
 */
void onComplete(DeleteOperation op) {
    Exception e = op.getOperationException();
    if (e == null) {
        BlobId blobId = op.getBlobId();
        Pair<Account, Container> accountContainer = RouterUtils.getAccountContainer(accountService, blobId.getAccountId(), blobId.getContainerId());
        notificationSystem.onBlobDeleted(blobId.getID(), op.getServiceId(), accountContainer.getFirst(), accountContainer.getSecond());
    } else {
        routerMetrics.onDeleteBlobError(e);
    }
    routerMetrics.operationDequeuingRate.mark();
    routerMetrics.deleteBlobOperationLatencyMs.update(time.milliseconds() - op.getSubmissionTimeMs());
    NonBlockingRouter.completeOperation(op.getFutureResult(), op.getCallback(), op.getOperationResult(), op.getOperationException());
}

Example 39 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class RouterUtils method getAccountContainer.

/**
 * Return {@link Account} and {@link Container} in a {@link Pair}.
 * @param accountService the accountService to translate accountId to name.
 * @param accountId the accountId to translate.
 * @return {@link Account} and {@link Container} in a {@link Pair}.
 */
static Pair<Account, Container> getAccountContainer(AccountService accountService, short accountId, short containerId) {
    Account account = accountService.getAccountById(accountId);
    Container container = account == null ? null : account.getContainerById(containerId);
    return new Pair<>(account, container);
}

Example 40 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class PutManager method getPartitionClass.

/**
 * @param blobProperties the properties of the blob being put
 * @return the partition class as required by the properties
 */
private String getPartitionClass(BlobProperties blobProperties) {
    String partitionClass = defaultPartitionClass;
    Account account = accountService.getAccountById(blobProperties.getAccountId());
    if (account != null) {
        Container container = account.getContainerById(blobProperties.getContainerId());
        if (container != null && !Utils.isNullOrEmpty(container.getReplicationPolicy())) {
            partitionClass = container.getReplicationPolicy();
        }
    }
    return partitionClass;
}