Examples with ContainerBuilder com.github.ambry.account.ContainerBuilder used on opensource projects

Example 6 with ContainerBuilder

use of com.github.ambry.account.ContainerBuilder in project ambry by linkedin.

the class StorageQuotaEnforcerTest method testStorageUsageRefresherListener.

/**
 * Test when storage usage updates.
 */
@Test
public void testStorageUsageRefresherListener() throws Exception {
    int initNumAccounts = 10;
    Map<String, Map<String, Long>> containerUsage = TestUtils.makeStorageMap(initNumAccounts, 10, 10000, 1000);
    InMemAccountService accountService = new InMemAccountService(false, false);
    Map<QuotaResource, Long> expectedStorageUsages = new HashMap<>();
    // Account and container id's base is 1, not 0
    for (int i = 1; i <= initNumAccounts; i++) {
        QuotaResourceType resourceType = i <= containerUsage.size() / 2 ? QuotaResourceType.CONTAINER : QuotaResourceType.ACCOUNT;
        AccountBuilder accountBuilder = new AccountBuilder((short) i, String.valueOf(i), Account.AccountStatus.ACTIVE, resourceType);
        for (int j = 1; j <= 10; j++) {
            accountBuilder.addOrUpdateContainer(new ContainerBuilder((short) j, String.valueOf(j), Container.ContainerStatus.ACTIVE, "", (short) i).build());
        }
        accountService.updateAccounts(Collections.singleton(accountBuilder.build()));
        if (resourceType == QuotaResourceType.ACCOUNT) {
            expectedStorageUsages.put(QuotaResource.fromAccountId((short) i), containerUsage.get(String.valueOf(i)).values().stream().mapToLong(Long::longValue).sum());
        } else {
            for (Map.Entry<String, Long> containerEntry : containerUsage.get(String.valueOf(i)).entrySet()) {
                expectedStorageUsages.put(QuotaResource.fromContainerId((short) i, Short.valueOf(containerEntry.getKey())), containerEntry.getValue());
            }
        }
    }
    StorageQuotaEnforcer enforcer = new StorageQuotaEnforcer(config, new JSONStringStorageQuotaSource(new HashMap<>(), accountService), (StorageUsageRefresher) null);
    enforcer.initStorageUsage(containerUsage);
    StorageUsageRefresher.Listener listener = enforcer.getUsageRefresherListener();
    int numUpdates = 10;
    for (int i = 1; i <= numUpdates; i++) {
        if (i % 2 == 0) {
            // add new storage usage
            Map<String, Map<String, Long>> additionalUsage = TestUtils.makeStorageMap(1, 10, 10000, 1000);
            short accountId = (short) (initNumAccounts + i);
            containerUsage.put(String.valueOf(accountId), additionalUsage.remove("1"));
            QuotaResourceType resourceType = i % 4 == 0 ? QuotaResourceType.CONTAINER : QuotaResourceType.ACCOUNT;
            AccountBuilder accountBuilder = new AccountBuilder(accountId, String.valueOf(accountId), Account.AccountStatus.ACTIVE, resourceType);
            for (int j = 1; j <= 10; j++) {
                accountBuilder.addOrUpdateContainer(new ContainerBuilder((short) j, String.valueOf(j), Container.ContainerStatus.ACTIVE, "", (short) accountId).build());
            }
            accountService.updateAccounts(Collections.singleton(accountBuilder.build()));
            if (resourceType == QuotaResourceType.ACCOUNT) {
                expectedStorageUsages.put(QuotaResource.fromAccountId(accountId), containerUsage.get(String.valueOf(accountId)).values().stream().mapToLong(Long::longValue).sum());
            } else {
                for (Map.Entry<String, Long> containerEntry : containerUsage.get(String.valueOf(accountId)).entrySet()) {
                    expectedStorageUsages.put(QuotaResource.fromContainerId(accountId, Short.valueOf(containerEntry.getKey())), containerEntry.getValue());
                }
            }
        } else {
            // change existing storage usage
            Random random = new Random();
            int accountId = random.nextInt(initNumAccounts) + 1;
            int containerId = random.nextInt(10) + 1;
            long newValue = random.nextLong();
            long oldValue = containerUsage.get(String.valueOf(accountId)).get(String.valueOf(containerId));
            containerUsage.get(String.valueOf(accountId)).put(String.valueOf(containerId), newValue);
            if (accountService.getAccountById((short) accountId).getQuotaResourceType() == QuotaResourceType.ACCOUNT) {
                QuotaResource resource = QuotaResource.fromAccountId((short) accountId);
                expectedStorageUsages.put(resource, expectedStorageUsages.get(resource) - oldValue + newValue);
            } else {
                expectedStorageUsages.put(QuotaResource.fromContainerId((short) accountId, (short) containerId), newValue);
            }
        }
        listener.onNewContainerStorageUsage(containerUsage);
        assertEquals(expectedStorageUsages, enforcer.getStorageUsages());
    }
}

Example 7 with ContainerBuilder

use of com.github.ambry.account.ContainerBuilder in project ambry by linkedin.

the class RouterUtilsTest method testGetAccountContainer.

/**
 * Test {@link RouterUtils#getAccountContainer(AccountService, short, short)}.
 */
@Test
public void testGetAccountContainer() throws AccountServiceException {
    AccountService accountService = new InMemAccountService(false, false);
    // Both accountId and containerId are not tracked by AccountService.
    Pair<Account, Container> accountContainer = RouterUtils.getAccountContainer(accountService, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID);
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    accountContainer = RouterUtils.getAccountContainer(accountService, Utils.getRandomShort(random), Utils.getRandomShort(random));
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // accountId is tracked by AccountService but containerId not.
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    Account account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // Both accountId and containerId are tracked by AccountService.
    Container container = new ContainerBuilder(containerId, "ContainerNameOf" + containerId, Container.ContainerStatus.ACTIVE, "description", accountId).build();
    account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).addOrUpdateContainer(container).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container doesn't match", container, accountContainer.getSecond());
}

Example 8 with ContainerBuilder

use of com.github.ambry.account.ContainerBuilder in project ambry by linkedin.

the class BlobStoreCompactorTest method testCloseLastLogSegmentIfQualified.

@Test
public void testCloseLastLogSegmentIfQualified() throws Exception {
    // create first active log segment
    refreshState(false, false, true);
    // leave five log segment space for auto close last log segment purpose
    long requiredCount = state.log.getCapacityInBytes() / state.log.getSegmentCapacity() - 5;
    writeDataToMeetRequiredSegmentCountForAutoCloseLogSegmentTest(requiredCount);
    // delete blobs in last index segment.
    ConcurrentSkipListMap<Offset, IndexSegment> AllIndexSegments = state.index.getIndexSegments();
    IndexSegment indexSegment = AllIndexSegments.floorEntry(state.log.getEndOffset()).getValue();
    ListIterator<IndexEntry> segmentListIter = indexSegment.listIterator(indexSegment.size());
    Set<Container> deleteInProgressSet = new HashSet<>();
    int cnt = 0;
    while (segmentListIter.hasNext()) {
        IndexEntry indexEntry = segmentListIter.next();
        MockId mockId = (MockId) indexEntry.getKey();
        Container container = new ContainerBuilder(mockId.getContainerId(), "containerName" + cnt++, Container.ContainerStatus.DELETE_IN_PROGRESS, null, mockId.getAccountId()).build();
        deleteInProgressSet.add(container);
    }
    Mockito.when(accountService.getContainersByStatus(Container.ContainerStatus.DELETE_IN_PROGRESS)).thenReturn(deleteInProgressSet);
    // reload index to make sure journal is on only the latest log segment
    state.reloadIndex(true, false);
    List<LogSegmentName> segmentsUnderCompaction = getLogSegments(0, state.index.getLogSegmentCount() - 1);
    // Prepare the compactionPolicyInfo before compaction.
    CompactionPolicySwitchInfo compactionPolicySwitchInfo = new CompactionPolicySwitchInfo(System.currentTimeMillis(), true);
    backUpCompactionPolicyInfo(tempDir.toString(), compactionPolicySwitchInfo);
    // instantiate compactor.
    compactor = getCompactor(state.log, DISK_IO_SCHEDULER, null, true);
    compactor.initialize(state.index);
    compactAndVerifyForContainerDeletion(segmentsUnderCompaction, state.time.milliseconds(), false);
    // increment the counter
    compactionPolicySwitchInfo.setNextRoundIsCompactAllPolicy(false);
    // reload index to make sure journal is on only the latest log segment
    backUpCompactionPolicyInfo(tempDir.toString(), compactionPolicySwitchInfo);
    segmentsUnderCompaction = getLogSegments(0, state.log.getCapacityInBytes() / state.log.getSegmentCapacity() - state.log.getRemainingUnallocatedSegments() - 1);
    compactAndVerifyForContainerDeletion(segmentsUnderCompaction, state.time.milliseconds(), true);
    // Edge case test: if the last log segment is empty, no need to auto close it.
    int beforeAutoCloseLogSegmentsCnt = state.log.getLogSegmentCount();
    compactionPolicySwitchInfo = new CompactionPolicySwitchInfo(System.currentTimeMillis(), true);
    backUpCompactionPolicyInfo(tempDir.toString(), compactionPolicySwitchInfo);
    compactor = getCompactor(state.log, DISK_IO_SCHEDULER, null, true);
    compactor.initialize(state.index);
    if (state.log.autoCloseLastLogSegmentIfQualified()) {
        // refresh journal.
        state.index.journal.cleanUpJournal();
    }
    int afterAutoCloseLogSegmentsCnt = state.log.getLogSegmentCount();
    assertEquals("No segments should be created since last log segment is empty", beforeAutoCloseLogSegmentsCnt, afterAutoCloseLogSegmentsCnt);
    // make sure new data will be added into the last log segment
    long logSegmentCountBeforeNewDataAddIntoAutoClosedLogSegment = state.index.getLogSegmentCount();
    state.addPutEntries(1, PUT_RECORD_SIZE, Utils.Infinite_Time);
    long logSegmentCountAfterNewDataAddIntoAutoClosedLogSegment = state.index.getLogSegmentCount();
    assertEquals("Log Segment count should be increased after some data has been added to the last log segment.", logSegmentCountBeforeNewDataAddIntoAutoClosedLogSegment, logSegmentCountAfterNewDataAddIntoAutoClosedLogSegment - 1);
    assertEquals("Last index segment should belongs to auto closed log segment", state.index.getIndexSegments().lastEntry().getValue().getLogSegmentName(), state.log.getLastSegment().getName());
    // close the last log segment again.
    beforeAutoCloseLogSegmentsCnt = state.log.getLogSegmentCount();
    if (state.log.autoCloseLastLogSegmentIfQualified()) {
        // refresh journal.
        state.index.journal.cleanUpJournal();
    }
    afterAutoCloseLogSegmentsCnt = state.log.getLogSegmentCount();
    assertEquals("One log segment should be created since last log segment is not empty", beforeAutoCloseLogSegmentsCnt, afterAutoCloseLogSegmentsCnt - 1);
    // make sure new data will be added into the last log segment
    logSegmentCountBeforeNewDataAddIntoAutoClosedLogSegment = state.index.getLogSegmentCount();
    state.addPutEntries(1, PUT_RECORD_SIZE, Utils.Infinite_Time);
    logSegmentCountAfterNewDataAddIntoAutoClosedLogSegment = state.index.getLogSegmentCount();
    assertEquals("Log Segment count should be increased after some data has been added to the last log segment.", logSegmentCountBeforeNewDataAddIntoAutoClosedLogSegment, logSegmentCountAfterNewDataAddIntoAutoClosedLogSegment - 1);
    assertEquals("Last index segment should belongs to auto closed log segment", state.index.getIndexSegments().lastEntry().getValue().getLogSegmentName(), state.log.getLastSegment().getName());
    // check flow after deployment
    state.reloadLog(true);
}

Example 9 with ContainerBuilder

use of com.github.ambry.account.ContainerBuilder in project ambry by linkedin.

the class AccountDao method convertContainersResultSet.

/**
 * Convert a query result set to a list of containers.
 * @param resultSet the result set.
 * @return a list of {@link Container}s.
 * @throws SQLException
 */
private List<Container> convertContainersResultSet(ResultSet resultSet) throws SQLException {
    List<Container> containers = new ArrayList<>();
    while (resultSet.next()) {
        int accountId = resultSet.getInt(ACCOUNT_ID);
        String containerJson = resultSet.getString(CONTAINER_INFO);
        Timestamp lastModifiedTime = resultSet.getTimestamp(LAST_MODIFIED_TIME);
        int version = resultSet.getInt(VERSION);
        try {
            Container deserialized = objectMapper.readValue(containerJson, Container.class);
            Container container = new ContainerBuilder(deserialized).setParentAccountId((short) accountId).setLastModifiedTime(lastModifiedTime.getTime()).setSnapshotVersion(version).build();
            containers.add(container);
        } catch (IOException e) {
            throw new SQLException(e);
        }
    }
    return containers;
}

Example 10 with ContainerBuilder

use of com.github.ambry.account.ContainerBuilder in project ambry by linkedin.

the class FrontendTestUrlSigningServiceFactory method injectAccountAndContainerForPutAndVerify.

/**
 * Puts blobs and verify injected target {@link Account} and {@link Container}.
 * @param container the {@link Container} to use.
 * @param shouldAllowServiceIdBasedPut {@code true} if PUT requests with serviceId parsed as {@link Account} name is
 *                                                 allowed; {@code false} otherwise.
 * @throws Exception
 */
private void injectAccountAndContainerForPutAndVerify(Container container, boolean shouldAllowServiceIdBasedPut) throws Exception {
    configProps.setProperty("frontend.allow.service.id.based.post.request", String.valueOf(shouldAllowServiceIdBasedPut));
    verifiableProperties = new VerifiableProperties(configProps);
    frontendConfig = new FrontendConfig(verifiableProperties);
    accountAndContainerInjector = new AccountAndContainerInjector(accountService, frontendMetrics, frontendConfig);
    ambryBlobStorageService = getAmbryBlobStorageService();
    ambryBlobStorageService.start();
    populateAccountService();
    // should succeed when serviceId-based PUT requests are allowed.
    postBlobAndVerifyWithAccountAndContainer(null, null, "serviceId", !container.isCacheable(), shouldAllowServiceIdBasedPut ? InMemAccountService.UNKNOWN_ACCOUNT : null, shouldAllowServiceIdBasedPut ? (container.isCacheable() ? Container.DEFAULT_PUBLIC_CONTAINER : Container.DEFAULT_PRIVATE_CONTAINER) : null, shouldAllowServiceIdBasedPut ? null : RestServiceErrorCode.BadRequest);
    // should fail, because accountName needs to be specified.
    postBlobAndVerifyWithAccountAndContainer(null, "dummyContainerName", "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    // should fail, because account name from serviceId could not be located in account service.
    postBlobAndVerifyWithAccountAndContainer(null, Container.UNKNOWN_CONTAINER_NAME, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidContainer);
    // should fail, because accountName needs to be specified.
    postBlobAndVerifyWithAccountAndContainer(null, refContainer.getName(), "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    // should fail, because accountName is not allowed.
    postBlobAndVerifyWithAccountAndContainer(Account.UNKNOWN_ACCOUNT_NAME, null, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName is not allowed.
    postBlobAndVerifyWithAccountAndContainer(Account.UNKNOWN_ACCOUNT_NAME, "dummyContainerName", "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName is not allowed.
    postBlobAndVerifyWithAccountAndContainer(Account.UNKNOWN_ACCOUNT_NAME, Container.UNKNOWN_CONTAINER_NAME, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName is not allowed.
    postBlobAndVerifyWithAccountAndContainer(Account.UNKNOWN_ACCOUNT_NAME, refContainer.getName(), "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because container name needs to be specified
    postBlobAndVerifyWithAccountAndContainer(refAccount.getName(), null, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    // should fail, because containerName does not exist.
    postBlobAndVerifyWithAccountAndContainer(refAccount.getName(), "dummyContainerName", "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidContainer);
    // should fail, because containerName is not allowed.
    postBlobAndVerifyWithAccountAndContainer(refAccount.getName(), Container.UNKNOWN_CONTAINER_NAME, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidContainer);
    // should succeed.
    String blobIdStr = postBlobAndVerifyWithAccountAndContainer(refAccount.getName(), refContainer.getName(), "serviceId", !container.isCacheable(), refAccount, refContainer, null);
    // should succeed.
    verifyAccountAndContainerFromBlobId(blobIdStr, refAccount, refContainer, null);
    // should fail, because containerName needs to be specified.
    postBlobAndVerifyWithAccountAndContainer("dummyAccountName", null, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    // should fail, because accountName does not exist.
    postBlobAndVerifyWithAccountAndContainer("dummyAccountName", "dummyContainerName", "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because container name is now allowed.
    postBlobAndVerifyWithAccountAndContainer("dummyAccountName", Container.UNKNOWN_CONTAINER_NAME, "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidContainer);
    // should fail, because accountName does not exist.
    postBlobAndVerifyWithAccountAndContainer("dummyAccountName", refContainer.getName(), "serviceId", !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName implicitly set by serviceId is not allowed.
    postBlobAndVerifyWithAccountAndContainer(null, null, Account.UNKNOWN_ACCOUNT_NAME, !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName implicitly set by serviceId is not allowed.
    postBlobAndVerifyWithAccountAndContainer(null, "dummyContainerName", Account.UNKNOWN_ACCOUNT_NAME, !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName implicitly set by serviceId is not allowed.
    postBlobAndVerifyWithAccountAndContainer(null, Container.UNKNOWN_CONTAINER_NAME, Account.UNKNOWN_ACCOUNT_NAME, !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should fail, because accountName implicitly set by serviceId is not allowed.
    postBlobAndVerifyWithAccountAndContainer(null, refContainer.getName(), Account.UNKNOWN_ACCOUNT_NAME, !container.isCacheable(), null, null, RestServiceErrorCode.InvalidAccount);
    // should succeed if the serviceId-based PUT requests are allowed, but this is a special case that account is
    // created without the legacy containers for public and private put.
    postBlobAndVerifyWithAccountAndContainer(null, null, refAccount.getName(), !container.isCacheable(), shouldAllowServiceIdBasedPut ? refAccount : null, shouldAllowServiceIdBasedPut ? (container.isCacheable() ? refDefaultPublicContainer : refDefaultPrivateContainer) : null, shouldAllowServiceIdBasedPut ? null : RestServiceErrorCode.BadRequest);
    // should fail, because accountName needs to be specified.
    postBlobAndVerifyWithAccountAndContainer(null, "dummyContainerName", refAccount.getName(), !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    // should fail, because accountName implicitly set by serviceId does not have the default container.
    postBlobAndVerifyWithAccountAndContainer(null, Container.UNKNOWN_CONTAINER_NAME, refAccount.getName(), !container.isCacheable(), null, null, RestServiceErrorCode.InvalidContainer);
    // should fail, because accountName needs to be specified.
    postBlobAndVerifyWithAccountAndContainer(null, refContainer.getName(), refAccount.getName(), !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    Container legacyContainerForPublicBlob = new ContainerBuilder(Container.DEFAULT_PUBLIC_CONTAINER_ID, "containerForLegacyPublicPut", Container.ContainerStatus.ACTIVE, "This is a container for putting legacy public blob", false, false, false, false, refAccount.getId()).build();
    Container legacyContainerForPrivateBlob = new ContainerBuilder(Container.DEFAULT_PRIVATE_CONTAINER_ID, "containerForLegacyPrivatePut", Container.ContainerStatus.ACTIVE, "This is a container for putting legacy private blob", false, false, true, false, refAccount.getId()).build();
    Account accountWithTwoDefaultContainers = new AccountBuilder(refAccount).addOrUpdateContainer(legacyContainerForPrivateBlob).addOrUpdateContainer(legacyContainerForPublicBlob).build();
    accountService.updateAccounts(Collections.singletonList(accountWithTwoDefaultContainers));
    if (!container.isCacheable()) {
        // should succeed if serviceId-based PUT requests are allowed.
        postBlobAndVerifyWithAccountAndContainer(null, null, accountWithTwoDefaultContainers.getName(), !container.isCacheable(), shouldAllowServiceIdBasedPut ? accountWithTwoDefaultContainers : null, shouldAllowServiceIdBasedPut ? accountWithTwoDefaultContainers.getContainerById(Container.DEFAULT_PRIVATE_CONTAINER_ID) : null, shouldAllowServiceIdBasedPut ? null : RestServiceErrorCode.BadRequest);
        // should fail, because accountName needs to be specified.
        postBlobAndVerifyWithAccountAndContainer(null, "dummyContainerName", accountWithTwoDefaultContainers.getName(), !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    } else {
        // should succeed if serviceId-based PUT requests are allowed.
        postBlobAndVerifyWithAccountAndContainer(null, null, accountWithTwoDefaultContainers.getName(), !container.isCacheable(), shouldAllowServiceIdBasedPut ? accountWithTwoDefaultContainers : null, shouldAllowServiceIdBasedPut ? accountWithTwoDefaultContainers.getContainerById(Container.DEFAULT_PUBLIC_CONTAINER_ID) : null, shouldAllowServiceIdBasedPut ? null : RestServiceErrorCode.BadRequest);
        // should fail, because accountName needs to be specified.
        postBlobAndVerifyWithAccountAndContainer(null, "dummyContainerName", accountWithTwoDefaultContainers.getName(), !container.isCacheable(), null, null, RestServiceErrorCode.MissingArgs);
    }
}