Examples with AccountUpdateInfo com.github.ambry.account.AccountUtils.AccountUpdateInfo used on opensource projects

Example 1 with AccountUpdateInfo

use of com.github.ambry.account.AccountUtils.AccountUpdateInfo in project ambry by linkedin.

the class MySqlAccountsDBTool method initialize.

/**
 * Initializes db from zk
 */
public void initialize() throws SQLException {
    // clean the account and container tables in DB
    cleanup();
    // get the list of accounts from zk in the form of map account id -> account json (as string)
    long startTimeMs = SystemTime.getInstance().milliseconds();
    Map<String, String> accountMap = fetchAccountMetadataFromZK();
    if (accountMap == null) {
        logger.info("Account metadata in ZK is empty");
        return;
    }
    long zkFetchTimeMs = SystemTime.getInstance().milliseconds();
    logger.info("Fetched account metadata from zk path={}, took time={} ms", fullZKAccountMetadataPath, zkFetchTimeMs - startTimeMs);
    AccountInfoMap accountInfoMap = new AccountInfoMap(new AccountServiceMetrics(new MetricRegistry()), accountMap);
    // Populate Account and Container tables in batches
    List<AccountUpdateInfo> accountUpdateInfos = new ArrayList<>();
    for (Account account : accountInfoMap.getAccounts()) {
        accountUpdateInfos.add(new AccountUpdateInfo(account, true, false, new ArrayList<>(account.getAllContainers()), new ArrayList<>()));
    }
    mySqlAccountStore.updateAccounts(accountUpdateInfos);
    logger.info("Initialized account metadata in DB from ZK path {}, took time={} ms", fullZKAccountMetadataPath, System.currentTimeMillis() - zkFetchTimeMs);
}

Example 2 with AccountUpdateInfo

use of com.github.ambry.account.AccountUtils.AccountUpdateInfo in project ambry by linkedin.

the class DatabaseTest method perfTest.

private static void perfTest(VerifiableProperties verifiableProperties) throws Exception {
    MySqlDataAccessor dataAccessor = new MySqlAccountStoreFactory(verifiableProperties, new MetricRegistry()).getMySqlAccountStore().getMySqlDataAccessor();
    AccountDao accountDao = new AccountDao(dataAccessor);
    // Use high account id to avoid conflict
    short startAccountId = 30000;
    int numAccounts = 10;
    int numContainers = 1000;
    cleanup(dataAccessor.getDatabaseConnection(true), startAccountId);
    ContainerBuilder builder = new ContainerBuilder((short) 0, "", Container.ContainerStatus.ACTIVE, "Test", (short) 0);
    long t0 = System.currentTimeMillis();
    int containersAdded = 0;
    List<AccountUpdateInfo> accountUpdateInfos = new ArrayList<>();
    for (short accountId = startAccountId; accountId < startAccountId + numAccounts; accountId++) {
        Account account = new AccountBuilder(accountId, "Account-" + accountId, Account.AccountStatus.ACTIVE).build();
        List<Container> containers = new ArrayList<>();
        for (short containerId = 1; containerId <= numContainers; containerId++) {
            containers.add(builder.setId(containerId).setParentAccountId(accountId).setName("Container-" + containerId).setTtlRequired(true).build());
            containersAdded++;
        }
        accountUpdateInfos.add(new AccountUpdateInfo(account, true, false, containers, new ArrayList<>()));
    }
    accountDao.updateAccounts(accountUpdateInfos, 100);
    long t1 = System.currentTimeMillis();
    long insertTime = t1 - t0;
    logger.info("Added {} containers in {} ms", containersAdded, insertTime);
    // Query containers since t0 (should be all)
    List<Container> allContainers = accountDao.getNewContainers(t0);
    long t2 = System.currentTimeMillis();
    logger.info("Queried {} containers in {} ms", allContainers.size(), t2 - t1);
    // Query containers since t2 (should be none)
    allContainers = accountDao.getNewContainers(t2);
    long t3 = System.currentTimeMillis();
    logger.info("Queried {} containers in {} ms", allContainers.size(), t3 - t2);
}

Example 3 with AccountUpdateInfo

use of com.github.ambry.account.AccountUtils.AccountUpdateInfo in project ambry by linkedin.

the class MySqlAccountServiceIntegrationTest method testInitCacheOnStartUp.

/**
 * Tests in-memory cache is updated with accounts from mysql db store on start up
 */
@Test
public void testInitCacheOnStartUp() throws Exception {
    Account testAccount = makeTestAccountWithContainer();
    AccountUpdateInfo accountUpdateInfo = new AccountUpdateInfo(testAccount, true, false, new ArrayList<>(testAccount.getAllContainers()), new ArrayList<>());
    mySqlAccountStore.updateAccounts(Collections.singletonList(accountUpdateInfo));
    mySqlAccountService = getAccountService();
    // Test in-memory cache is updated with accounts from mysql store on start up.
    List<Account> accounts = new ArrayList<>(mySqlAccountService.getAllAccounts());
    assertEquals("Mismatch in number of accounts", 1, accounts.size());
    assertEquals("Mismatch in account information", testAccount, accounts.get(0));
}

Example 4 with AccountUpdateInfo

use of com.github.ambry.account.AccountUtils.AccountUpdateInfo in project ambry by linkedin.

the class MySqlAccountServiceIntegrationTest method testAccountRefresh.

/**
 * Producer-consumer test for multiple account services.
 */
@Test
public void testAccountRefresh() throws Exception {
    MySqlAccountService producerAccountService = mySqlAccountService;
    MySqlAccountStore producerAccountStore = mySqlAccountStore;
    // Create second account service with scheduled polling disabled
    mySqlConfigProps.setProperty(UPDATER_POLLING_INTERVAL_SECONDS, "0");
    accountServiceConfig = new MySqlAccountServiceConfig(new VerifiableProperties(mySqlConfigProps));
    MySqlAccountStore consumerAccountStore = spy(new MySqlAccountStoreFactory(new VerifiableProperties(mySqlConfigProps), new MetricRegistry()).getMySqlAccountStore());
    MySqlAccountStoreFactory mockMySqlAccountStoreFactory = mock(MySqlAccountStoreFactory.class);
    when(mockMySqlAccountStoreFactory.getMySqlAccountStore()).thenReturn(consumerAccountStore);
    AccountServiceMetrics accountServiceMetrics = new AccountServiceMetrics(new MetricRegistry());
    MySqlAccountService consumerAccountService = new MySqlAccountService(accountServiceMetrics, accountServiceConfig, mockMySqlAccountStoreFactory, mockNotifier);
    // Add account with 3 containers
    short accountId = 101;
    String accountName = "a1";
    // Number of calls expected in producer account store
    List<Container> containers = new ArrayList<>();
    containers.add(new ContainerBuilder((short) 1, "c1", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    containers.add(new ContainerBuilder((short) 2, "c2", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    containers.add(new ContainerBuilder((short) 3, "c3", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    Account a1 = new AccountBuilder(accountId, accountName, Account.AccountStatus.ACTIVE).containers(containers).build();
    producerAccountService.updateAccounts(Collections.singletonList(a1));
    Account finalA = a1;
    verify(producerAccountStore).updateAccounts(argThat(accountInfos -> {
        AccountUpdateInfo accountInfo = accountInfos.get(0);
        return accountInfo.getAccount().equals(finalA) && accountInfo.getAddedContainers().equals(containers) && accountInfo.isAdded() && !accountInfo.isUpdated() && accountInfo.getUpdatedContainers().isEmpty();
    }));
    // Note: because consumer is notified of changes, its cache should already be in sync with DB
    long lmt = consumerAccountService.accountInfoMapRef.get().getLastModifiedTime();
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    // Update account only
    String newAccountName = "a1-updated";
    a1 = new AccountBuilder(a1).name(newAccountName).build();
    producerAccountService.updateAccounts(Collections.singletonList(a1));
    Account finalA1 = a1;
    verify(producerAccountStore).updateAccounts(argThat(accountInfos -> {
        AccountUpdateInfo accountInfo = accountInfos.get(0);
        return accountInfo.getAccount().equals(finalA1) && accountInfo.getAddedContainers().isEmpty() && !accountInfo.isAdded() && accountInfo.isUpdated() && accountInfo.getUpdatedContainers().isEmpty();
    }));
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(newAccountName));
    assertNull("Expected no account with old name", consumerAccountService.getAccountByName(accountName));
    accountName = newAccountName;
    // Update container only
    Container c1Mod = new ContainerBuilder(containers.get(0)).setStatus(ContainerStatus.DELETE_IN_PROGRESS).build();
    containers.set(0, c1Mod);
    a1 = new AccountBuilder(a1).containers(containers).build();
    Collection<Container> result = producerAccountService.updateContainers(accountName, Collections.singletonList(c1Mod));
    // Account should not have been touched
    verify(producerAccountStore).updateAccounts(argThat(accountsInfo -> accountsInfo.get(0).getUpdatedContainers().size() == 1));
    assertEquals("Expected one result", 1, result.size());
    assertEquals("Container mismatch", c1Mod, result.iterator().next());
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Container mismatch", c1Mod, consumerAccountService.getContainerByName(accountName, "c1"));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    lmt = consumerAccountService.accountInfoMapRef.get().getLastModifiedTime();
    // Add container only
    Container cNew = makeNewContainer("c4", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNew));
    verify(producerAccountStore).updateAccounts(argThat(accountsInfo -> accountsInfo.get(0).getAddedContainers().size() == 1));
    assertEquals("Expected one result", 1, result.size());
    cNew = result.iterator().next();
    containers.add(cNew);
    a1 = new AccountBuilder(a1).containers(containers).build();
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Container mismatch", cNew, consumerAccountService.getContainerByName(accountName, "c4"));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    // For this section, consumer must get out of date so need to unsubscribe its notifier
    mockNotifier.unsubscribeAll();
    // TODO:
    // Add container in AS1, call AS2.getContainer() to force fetch
    // Add C1 in AS1, add C1 in AS2 (should succeed and return existing id)
    Container cNewProd = makeNewContainer("c5", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    short newId = result.iterator().next().getId();
    // Add the same container to second AS with stale cache
    // Expect it to fail first time (conflict), refresh cache and succeed on retry
    result = consumerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    assertEquals(newId, result.iterator().next().getId());
    assertEquals(1, accountServiceMetrics.updateAccountErrorCount.getCount());
    assertEquals(1, accountServiceMetrics.conflictRetryCount.getCount());
    // Add C1 in AS1, add C2 in AS2
    cNewProd = makeNewContainer("c6", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    newId = result.iterator().next().getId();
    Container cNewCons = makeNewContainer("c7", accountId, ContainerStatus.ACTIVE);
    result = consumerAccountService.updateContainers(accountName, Collections.singletonList(cNewCons));
    assertNotSame(newId, result.iterator().next().getId());
    assertEquals(2, accountServiceMetrics.updateAccountErrorCount.getCount());
    assertEquals(2, accountServiceMetrics.conflictRetryCount.getCount());
    // Check gauge values
    assertTrue("Sync time not updated", accountServiceMetrics.timeInSecondsSinceLastSyncGauge.getValue() < 10);
    assertEquals("Unexpected container count", 7, accountServiceMetrics.containerCountGauge.getValue().intValue());
}

Example 5 with AccountUpdateInfo

use of com.github.ambry.account.AccountUtils.AccountUpdateInfo in project ambry by linkedin.

the class AccountDaoTest method testBatchOperations.

@Test
public void testBatchOperations() throws SQLException {
    List<AccountUpdateInfo> accountUpdateInfos = new ArrayList<>();
    int size = 11;
    int batchSize = 5;
    // test batch account inserts
    for (int i = 1; i <= size; i++) {
        Account account = new AccountBuilder((short) i, "test account " + i, Account.AccountStatus.ACTIVE).build();
        accountUpdateInfos.add(new AccountUpdateInfo(account, true, false, new ArrayList<>(), new ArrayList<>()));
    }
    accountDao.updateAccounts(accountUpdateInfos, batchSize);
    verify(mockAccountInsertStatement, times(size)).addBatch();
    verify(mockAccountInsertStatement, times(size / batchSize + 1)).executeBatch();
    // test batch account updates
    accountUpdateInfos.clear();
    for (int i = 1; i <= size; i++) {
        Account account = new AccountBuilder((short) i, "test account " + i, Account.AccountStatus.ACTIVE).snapshotVersion(1).build();
        accountUpdateInfos.add(new AccountUpdateInfo(account, false, true, new ArrayList<>(), new ArrayList<>()));
    }
    accountDao.updateAccounts(accountUpdateInfos, batchSize);
    verify(mockAccountUpdateStatement, times(size)).addBatch();
    verify(mockAccountUpdateStatement, times(size / batchSize + 1)).executeBatch();
    Account account = new AccountBuilder((short) 1, "test account " + 1, Account.AccountStatus.ACTIVE).build();
    List<Container> containers = new ArrayList<>();
    for (int i = 1; i <= size; i++) {
        containers.add(new ContainerBuilder((short) i, "test container " + i, Container.ContainerStatus.ACTIVE, "", (short) 1).build());
    }
    // test batch container inserts
    accountUpdateInfos.clear();
    accountUpdateInfos.add(new AccountUpdateInfo(account, false, false, containers, new ArrayList<>()));
    accountDao.updateAccounts(accountUpdateInfos, batchSize);
    verify(mockContainerInsertStatement, times(size)).addBatch();
    // Execute batch should be invoked only once since all containers belong to same account
    verify(mockContainerInsertStatement, times(1)).executeBatch();
    // test batch container updates
    accountUpdateInfos.clear();
    accountUpdateInfos.add(new AccountUpdateInfo(account, false, false, new ArrayList<>(), containers));
    accountDao.updateAccounts(accountUpdateInfos, batchSize);
    verify(mockContainerUpdateStatement, times(size)).addBatch();
    // Execute batch should be invoked only once since all containers belong to same account
    verify(mockContainerUpdateStatement, times(1)).executeBatch();
}