Examples with MySqlAccountStoreFactory com.github.ambry.account.mysql.MySqlAccountStoreFactory used on opensource projects

Example 1 with MySqlAccountStoreFactory

use of com.github.ambry.account.mysql.MySqlAccountStoreFactory in project ambry by linkedin.

the class DatabaseTest method perfTest.

private static void perfTest(VerifiableProperties verifiableProperties) throws Exception {
    MySqlDataAccessor dataAccessor = new MySqlAccountStoreFactory(verifiableProperties, new MetricRegistry()).getMySqlAccountStore().getMySqlDataAccessor();
    AccountDao accountDao = new AccountDao(dataAccessor);
    // Use high account id to avoid conflict
    short startAccountId = 30000;
    int numAccounts = 10;
    int numContainers = 1000;
    cleanup(dataAccessor.getDatabaseConnection(true), startAccountId);
    ContainerBuilder builder = new ContainerBuilder((short) 0, "", Container.ContainerStatus.ACTIVE, "Test", (short) 0);
    long t0 = System.currentTimeMillis();
    int containersAdded = 0;
    List<AccountUpdateInfo> accountUpdateInfos = new ArrayList<>();
    for (short accountId = startAccountId; accountId < startAccountId + numAccounts; accountId++) {
        Account account = new AccountBuilder(accountId, "Account-" + accountId, Account.AccountStatus.ACTIVE).build();
        List<Container> containers = new ArrayList<>();
        for (short containerId = 1; containerId <= numContainers; containerId++) {
            containers.add(builder.setId(containerId).setParentAccountId(accountId).setName("Container-" + containerId).setTtlRequired(true).build());
            containersAdded++;
        }
        accountUpdateInfos.add(new AccountUpdateInfo(account, true, false, containers, new ArrayList<>()));
    }
    accountDao.updateAccounts(accountUpdateInfos, 100);
    long t1 = System.currentTimeMillis();
    long insertTime = t1 - t0;
    logger.info("Added {} containers in {} ms", containersAdded, insertTime);
    // Query containers since t0 (should be all)
    List<Container> allContainers = accountDao.getNewContainers(t0);
    long t2 = System.currentTimeMillis();
    logger.info("Queried {} containers in {} ms", allContainers.size(), t2 - t1);
    // Query containers since t2 (should be none)
    allContainers = accountDao.getNewContainers(t2);
    long t3 = System.currentTimeMillis();
    logger.info("Queried {} containers in {} ms", allContainers.size(), t3 - t2);
}

Example 2 with MySqlAccountStoreFactory

use of com.github.ambry.account.mysql.MySqlAccountStoreFactory in project ambry by linkedin.

the class MySqlAccountServiceIntegrationTest method testFailover.

@Test
public void testFailover() throws Exception {
    String dbInfoJsonString = mySqlConfigProps.getProperty(DB_INFO);
    JSONArray dbInfo = new JSONArray(dbInfoJsonString);
    JSONObject entry = dbInfo.getJSONObject(0);
    DbEndpoint origEndpoint = DbEndpoint.fromJson(entry);
    // Make two endpoints, one with bad url, writeable, remote; second with good url, writeable, local
    String localDc = "local", remoteDc = "remote";
    String badUrl = "jdbc:mysql://badhost/AccountMetadata";
    DbEndpoint localGoodEndpoint = new DbEndpoint(origEndpoint.getUrl(), localDc, false, origEndpoint.getUsername(), origEndpoint.getPassword());
    DbEndpoint remoteBadEndpoint = new DbEndpoint(badUrl, remoteDc, true, origEndpoint.getUsername(), origEndpoint.getPassword());
    JSONArray endpointsJson = new JSONArray().put(localGoodEndpoint.toJson()).put(remoteBadEndpoint.toJson());
    mySqlConfigProps.setProperty(DB_INFO, endpointsJson.toString());
    mySqlConfigProps.setProperty(ClusterMapConfig.CLUSTERMAP_DATACENTER_NAME, localDc);
    mySqlAccountStore = spy(new MySqlAccountStoreFactory(new VerifiableProperties(mySqlConfigProps), new MetricRegistry()).getMySqlAccountStore());
    when(mockMySqlAccountStoreFactory.getMySqlAccountStore()).thenReturn(mySqlAccountStore);
    // constructor does initial fetch which will fail on first endpoint and succeed on second
    AccountServiceMetrics accountServiceMetrics = new AccountServiceMetrics(new MetricRegistry());
    mySqlAccountService = new MySqlAccountService(accountServiceMetrics, accountServiceConfig, mockMySqlAccountStoreFactory, mockNotifier);
    MySqlMetrics storeMetrics = mySqlAccountStore.getMySqlDataAccessor().getMetrics();
    // At this point, should have at least one connection failure (bad endpoint) and one success
    long expectedConnectionFail = storeMetrics.connectionFailureCount.getCount();
    long expectedConnectionSuccess = storeMetrics.connectionSuccessCount.getCount();
    assertTrue(expectedConnectionFail > 0);
    assertTrue(expectedConnectionSuccess > 0);
    // Try to update, should fail to get connection
    Account account = makeTestAccountWithContainer();
    try {
        mySqlAccountService.updateAccounts(Collections.singletonList(account));
        fail("Expected failure due to no writeable accounts");
    } catch (AccountServiceException ase) {
        assertEquals(AccountServiceErrorCode.InternalError, ase.getErrorCode());
    }
    expectedConnectionFail++;
    assertEquals(1, accountServiceMetrics.updateAccountErrorCount.getCount());
    assertEquals(expectedConnectionFail, storeMetrics.connectionFailureCount.getCount());
    assertEquals(expectedConnectionSuccess, storeMetrics.connectionSuccessCount.getCount());
    mySqlAccountService.fetchAndUpdateCache();
}

Example 3 with MySqlAccountStoreFactory

use of com.github.ambry.account.mysql.MySqlAccountStoreFactory in project ambry by linkedin.

the class MySqlAccountServiceIntegrationTest method testContainerFetchOnDemand.

/**
 * Container on-demand fetch for multiple account services.
 */
@Test
public void testContainerFetchOnDemand() throws Exception {
    MySqlAccountService producerAccountService = mySqlAccountService;
    // Create second account service with scheduled polling disabled
    mySqlConfigProps.setProperty(UPDATER_POLLING_INTERVAL_SECONDS, "0");
    accountServiceConfig = new MySqlAccountServiceConfig(new VerifiableProperties(mySqlConfigProps));
    MySqlAccountStore consumerAccountStore = spy(new MySqlAccountStoreFactory(new VerifiableProperties(mySqlConfigProps), new MetricRegistry()).getMySqlAccountStore());
    MySqlAccountStoreFactory mockMySqlAccountStoreFactory = mock(MySqlAccountStoreFactory.class);
    when(mockMySqlAccountStoreFactory.getMySqlAccountStore()).thenReturn(consumerAccountStore);
    AccountServiceMetrics accountServiceMetrics = new AccountServiceMetrics(new MetricRegistry());
    // Note: for these tests, consumer must NOT be notified of changes
    MySqlAccountService consumerAccountService = new MySqlAccountService(accountServiceMetrics, accountServiceConfig, mockMySqlAccountStoreFactory, null);
    // Add new account "a1" on producer account service
    short accountId = 101;
    String accountName = "a1";
    int onDemandContainerFetchCount = 0;
    Container container = new ContainerBuilder((short) 1, "c1", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build();
    Account account = new AccountBuilder(accountId, accountName, Account.AccountStatus.ACTIVE).containers(Collections.singletonList(container)).build();
    producerAccountService.updateAccounts(Collections.singletonList(account));
    // Test getContainer on consumer account service should fail since account doesn't exist
    assertNull("Expected null since Account doesn't exist in consumerAccountService cache", consumerAccountService.getContainerByName(accountName, "c1"));
    assertNull("Expected null since Account doesn't exist in consumerAccountService cache", consumerAccountService.getContainerById(accountId, (short) 1));
    // Fetch and update cache in consumer account service
    consumerAccountService.fetchAndUpdateCache();
    assertEquals("Account mismatch", account, consumerAccountService.getAccountByName(accountName));
    assertEquals("Container mismatch", container, consumerAccountService.getContainerByName(accountName, "c1"));
    assertEquals("Container mismatch", container, consumerAccountService.getContainerById(accountId, (short) 1));
    // Add new container in producer account service
    producerAccountService.updateContainers(accountName, Collections.singletonList(new ContainerBuilder((short) -1, "c2", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build()));
    Container newContainer = producerAccountService.getContainerByName(accountName, "c2");
    // Test getContainerByName() on consumer account service is successful (by fetching from mysql on demand)
    assertEquals("getContainerByName() expected to fetch container from MySql db", newContainer, consumerAccountService.getContainerByName(accountName, "c2"));
    verify(consumerAccountStore).getContainerByName(eq((int) accountId), eq(newContainer.getName()));
    assertEquals("Number of on-demand container requests should be 1", ++onDemandContainerFetchCount, accountServiceMetrics.onDemandContainerFetchCount.getCount());
    // verify in-memory cache is updated with the fetched container "c2"
    assertEquals("Container c2 should be present in consumer account service", newContainer, consumerAccountService.getAccountByName(accountName).getContainerByName(newContainer.getName()));
    // Add new container in producer account service
    producerAccountService.updateContainers(accountName, Collections.singletonList(new ContainerBuilder((short) -1, "c3", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build()));
    newContainer = producerAccountService.getContainerById(accountId, (short) 3);
    // Test getContainerById() on consumer account service is successful (by fetching from mysql on demand)
    assertEquals("getContainerById() expected to fetch container from MySql db", newContainer, consumerAccountService.getContainerById(accountId, (short) 3));
    verify(consumerAccountStore).getContainerById(eq((int) accountId), eq((int) newContainer.getId()));
    assertEquals("Number of on-demand container requests should be 2", ++onDemandContainerFetchCount, accountServiceMetrics.onDemandContainerFetchCount.getCount());
    // verify in-memory cache is updated with the fetched container "c3"
    assertEquals("Container c3 should be present in consumer account service", newContainer, consumerAccountService.getAccountByName(accountName).getContainerById(newContainer.getId()));
}

Example 4 with MySqlAccountStoreFactory

use of com.github.ambry.account.mysql.MySqlAccountStoreFactory in project ambry by linkedin.

the class MySqlAccountServiceIntegrationTest method testAccountRefresh.

/**
 * Producer-consumer test for multiple account services.
 */
@Test
public void testAccountRefresh() throws Exception {
    MySqlAccountService producerAccountService = mySqlAccountService;
    MySqlAccountStore producerAccountStore = mySqlAccountStore;
    // Create second account service with scheduled polling disabled
    mySqlConfigProps.setProperty(UPDATER_POLLING_INTERVAL_SECONDS, "0");
    accountServiceConfig = new MySqlAccountServiceConfig(new VerifiableProperties(mySqlConfigProps));
    MySqlAccountStore consumerAccountStore = spy(new MySqlAccountStoreFactory(new VerifiableProperties(mySqlConfigProps), new MetricRegistry()).getMySqlAccountStore());
    MySqlAccountStoreFactory mockMySqlAccountStoreFactory = mock(MySqlAccountStoreFactory.class);
    when(mockMySqlAccountStoreFactory.getMySqlAccountStore()).thenReturn(consumerAccountStore);
    AccountServiceMetrics accountServiceMetrics = new AccountServiceMetrics(new MetricRegistry());
    MySqlAccountService consumerAccountService = new MySqlAccountService(accountServiceMetrics, accountServiceConfig, mockMySqlAccountStoreFactory, mockNotifier);
    // Add account with 3 containers
    short accountId = 101;
    String accountName = "a1";
    // Number of calls expected in producer account store
    List<Container> containers = new ArrayList<>();
    containers.add(new ContainerBuilder((short) 1, "c1", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    containers.add(new ContainerBuilder((short) 2, "c2", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    containers.add(new ContainerBuilder((short) 3, "c3", ContainerStatus.ACTIVE, DESCRIPTION, accountId).build());
    Account a1 = new AccountBuilder(accountId, accountName, Account.AccountStatus.ACTIVE).containers(containers).build();
    producerAccountService.updateAccounts(Collections.singletonList(a1));
    Account finalA = a1;
    verify(producerAccountStore).updateAccounts(argThat(accountInfos -> {
        AccountUpdateInfo accountInfo = accountInfos.get(0);
        return accountInfo.getAccount().equals(finalA) && accountInfo.getAddedContainers().equals(containers) && accountInfo.isAdded() && !accountInfo.isUpdated() && accountInfo.getUpdatedContainers().isEmpty();
    }));
    // Note: because consumer is notified of changes, its cache should already be in sync with DB
    long lmt = consumerAccountService.accountInfoMapRef.get().getLastModifiedTime();
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    // Update account only
    String newAccountName = "a1-updated";
    a1 = new AccountBuilder(a1).name(newAccountName).build();
    producerAccountService.updateAccounts(Collections.singletonList(a1));
    Account finalA1 = a1;
    verify(producerAccountStore).updateAccounts(argThat(accountInfos -> {
        AccountUpdateInfo accountInfo = accountInfos.get(0);
        return accountInfo.getAccount().equals(finalA1) && accountInfo.getAddedContainers().isEmpty() && !accountInfo.isAdded() && accountInfo.isUpdated() && accountInfo.getUpdatedContainers().isEmpty();
    }));
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(newAccountName));
    assertNull("Expected no account with old name", consumerAccountService.getAccountByName(accountName));
    accountName = newAccountName;
    // Update container only
    Container c1Mod = new ContainerBuilder(containers.get(0)).setStatus(ContainerStatus.DELETE_IN_PROGRESS).build();
    containers.set(0, c1Mod);
    a1 = new AccountBuilder(a1).containers(containers).build();
    Collection<Container> result = producerAccountService.updateContainers(accountName, Collections.singletonList(c1Mod));
    // Account should not have been touched
    verify(producerAccountStore).updateAccounts(argThat(accountsInfo -> accountsInfo.get(0).getUpdatedContainers().size() == 1));
    assertEquals("Expected one result", 1, result.size());
    assertEquals("Container mismatch", c1Mod, result.iterator().next());
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Container mismatch", c1Mod, consumerAccountService.getContainerByName(accountName, "c1"));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    lmt = consumerAccountService.accountInfoMapRef.get().getLastModifiedTime();
    // Add container only
    Container cNew = makeNewContainer("c4", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNew));
    verify(producerAccountStore).updateAccounts(argThat(accountsInfo -> accountsInfo.get(0).getAddedContainers().size() == 1));
    assertEquals("Expected one result", 1, result.size());
    cNew = result.iterator().next();
    containers.add(cNew);
    a1 = new AccountBuilder(a1).containers(containers).build();
    verify(consumerAccountStore).getNewAccounts(eq(lmt));
    verify(consumerAccountStore).getNewContainers(eq(lmt));
    assertEquals("Container mismatch", cNew, consumerAccountService.getContainerByName(accountName, "c4"));
    assertEquals("Account mismatch", a1, consumerAccountService.getAccountByName(accountName));
    // For this section, consumer must get out of date so need to unsubscribe its notifier
    mockNotifier.unsubscribeAll();
    // TODO:
    // Add container in AS1, call AS2.getContainer() to force fetch
    // Add C1 in AS1, add C1 in AS2 (should succeed and return existing id)
    Container cNewProd = makeNewContainer("c5", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    short newId = result.iterator().next().getId();
    // Add the same container to second AS with stale cache
    // Expect it to fail first time (conflict), refresh cache and succeed on retry
    result = consumerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    assertEquals(newId, result.iterator().next().getId());
    assertEquals(1, accountServiceMetrics.updateAccountErrorCount.getCount());
    assertEquals(1, accountServiceMetrics.conflictRetryCount.getCount());
    // Add C1 in AS1, add C2 in AS2
    cNewProd = makeNewContainer("c6", accountId, ContainerStatus.ACTIVE);
    result = producerAccountService.updateContainers(accountName, Collections.singletonList(cNewProd));
    newId = result.iterator().next().getId();
    Container cNewCons = makeNewContainer("c7", accountId, ContainerStatus.ACTIVE);
    result = consumerAccountService.updateContainers(accountName, Collections.singletonList(cNewCons));
    assertNotSame(newId, result.iterator().next().getId());
    assertEquals(2, accountServiceMetrics.updateAccountErrorCount.getCount());
    assertEquals(2, accountServiceMetrics.conflictRetryCount.getCount());
    // Check gauge values
    assertTrue("Sync time not updated", accountServiceMetrics.timeInSecondsSinceLastSyncGauge.getValue() < 10);
    assertEquals("Unexpected container count", 7, accountServiceMetrics.containerCountGauge.getValue().intValue());
}