Examples with AccountStatsMySqlStore com.github.ambry.accountstats.AccountStatsMySqlStore used on opensource projects

Example 1 with AccountStatsMySqlStore

use of com.github.ambry.accountstats.AccountStatsMySqlStore in project ambry by linkedin.

the class AmbryServer method startup.

public void startup() throws InstantiationException {
    try {
        logger.info("starting");
        clusterParticipants = clusterAgentsFactory.getClusterParticipants();
        logger.info("Setting up JMX.");
        long startTime = SystemTime.getInstance().milliseconds();
        reporter = reporterFactory != null ? reporterFactory.apply(registry) : JmxReporter.forRegistry(registry).build();
        reporter.start();
        logger.info("creating configs");
        NetworkConfig networkConfig = new NetworkConfig(properties);
        StoreConfig storeConfig = new StoreConfig(properties);
        DiskManagerConfig diskManagerConfig = new DiskManagerConfig(properties);
        ServerConfig serverConfig = new ServerConfig(properties);
        ReplicationConfig replicationConfig = new ReplicationConfig(properties);
        ConnectionPoolConfig connectionPoolConfig = new ConnectionPoolConfig(properties);
        SSLConfig sslConfig = new SSLConfig(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
        StatsManagerConfig statsConfig = new StatsManagerConfig(properties);
        CloudConfig cloudConfig = new CloudConfig(properties);
        // verify the configs
        properties.verify();
        scheduler = Utils.newScheduler(serverConfig.serverSchedulerNumOfthreads, false);
        // mismatch in sealed/stopped replica lists that maintained by each participant.
        if (clusterParticipants != null && clusterParticipants.size() > 1 && serverConfig.serverParticipantsConsistencyCheckerPeriodSec > 0) {
            consistencyChecker = new ParticipantsConsistencyChecker(clusterParticipants, metrics);
            logger.info("Scheduling participants consistency checker with a period of {} secs", serverConfig.serverParticipantsConsistencyCheckerPeriodSec);
            consistencyCheckerScheduler = Utils.newScheduler(1, "consistency-checker-", false);
            consistencyCheckerTask = consistencyCheckerScheduler.scheduleAtFixedRate(consistencyChecker, 0, serverConfig.serverParticipantsConsistencyCheckerPeriodSec, TimeUnit.SECONDS);
        }
        logger.info("checking if node exists in clustermap host {} port {}", networkConfig.hostName, networkConfig.port);
        DataNodeId nodeId = clusterMap.getDataNodeId(networkConfig.hostName, networkConfig.port);
        if (nodeId == null) {
            throw new IllegalArgumentException("The node " + networkConfig.hostName + ":" + networkConfig.port + "is not present in the clustermap. Failing to start the datanode");
        }
        AccountServiceFactory accountServiceFactory = Utils.getObj(serverConfig.serverAccountServiceFactory, properties, registry);
        AccountService accountService = accountServiceFactory.getAccountService();
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        // In most cases, there should be only one participant in the clusterParticipants list. If there are more than one
        // and some components require sole participant, the first one in the list will be primary participant.
        storageManager = new StorageManager(storeConfig, diskManagerConfig, scheduler, registry, storeKeyFactory, clusterMap, nodeId, new BlobStoreHardDelete(), clusterParticipants, time, new BlobStoreRecovery(), accountService);
        storageManager.start();
        SSLFactory sslFactory = new NettySslHttp2Factory(sslConfig);
        if (clusterMapConfig.clusterMapEnableHttp2Replication) {
            connectionPool = new Http2BlockingChannelPool(sslFactory, new Http2ClientConfig(properties), new Http2ClientMetrics(registry));
        } else {
            connectionPool = new BlockingChannelConnectionPool(connectionPoolConfig, sslConfig, clusterMapConfig, registry);
        }
        connectionPool.start();
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, registry);
        Predicate<MessageInfo> skipPredicate = new ReplicationSkipPredicate(accountService, replicationConfig);
        replicationManager = new ReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, clusterParticipants.get(0), skipPredicate);
        replicationManager.start();
        if (replicationConfig.replicationEnabledWithVcrCluster) {
            logger.info("Creating Helix cluster spectator for cloud to store replication.");
            vcrClusterSpectator = _vcrClusterAgentsFactory.getVcrClusterSpectator(cloudConfig, clusterMapConfig);
            cloudToStoreReplicationManager = new CloudToStoreReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, vcrClusterSpectator, clusterParticipants.get(0));
            cloudToStoreReplicationManager.start();
        }
        logger.info("Creating StatsManager to publish stats");
        accountStatsMySqlStore = statsConfig.enableMysqlReport ? (AccountStatsMySqlStore) new AccountStatsMySqlStoreFactory(properties, clusterMapConfig, registry).getAccountStatsStore() : null;
        statsManager = new StatsManager(storageManager, clusterMap.getReplicaIds(nodeId), registry, statsConfig, time, clusterParticipants.get(0), accountStatsMySqlStore, accountService);
        if (serverConfig.serverStatsPublishLocalEnabled) {
            statsManager.start();
        }
        ArrayList<Port> ports = new ArrayList<Port>();
        ports.add(new Port(networkConfig.port, PortType.PLAINTEXT));
        if (nodeId.hasSSLPort()) {
            ports.add(new Port(nodeId.getSSLPort(), PortType.SSL));
        }
        networkServer = new SocketServer(networkConfig, sslConfig, registry, ports);
        FindTokenHelper findTokenHelper = new FindTokenHelper(storeKeyFactory, replicationConfig);
        requests = new AmbryServerRequests(storageManager, networkServer.getRequestResponseChannel(), clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
        requestHandlerPool = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, networkServer.getRequestResponseChannel(), requests);
        networkServer.start();
        // Start netty http2 server
        if (nodeId.hasHttp2Port()) {
            NettyConfig nettyConfig = new NettyConfig(properties);
            NettyMetrics nettyMetrics = new NettyMetrics(registry);
            Http2ServerMetrics http2ServerMetrics = new Http2ServerMetrics(registry);
            Http2ClientConfig http2ClientConfig = new Http2ClientConfig(properties);
            logger.info("Http2 port {} is enabled. Starting HTTP/2 service.", nodeId.getHttp2Port());
            NettyServerRequestResponseChannel requestResponseChannel = new NettyServerRequestResponseChannel(networkConfig.queuedMaxRequests, http2ServerMetrics);
            AmbryServerRequests ambryServerRequestsForHttp2 = new AmbryServerRequests(storageManager, requestResponseChannel, clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
            requestHandlerPoolForHttp2 = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, requestResponseChannel, ambryServerRequestsForHttp2);
            NioServerFactory nioServerFactory = new StorageServerNettyFactory(nodeId.getHttp2Port(), requestResponseChannel, sslFactory, nettyConfig, http2ClientConfig, metrics, nettyMetrics, http2ServerMetrics, serverSecurityService);
            nettyHttp2Server = nioServerFactory.getNioServer();
            nettyHttp2Server.start();
        }
        // Other code
        List<AmbryStatsReport> ambryStatsReports = new ArrayList<>();
        Set<String> validStatsTypes = new HashSet<>();
        for (StatsReportType type : StatsReportType.values()) {
            validStatsTypes.add(type.toString());
        }
        if (serverConfig.serverStatsPublishReportEnabled) {
            serverConfig.serverStatsReportsToPublish.forEach(e -> {
                if (validStatsTypes.contains(e)) {
                    ambryStatsReports.add(new AmbryStatsReportImpl(serverConfig.serverQuotaStatsAggregateIntervalInMinutes, StatsReportType.valueOf(e)));
                }
            });
        }
        if (vcrClusterSpectator != null) {
            vcrClusterSpectator.spectate();
        }
        Callback<StatsSnapshot> accountServiceCallback = new AccountServiceCallback(accountService);
        for (ClusterParticipant clusterParticipant : clusterParticipants) {
            clusterParticipant.participate(ambryStatsReports, accountStatsMySqlStore, accountServiceCallback);
        }
        if (nettyInternalMetrics != null) {
            nettyInternalMetrics.start();
            logger.info("NettyInternalMetric starts");
        }
        logger.info("started");
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        metrics.serverStartTimeInMs.update(processingTime);
        logger.info("Server startup time in Ms {}", processingTime);
    } catch (Exception e) {
        logger.error("Error during startup", e);
        throw new InstantiationException("failure during startup " + e);
    }
}

Example 2 with AccountStatsMySqlStore

use of com.github.ambry.accountstats.AccountStatsMySqlStore in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testFetchMonthlyStorageUsage.

/**
 * Test to update container storage usage monthly base.
 */
@Test
public void testFetchMonthlyStorageUsage() throws Exception {
    MockTime mockTime = new MockTime(SystemTime.getInstance().milliseconds());
    MySqlStorageUsageRefresher.time = mockTime;
    try {
        String currentMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        properties.remove(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS);
        StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
        AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
        MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
        // Fetch monthly storage usage
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month
        String notCurrentMonth = "1970-01";
        Map<String, Map<String, Long>> newContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(newContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(notCurrentMonth);
        refresher.fetchStorageUsageMonthlyBase();
        // Monthly storage usage still the old one
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month back to the current month
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        // Wait for schedule to retry
        Thread.sleep(MYSQL_RETRY_BACKOFF_MS * 2);
        assertEquals(newContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Forward the time to next month
        mockTime.sleep((MySqlStorageUsageRefresher.secondsToNextMonthTick(currentMonth, storageQuotaConfig.mysqlMonthlyBaseFetchOffsetSec) + 10) * 1000);
        String nextMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Function<String, Integer> stringMonthToInteger = (monthInStr) -> {
            String[] parts = monthInStr.split("-");
            int year = Integer.parseInt(parts[0]);
            int month = Integer.parseInt(parts[1]);
            return year * 12 + month;
        };
        assertEquals(stringMonthToInteger.apply(currentMonth) + 1, (int) stringMonthToInteger.apply(nextMonth));
        // Update the month to next month
        Map<String, Map<String, Long>> nextContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(nextContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(nextMonth);
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(nextContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // A backup file should be create as well
        assertEquals(nextContainerStorageUsages, refresher.getBackupFileManager().getBackupFileContent(nextMonth));
    } finally {
        MySqlStorageUsageRefresher.time = SystemTime.getInstance();
    }
}

Example 3 with AccountStatsMySqlStore

use of com.github.ambry.accountstats.AccountStatsMySqlStore in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testStartRefresher.

/**
 * Tests to start refresher with or without back up files.
 * @throws Exception
 */
@Test
public void testStartRefresher() throws Exception {
    // Store something to mysql database as container usage and monthly container usage
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    // use new mysql store, so we don't have connection conflict with the accountStatsMySqlStore.
    AccountStatsMySqlStore newAccountStatsMysqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    // we should get an container storage usage full of zero
    Map<String, Map<String, Long>> usage = refresher.getContainerStorageUsage();
    assertContainerUsageMapAllZero(usage);
    // we should have backup files now
    Path backupDirPath = Paths.get(properties.getProperty(StorageQuotaConfig.BACKUP_FILE_DIR));
    Path backupFilePath = backupDirPath.resolve(MySqlStorageUsageRefresher.getCurrentMonth());
    Map<String, Map<String, Long>> backupContainerStorageUsages = new ObjectMapper().readValue(backupFilePath.toFile(), new TypeReference<Map<String, Map<String, Long>>>() {
    });
    assertEquals(containerStorageUsages, backupContainerStorageUsages);
    // recreate a refresher, but change the monthly container usages, new refresher should load it from backup
    containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    Map<String, Map<String, Long>> currentMonthlyStorageUsages = refresher.getContainerStorageUsageMonthlyBase();
    assertEquals(backupContainerStorageUsages, currentMonthlyStorageUsages);
}

Example 4 with AccountStatsMySqlStore

use of com.github.ambry.accountstats.AccountStatsMySqlStore in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testRefresherUpdateAndListener.

/**
 * Test when updating container total usage.
 * @throws Exception
 */
@Test
public void testRefresherUpdateAndListener() throws Exception {
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    // Set polling interval to 2 seconds
    properties.setProperty(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS, "2000");
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
    AtomicReference<Map<String, Map<String, Long>>> containerUsageRef = new AtomicReference<>(null);
    AtomicReference<CountDownLatch> latchRef = new AtomicReference<>(null);
    refresher.registerListener(containerStorageUsage -> {
        containerUsageRef.set(containerStorageUsage);
        latchRef.get().countDown();
    });
    // Keep storage usage unchanged, listener should get an all-zero map
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch1 = new CountDownLatch(1);
    latchRef.set(latch1);
    latch1.await(10, TimeUnit.SECONDS);
    assertContainerUsageMapAllZero(containerUsageRef.get());
    // Change some usage, listener should get
    containerStorageUsages.get("1").compute("1", (k, v) -> v + 1L);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch2 = new CountDownLatch(1);
    latchRef.set(latch2);
    latch2.await(10, TimeUnit.SECONDS);
    assertEquals((long) containerUsageRef.get().get("1").get("1"), 1L);
    Map<String, Map<String, Long>> clone = cloneMap(containerUsageRef.get());
    clone.get("1").put("1", 0L);
    assertContainerUsageMapAllZero(clone);
}