Search in sources :

Example 6 with AccountService

use of com.github.ambry.account.AccountService in project ambry by linkedin.

the class AmbryServer method startup.

public void startup() throws InstantiationException {
    try {
        logger.info("starting");
        clusterParticipants = clusterAgentsFactory.getClusterParticipants();
        logger.info("Setting up JMX.");
        long startTime = SystemTime.getInstance().milliseconds();
        reporter = reporterFactory != null ? reporterFactory.apply(registry) : JmxReporter.forRegistry(registry).build();
        reporter.start();
        logger.info("creating configs");
        NetworkConfig networkConfig = new NetworkConfig(properties);
        StoreConfig storeConfig = new StoreConfig(properties);
        DiskManagerConfig diskManagerConfig = new DiskManagerConfig(properties);
        ServerConfig serverConfig = new ServerConfig(properties);
        ReplicationConfig replicationConfig = new ReplicationConfig(properties);
        ConnectionPoolConfig connectionPoolConfig = new ConnectionPoolConfig(properties);
        SSLConfig sslConfig = new SSLConfig(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
        StatsManagerConfig statsConfig = new StatsManagerConfig(properties);
        CloudConfig cloudConfig = new CloudConfig(properties);
        // verify the configs
        properties.verify();
        scheduler = Utils.newScheduler(serverConfig.serverSchedulerNumOfthreads, false);
        // mismatch in sealed/stopped replica lists that maintained by each participant.
        if (clusterParticipants != null && clusterParticipants.size() > 1 && serverConfig.serverParticipantsConsistencyCheckerPeriodSec > 0) {
            consistencyChecker = new ParticipantsConsistencyChecker(clusterParticipants, metrics);
            logger.info("Scheduling participants consistency checker with a period of {} secs", serverConfig.serverParticipantsConsistencyCheckerPeriodSec);
            consistencyCheckerScheduler = Utils.newScheduler(1, "consistency-checker-", false);
            consistencyCheckerTask = consistencyCheckerScheduler.scheduleAtFixedRate(consistencyChecker, 0, serverConfig.serverParticipantsConsistencyCheckerPeriodSec, TimeUnit.SECONDS);
        }
        logger.info("checking if node exists in clustermap host {} port {}", networkConfig.hostName, networkConfig.port);
        DataNodeId nodeId = clusterMap.getDataNodeId(networkConfig.hostName, networkConfig.port);
        if (nodeId == null) {
            throw new IllegalArgumentException("The node " + networkConfig.hostName + ":" + networkConfig.port + "is not present in the clustermap. Failing to start the datanode");
        }
        AccountServiceFactory accountServiceFactory = Utils.getObj(serverConfig.serverAccountServiceFactory, properties, registry);
        AccountService accountService = accountServiceFactory.getAccountService();
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        // In most cases, there should be only one participant in the clusterParticipants list. If there are more than one
        // and some components require sole participant, the first one in the list will be primary participant.
        storageManager = new StorageManager(storeConfig, diskManagerConfig, scheduler, registry, storeKeyFactory, clusterMap, nodeId, new BlobStoreHardDelete(), clusterParticipants, time, new BlobStoreRecovery(), accountService);
        storageManager.start();
        SSLFactory sslFactory = new NettySslHttp2Factory(sslConfig);
        if (clusterMapConfig.clusterMapEnableHttp2Replication) {
            connectionPool = new Http2BlockingChannelPool(sslFactory, new Http2ClientConfig(properties), new Http2ClientMetrics(registry));
        } else {
            connectionPool = new BlockingChannelConnectionPool(connectionPoolConfig, sslConfig, clusterMapConfig, registry);
        }
        connectionPool.start();
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, registry);
        Predicate<MessageInfo> skipPredicate = new ReplicationSkipPredicate(accountService, replicationConfig);
        replicationManager = new ReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, clusterParticipants.get(0), skipPredicate);
        replicationManager.start();
        if (replicationConfig.replicationEnabledWithVcrCluster) {
            logger.info("Creating Helix cluster spectator for cloud to store replication.");
            vcrClusterSpectator = _vcrClusterAgentsFactory.getVcrClusterSpectator(cloudConfig, clusterMapConfig);
            cloudToStoreReplicationManager = new CloudToStoreReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, storeKeyFactory, clusterMap, scheduler, nodeId, connectionPool, registry, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer, vcrClusterSpectator, clusterParticipants.get(0));
            cloudToStoreReplicationManager.start();
        }
        logger.info("Creating StatsManager to publish stats");
        accountStatsMySqlStore = statsConfig.enableMysqlReport ? (AccountStatsMySqlStore) new AccountStatsMySqlStoreFactory(properties, clusterMapConfig, registry).getAccountStatsStore() : null;
        statsManager = new StatsManager(storageManager, clusterMap.getReplicaIds(nodeId), registry, statsConfig, time, clusterParticipants.get(0), accountStatsMySqlStore, accountService);
        if (serverConfig.serverStatsPublishLocalEnabled) {
            statsManager.start();
        }
        ArrayList<Port> ports = new ArrayList<Port>();
        ports.add(new Port(networkConfig.port, PortType.PLAINTEXT));
        if (nodeId.hasSSLPort()) {
            ports.add(new Port(nodeId.getSSLPort(), PortType.SSL));
        }
        networkServer = new SocketServer(networkConfig, sslConfig, registry, ports);
        FindTokenHelper findTokenHelper = new FindTokenHelper(storeKeyFactory, replicationConfig);
        requests = new AmbryServerRequests(storageManager, networkServer.getRequestResponseChannel(), clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
        requestHandlerPool = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, networkServer.getRequestResponseChannel(), requests);
        networkServer.start();
        // Start netty http2 server
        if (nodeId.hasHttp2Port()) {
            NettyConfig nettyConfig = new NettyConfig(properties);
            NettyMetrics nettyMetrics = new NettyMetrics(registry);
            Http2ServerMetrics http2ServerMetrics = new Http2ServerMetrics(registry);
            Http2ClientConfig http2ClientConfig = new Http2ClientConfig(properties);
            logger.info("Http2 port {} is enabled. Starting HTTP/2 service.", nodeId.getHttp2Port());
            NettyServerRequestResponseChannel requestResponseChannel = new NettyServerRequestResponseChannel(networkConfig.queuedMaxRequests, http2ServerMetrics);
            AmbryServerRequests ambryServerRequestsForHttp2 = new AmbryServerRequests(storageManager, requestResponseChannel, clusterMap, nodeId, registry, metrics, findTokenHelper, notificationSystem, replicationManager, storeKeyFactory, serverConfig, storeKeyConverterFactory, statsManager, clusterParticipants.get(0));
            requestHandlerPoolForHttp2 = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, requestResponseChannel, ambryServerRequestsForHttp2);
            NioServerFactory nioServerFactory = new StorageServerNettyFactory(nodeId.getHttp2Port(), requestResponseChannel, sslFactory, nettyConfig, http2ClientConfig, metrics, nettyMetrics, http2ServerMetrics, serverSecurityService);
            nettyHttp2Server = nioServerFactory.getNioServer();
            nettyHttp2Server.start();
        }
        // Other code
        List<AmbryStatsReport> ambryStatsReports = new ArrayList<>();
        Set<String> validStatsTypes = new HashSet<>();
        for (StatsReportType type : StatsReportType.values()) {
            validStatsTypes.add(type.toString());
        }
        if (serverConfig.serverStatsPublishReportEnabled) {
            serverConfig.serverStatsReportsToPublish.forEach(e -> {
                if (validStatsTypes.contains(e)) {
                    ambryStatsReports.add(new AmbryStatsReportImpl(serverConfig.serverQuotaStatsAggregateIntervalInMinutes, StatsReportType.valueOf(e)));
                }
            });
        }
        if (vcrClusterSpectator != null) {
            vcrClusterSpectator.spectate();
        }
        Callback<StatsSnapshot> accountServiceCallback = new AccountServiceCallback(accountService);
        for (ClusterParticipant clusterParticipant : clusterParticipants) {
            clusterParticipant.participate(ambryStatsReports, accountStatsMySqlStore, accountServiceCallback);
        }
        if (nettyInternalMetrics != null) {
            nettyInternalMetrics.start();
            logger.info("NettyInternalMetric starts");
        }
        logger.info("started");
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        metrics.serverStartTimeInMs.update(processingTime);
        logger.info("Server startup time in Ms {}", processingTime);
    } catch (Exception e) {
        logger.error("Error during startup", e);
        throw new InstantiationException("failure during startup " + e);
    }
}
Also used : DiskManagerConfig(com.github.ambry.config.DiskManagerConfig) SSLFactory(com.github.ambry.commons.SSLFactory) Http2ClientMetrics(com.github.ambry.network.http2.Http2ClientMetrics) Port(com.github.ambry.network.Port) StorageManager(com.github.ambry.store.StorageManager) ReplicationSkipPredicate(com.github.ambry.replication.ReplicationSkipPredicate) ArrayList(java.util.ArrayList) NettySslHttp2Factory(com.github.ambry.commons.NettySslHttp2Factory) NettyConfig(com.github.ambry.config.NettyConfig) ServerConfig(com.github.ambry.config.ServerConfig) StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) StoreKeyConverterFactory(com.github.ambry.store.StoreKeyConverterFactory) AccountStatsMySqlStoreFactory(com.github.ambry.accountstats.AccountStatsMySqlStoreFactory) HashSet(java.util.HashSet) CloudToStoreReplicationManager(com.github.ambry.replication.CloudToStoreReplicationManager) ReplicationManager(com.github.ambry.replication.ReplicationManager) ReplicationConfig(com.github.ambry.config.ReplicationConfig) Http2BlockingChannelPool(com.github.ambry.network.http2.Http2BlockingChannelPool) Http2ClientConfig(com.github.ambry.config.Http2ClientConfig) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) BlockingChannelConnectionPool(com.github.ambry.network.BlockingChannelConnectionPool) AccountServiceCallback(com.github.ambry.account.AccountServiceCallback) Http2ServerMetrics(com.github.ambry.network.http2.Http2ServerMetrics) BlobStoreRecovery(com.github.ambry.messageformat.BlobStoreRecovery) RequestHandlerPool(com.github.ambry.protocol.RequestHandlerPool) StoreConfig(com.github.ambry.config.StoreConfig) DataNodeId(com.github.ambry.clustermap.DataNodeId) AccountService(com.github.ambry.account.AccountService) ConnectionPoolConfig(com.github.ambry.config.ConnectionPoolConfig) CloudConfig(com.github.ambry.config.CloudConfig) StorageServerNettyFactory(com.github.ambry.rest.StorageServerNettyFactory) AccountStatsMySqlStore(com.github.ambry.accountstats.AccountStatsMySqlStore) SSLConfig(com.github.ambry.config.SSLConfig) StatsManagerConfig(com.github.ambry.config.StatsManagerConfig) SocketServer(com.github.ambry.network.SocketServer) NettyMetrics(com.github.ambry.rest.NettyMetrics) FindTokenHelper(com.github.ambry.replication.FindTokenHelper) NettyServerRequestResponseChannel(com.github.ambry.network.NettyServerRequestResponseChannel) NetworkConfig(com.github.ambry.config.NetworkConfig) BlobStoreHardDelete(com.github.ambry.messageformat.BlobStoreHardDelete) IOException(java.io.IOException) MessageInfo(com.github.ambry.store.MessageInfo) NioServerFactory(com.github.ambry.rest.NioServerFactory) CloudToStoreReplicationManager(com.github.ambry.replication.CloudToStoreReplicationManager) ClusterParticipant(com.github.ambry.clustermap.ClusterParticipant) AccountServiceFactory(com.github.ambry.account.AccountServiceFactory)

Example 7 with AccountService

use of com.github.ambry.account.AccountService in project ambry by linkedin.

the class RouterUtilsTest method testGetAccountContainer.

/**
 * Test {@link RouterUtils#getAccountContainer(AccountService, short, short)}.
 */
@Test
public void testGetAccountContainer() throws AccountServiceException {
    AccountService accountService = new InMemAccountService(false, false);
    // Both accountId and containerId are not tracked by AccountService.
    Pair<Account, Container> accountContainer = RouterUtils.getAccountContainer(accountService, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID);
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    accountContainer = RouterUtils.getAccountContainer(accountService, Utils.getRandomShort(random), Utils.getRandomShort(random));
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // accountId is tracked by AccountService but containerId not.
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    Account account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // Both accountId and containerId are tracked by AccountService.
    Container container = new ContainerBuilder(containerId, "ContainerNameOf" + containerId, Container.ContainerStatus.ACTIVE, "description", accountId).build();
    account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).addOrUpdateContainer(container).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container doesn't match", container, accountContainer.getSecond());
}
Also used : Account(com.github.ambry.account.Account) InMemAccountService(com.github.ambry.account.InMemAccountService) Container(com.github.ambry.account.Container) ContainerBuilder(com.github.ambry.account.ContainerBuilder) AccountBuilder(com.github.ambry.account.AccountBuilder) AccountService(com.github.ambry.account.AccountService) InMemAccountService(com.github.ambry.account.InMemAccountService) Test(org.junit.Test)

Example 8 with AccountService

use of com.github.ambry.account.AccountService in project ambry by linkedin.

the class VcrServer method startup.

/**
 * Start the VCR Server.
 * @throws InstantiationException if an error was encountered during startup.
 */
public void startup() throws InstantiationException {
    try {
        logger.info("starting");
        ServerConfig serverConfig = new ServerConfig(properties);
        ServerSecurityServiceFactory serverSecurityServiceFactory = Utils.getObj(serverConfig.serverSecurityServiceFactory, properties, serverMetrics, registry);
        serverSecurityService = serverSecurityServiceFactory.getServerSecurityService();
        clusterMap = clusterAgentsFactory.getClusterMap();
        logger.info("Initialized clusterMap");
        registry = clusterMap.getMetricRegistry();
        serverMetrics = new ServerMetrics(registry, VcrServer.class, VcrServer.class);
        logger.info("Setting up JMX.");
        long startTime = SystemTime.getInstance().milliseconds();
        registry = clusterMap.getMetricRegistry();
        reporter = reporterFactory != null ? reporterFactory.apply(registry) : JmxReporter.forRegistry(registry).build();
        reporter.start();
        logger.info("creating configs");
        NetworkConfig networkConfig = new NetworkConfig(properties);
        StoreConfig storeConfig = new StoreConfig(properties);
        ReplicationConfig replicationConfig = new ReplicationConfig(properties);
        CloudConfig cloudConfig = new CloudConfig(properties);
        ConnectionPoolConfig connectionPoolConfig = new ConnectionPoolConfig(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
        SSLConfig sslConfig = new SSLConfig(properties);
        // verify the configs
        properties.verify();
        // initialize cloud destination
        if (cloudDestinationFactory == null) {
            cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, properties, registry, clusterMap);
        }
        cloudDestination = cloudDestinationFactory.getCloudDestination();
        // TODO Make sure that config.updaterPollingIntervalMs value is large (~one day) for VCR.
        AccountServiceFactory accountServiceFactory = Utils.getObj(serverConfig.serverAccountServiceFactory, properties, registry);
        AccountService accountService = accountServiceFactory.getAccountService();
        vcrClusterParticipant = ((VcrClusterAgentsFactory) Utils.getObj(cloudConfig.vcrClusterAgentsFactoryClass, cloudConfig, clusterMapConfig, clusterMap, accountService, storeConfig, cloudDestination, registry)).getVcrClusterParticipant();
        scheduler = Utils.newScheduler(serverConfig.serverSchedulerNumOfthreads, false);
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        SSLFactory sslFactory = new NettySslHttp2Factory(sslConfig);
        if (clusterMapConfig.clusterMapEnableHttp2Replication) {
            connectionPool = new Http2BlockingChannelPool(sslFactory, new Http2ClientConfig(properties), new Http2ClientMetrics(registry));
            logger.info("Using http2 for VCR replication.");
        } else {
            connectionPool = new BlockingChannelConnectionPool(connectionPoolConfig, sslConfig, clusterMapConfig, registry);
            logger.info("Using blocking channel for VCR replication.");
        }
        connectionPool.start();
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, registry);
        VcrMetrics vcrMetrics = new VcrMetrics(registry);
        CloudStorageManager cloudStorageManager = new CloudStorageManager(properties, vcrMetrics, cloudDestination, clusterMap);
        vcrReplicationManager = new VcrReplicationManager(cloudConfig, replicationConfig, clusterMapConfig, storeConfig, cloudStorageManager, storeKeyFactory, clusterMap, vcrClusterParticipant, cloudDestination, scheduler, connectionPool, vcrMetrics, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer);
        vcrReplicationManager.start();
        DataNodeId currentNode = vcrClusterParticipant.getCurrentDataNodeId();
        ArrayList<Port> ports = new ArrayList<Port>();
        ports.add(new Port(networkConfig.port, PortType.PLAINTEXT));
        if (currentNode.hasSSLPort()) {
            ports.add(new Port(cloudConfig.vcrSslPort, PortType.SSL));
        }
        networkServer = new SocketServer(networkConfig, sslConfig, registry, ports);
        // todo fix enableDataPrefetch
        ServerMetrics serverMetrics = new ServerMetrics(registry, VcrRequests.class, VcrServer.class);
        VcrRequests requests = new VcrRequests(cloudStorageManager, networkServer.getRequestResponseChannel(), clusterMap, currentNode, registry, serverMetrics, new FindTokenHelper(storeKeyFactory, replicationConfig), notificationSystem, vcrReplicationManager, storeKeyFactory, storeKeyConverterFactory);
        requestHandlerPool = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, networkServer.getRequestResponseChannel(), requests);
        networkServer.start();
        // Start netty http2 server
        if (currentNode.hasHttp2Port()) {
            logger.info("Http2 port {} is enabled. Starting HTTP/2 service.", currentNode.getHttp2Port());
            NettyConfig nettyConfig = new NettyConfig(properties);
            NettyMetrics nettyMetrics = new NettyMetrics(registry);
            Http2ServerMetrics http2ServerMetrics = new Http2ServerMetrics(registry);
            Http2ClientConfig http2ClientConfig = new Http2ClientConfig(properties);
            NettyServerRequestResponseChannel requestResponseChannel = new NettyServerRequestResponseChannel(networkConfig.queuedMaxRequests, http2ServerMetrics);
            VcrRequests vcrRequestsForHttp2 = new VcrRequests(cloudStorageManager, requestResponseChannel, clusterMap, currentNode, registry, serverMetrics, new FindTokenHelper(storeKeyFactory, replicationConfig), notificationSystem, vcrReplicationManager, storeKeyFactory, storeKeyConverterFactory);
            requestHandlerPoolForHttp2 = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, requestResponseChannel, vcrRequestsForHttp2);
            NioServerFactory nioServerFactory = new StorageServerNettyFactory(currentNode.getHttp2Port(), requestResponseChannel, sslFactory, nettyConfig, http2ClientConfig, serverMetrics, nettyMetrics, http2ServerMetrics, serverSecurityService);
            nettyHttp2Server = nioServerFactory.getNioServer();
            nettyHttp2Server.start();
        }
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        logger.info("VCR startup time in Ms {}", processingTime);
    } catch (Exception e) {
        logger.error("Error during VCR startup", e);
        throw new InstantiationException("failure during VCR startup " + e);
    }
}
Also used : ConnectionPoolConfig(com.github.ambry.config.ConnectionPoolConfig) SSLFactory(com.github.ambry.commons.SSLFactory) Http2ClientMetrics(com.github.ambry.network.http2.Http2ClientMetrics) Port(com.github.ambry.network.Port) CloudConfig(com.github.ambry.config.CloudConfig) ArrayList(java.util.ArrayList) StorageServerNettyFactory(com.github.ambry.rest.StorageServerNettyFactory) NettySslHttp2Factory(com.github.ambry.commons.NettySslHttp2Factory) NettyConfig(com.github.ambry.config.NettyConfig) ServerConfig(com.github.ambry.config.ServerConfig) StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) StoreKeyConverterFactory(com.github.ambry.store.StoreKeyConverterFactory) ServerMetrics(com.github.ambry.commons.ServerMetrics) Http2ServerMetrics(com.github.ambry.network.http2.Http2ServerMetrics) SSLConfig(com.github.ambry.config.SSLConfig) ReplicationConfig(com.github.ambry.config.ReplicationConfig) Http2BlockingChannelPool(com.github.ambry.network.http2.Http2BlockingChannelPool) SocketServer(com.github.ambry.network.SocketServer) NettyMetrics(com.github.ambry.rest.NettyMetrics) FindTokenHelper(com.github.ambry.replication.FindTokenHelper) NettyServerRequestResponseChannel(com.github.ambry.network.NettyServerRequestResponseChannel) NetworkConfig(com.github.ambry.config.NetworkConfig) Http2ClientConfig(com.github.ambry.config.Http2ClientConfig) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) BlockingChannelConnectionPool(com.github.ambry.network.BlockingChannelConnectionPool) IOException(java.io.IOException) ServerSecurityServiceFactory(com.github.ambry.rest.ServerSecurityServiceFactory) Http2ServerMetrics(com.github.ambry.network.http2.Http2ServerMetrics) NioServerFactory(com.github.ambry.rest.NioServerFactory) RequestHandlerPool(com.github.ambry.protocol.RequestHandlerPool) StoreConfig(com.github.ambry.config.StoreConfig) AccountService(com.github.ambry.account.AccountService) DataNodeId(com.github.ambry.clustermap.DataNodeId) AccountServiceFactory(com.github.ambry.account.AccountServiceFactory)

Example 9 with AccountService

use of com.github.ambry.account.AccountService in project ambry by linkedin.

the class AmbryBlobStorageServiceFactory method getBlobStorageService.

/**
 * Returns a new instance of {@link AmbryBlobStorageService}.
 * @return a new instance of {@link AmbryBlobStorageService}.
 */
@Override
public BlobStorageService getBlobStorageService() {
    try {
        AccountServiceFactory accountServiceFactory = Utils.getObj(frontendConfig.frontendAccountServiceFactory, verifiableProperties, clusterMap.getMetricRegistry(), notifier);
        AccountService accountService = accountServiceFactory.getAccountService();
        IdConverterFactory idConverterFactory = Utils.getObj(frontendConfig.frontendIdConverterFactory, verifiableProperties, clusterMap.getMetricRegistry());
        UrlSigningServiceFactory urlSigningServiceFactory = Utils.getObj(frontendConfig.frontendUrlSigningServiceFactory, verifiableProperties, clusterMap.getMetricRegistry());
        UrlSigningService urlSigningService = urlSigningServiceFactory.getUrlSigningService();
        AccountAndContainerInjector accountAndContainerInjector = new AccountAndContainerInjector(accountService, frontendMetrics, frontendConfig);
        SecurityServiceFactory securityServiceFactory = Utils.getObj(frontendConfig.frontendSecurityServiceFactory, verifiableProperties, clusterMap, accountService, urlSigningService, accountAndContainerInjector);
        return new AmbryBlobStorageService(frontendConfig, frontendMetrics, responseHandler, router, clusterMap, idConverterFactory, securityServiceFactory, accountService, urlSigningService, accountAndContainerInjector);
    } catch (Exception e) {
        throw new IllegalStateException("Could not instantiate AmbryBlobStorageService", e);
    }
}
Also used : AccountService(com.github.ambry.account.AccountService) AccountServiceFactory(com.github.ambry.account.AccountServiceFactory)

Example 10 with AccountService

use of com.github.ambry.account.AccountService in project ambry by linkedin.

the class FrontendTestUrlSigningServiceFactory method injectionAccountAndContainerForGetHeadDeleteBlobIdTest.

/**
 * Tests injecting target {@link Account} and {@link Container} for GET/HEAD/DELETE blobId string in {@link BlobId#BLOB_ID_V2}.
 * The {@link AccountService} is prepopulated with a reference account and {@link InMemAccountService#UNKNOWN_ACCOUNT}. The expected
 * behavior should be:
 *
 * <pre>
 *   AId in blobId    CId in blobId     expected Error      injected account      injected container
 *    realAId           realCId          NotFound            refAccount            refContainer       This can succeed if the blob exists in backend.
 *    realAId           UNKNOWN          InvalidContainer    null                  null
 *    realAId           nonExistCId      InvalidContainer    null                  null
 *    UNKNOWN           realCId          InvalidContainer    null                  null
 *    UNKNOWN           UNKNOWN          NotFound            UNKNOWN               UNKNOWN            This can succeed if the blob exists in backend.
 *    UNKNOWN           nonExistCId      InvalidContainer    null                  null
 *    nonExistAId       realCId          InvalidAccount      null                  null
 *    nonExistAId       UNKNOWN          InvalidAccount      null                  null
 *    nonExistAId       nonExistCId      InvalidAccount      null                  null
 * </pre>
 *
 * @throws Exception
 */
@Test
public void injectionAccountAndContainerForGetHeadDeleteBlobIdTest() throws Exception {
    List<Short> blobIdVersions = Arrays.stream(BlobId.getAllValidVersions()).filter(version -> version >= BlobId.BLOB_ID_V2).collect(Collectors.toList());
    for (short version : blobIdVersions) {
        populateAccountService();
        // aid=refAId, cid=refCId
        String blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, refAccount.getId(), refContainer.getId(), clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, refAccount, refContainer, RestServiceErrorCode.NotFound);
        // aid=refAId, cid=unknownCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, refAccount.getId(), Container.UNKNOWN_CONTAINER_ID, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidContainer);
        // aid=refAId, cid=nonExistCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, refAccount.getId(), (short) -1234, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidContainer);
        // aid=unknownAId, cid=refCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Account.UNKNOWN_ACCOUNT_ID, refContainer.getId(), clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidContainer);
        // aid=unknownAId, cid=unknownCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, InMemAccountService.UNKNOWN_ACCOUNT, Container.UNKNOWN_CONTAINER, RestServiceErrorCode.NotFound);
        // aid=unknownAId, cid=nonExistCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Account.UNKNOWN_ACCOUNT_ID, (short) -1234, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidContainer);
        // aid=nonExistAId, cid=refCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, (short) -1234, refContainer.getId(), clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidAccount);
        // aid=nonExistAId, cid=unknownCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, (short) -1234, Container.UNKNOWN_CONTAINER_ID, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidAccount);
        // aid=nonExistAId, cid=nonExistCId
        blobId = new BlobId(version, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, (short) -1234, (short) -11, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK).getID();
        verifyAccountAndContainerFromBlobId(blobId, null, null, RestServiceErrorCode.InvalidAccount);
    }
}
Also used : GetOption(com.github.ambry.protocol.GetOption) MockRestRequest(com.github.ambry.rest.MockRestRequest) Arrays(java.util.Arrays) FrontendConfig(com.github.ambry.config.FrontendConfig) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) BlobProperties(com.github.ambry.messageformat.BlobProperties) FutureResult(com.github.ambry.router.FutureResult) Future(java.util.concurrent.Future) JSONException(org.json.JSONException) TestUtils(com.github.ambry.utils.TestUtils) Map(java.util.Map) ChunkInfo(com.github.ambry.router.ChunkInfo) EnumSet(java.util.EnumSet) RouterErrorCode(com.github.ambry.router.RouterErrorCode) RestResponseHandler(com.github.ambry.rest.RestResponseHandler) ReadableStreamChannel(com.github.ambry.router.ReadableStreamChannel) ClusterMapSnapshotConstants(com.github.ambry.clustermap.ClusterMapSnapshotConstants) RestResponseChannel(com.github.ambry.rest.RestResponseChannel) Set(java.util.Set) StandardCharsets(java.nio.charset.StandardCharsets) AccountStatsStore(com.github.ambry.accountstats.AccountStatsStore) BlobInfo(com.github.ambry.messageformat.BlobInfo) InvocationTargetException(java.lang.reflect.InvocationTargetException) AggregatedAccountStorageStats(com.github.ambry.server.storagestats.AggregatedAccountStorageStats) CountDownLatch(java.util.concurrent.CountDownLatch) RestServiceException(com.github.ambry.rest.RestServiceException) MockRestResponseChannel(com.github.ambry.rest.MockRestResponseChannel) Account(com.github.ambry.account.Account) InMemoryRouter(com.github.ambry.router.InMemoryRouter) SimpleDateFormat(java.text.SimpleDateFormat) RestTestUtils(com.github.ambry.rest.RestTestUtils) PutBlobOptions(com.github.ambry.router.PutBlobOptions) ArrayList(java.util.ArrayList) QuotaConfig(com.github.ambry.config.QuotaConfig) QuotaMode(com.github.ambry.quota.QuotaMode) Lists(com.google.common.collect.Lists) SSLSession(javax.net.ssl.SSLSession) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) SystemTime(com.github.ambry.utils.SystemTime) MetricRegistry(com.codahale.metrics.MetricRegistry) Properties(java.util.Properties) RestMethod(com.github.ambry.rest.RestMethod) VerifiableProperties(com.github.ambry.config.VerifiableProperties) IOException(java.io.IOException) Test(org.junit.Test) RouterException(com.github.ambry.router.RouterException) InMemAccountServiceFactory(com.github.ambry.account.InMemAccountServiceFactory) AmbryQuotaManager(com.github.ambry.quota.AmbryQuotaManager) QuotaMetrics(com.github.ambry.quota.QuotaMetrics) GetBlobResult(com.github.ambry.router.GetBlobResult) PutBlobOptionsBuilder(com.github.ambry.router.PutBlobOptionsBuilder) SimpleQuotaRecommendationMergePolicy(com.github.ambry.quota.SimpleQuotaRecommendationMergePolicy) Assert(org.junit.Assert) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) ResponseStatus(com.github.ambry.rest.ResponseStatus) ByteBufferReadableStreamChannel(com.github.ambry.commons.ByteBufferReadableStreamChannel) ContainerBuilder(com.github.ambry.account.ContainerBuilder) ByteBuffer(java.nio.ByteBuffer) AccountCollectionSerde(com.github.ambry.account.AccountCollectionSerde) RestUtilsTest(com.github.ambry.rest.RestUtilsTest) JSONObject(org.json.JSONObject) ByteArrayInputStream(java.io.ByteArrayInputStream) Locale(java.util.Locale) After(org.junit.After) Method(java.lang.reflect.Method) Container(com.github.ambry.account.Container) TimeZone(java.util.TimeZone) RestServiceErrorCode(com.github.ambry.rest.RestServiceErrorCode) Utils(com.github.ambry.utils.Utils) AccountBuilder(com.github.ambry.account.AccountBuilder) Collectors(java.util.stream.Collectors) NamedBlobRecord(com.github.ambry.named.NamedBlobRecord) AsyncWritableChannel(com.github.ambry.router.AsyncWritableChannel) List(java.util.List) QuotaTestUtils(com.github.ambry.quota.QuotaTestUtils) RestRequestMetricsTracker(com.github.ambry.rest.RestRequestMetricsTracker) Callback(com.github.ambry.commons.Callback) RestUtils(com.github.ambry.rest.RestUtils) UnsupportedEncodingException(java.io.UnsupportedEncodingException) InMemAccountService(com.github.ambry.account.InMemAccountService) PartitionId(com.github.ambry.clustermap.PartitionId) BlobId(com.github.ambry.commons.BlobId) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest) ByteRanges(com.github.ambry.router.ByteRanges) AccountService(com.github.ambry.account.AccountService) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) QuotaManager(com.github.ambry.quota.QuotaManager) HashSet(java.util.HashSet) NamedBlobDb(com.github.ambry.named.NamedBlobDb) CommonTestUtils(com.github.ambry.commons.CommonTestUtils) LinkedList(java.util.LinkedList) Router(com.github.ambry.router.Router) Pair(com.github.ambry.utils.Pair) Iterator(java.util.Iterator) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) QuotaChargeCallback(com.github.ambry.quota.QuotaChargeCallback) ByteRange(com.github.ambry.router.ByteRange) ClusterMap(com.github.ambry.clustermap.ClusterMap) GetBlobOptions(com.github.ambry.router.GetBlobOptions) AccountServiceException(com.github.ambry.account.AccountServiceException) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) StatsReportType(com.github.ambry.server.StatsReportType) RestRequest(com.github.ambry.rest.RestRequest) Collections(java.util.Collections) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) BlobId(com.github.ambry.commons.BlobId) Test(org.junit.Test) RestUtilsTest(com.github.ambry.rest.RestUtilsTest) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest)

Aggregations

AccountService (com.github.ambry.account.AccountService)12 InMemAccountService (com.github.ambry.account.InMemAccountService)8 VerifiableProperties (com.github.ambry.config.VerifiableProperties)8 IOException (java.io.IOException)7 ArrayList (java.util.ArrayList)7 Properties (java.util.Properties)7 MockClusterMap (com.github.ambry.clustermap.MockClusterMap)6 Router (com.github.ambry.router.Router)6 PartitionId (com.github.ambry.clustermap.PartitionId)5 BlobId (com.github.ambry.commons.BlobId)5 BlobProperties (com.github.ambry.messageformat.BlobProperties)5 RouterException (com.github.ambry.router.RouterException)5 DataNodeId (com.github.ambry.clustermap.DataNodeId)4 MockPartitionId (com.github.ambry.clustermap.MockPartitionId)4 AccountServiceFactory (com.github.ambry.account.AccountServiceFactory)3 ClusterMap (com.github.ambry.clustermap.ClusterMap)3 MockDataNodeId (com.github.ambry.clustermap.MockDataNodeId)3 BlobIdFactory (com.github.ambry.commons.BlobIdFactory)3 SSLConfig (com.github.ambry.config.SSLConfig)3 MessageFormatException (com.github.ambry.messageformat.MessageFormatException)3