Examples with StoreConfig com.github.ambry.config.StoreConfig used on opensource projects

Example 6 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class VcrServer method startup.

/**
 * Start the VCR Server.
 * @throws InstantiationException if an error was encountered during startup.
 */
public void startup() throws InstantiationException {
    try {
        logger.info("starting");
        ServerConfig serverConfig = new ServerConfig(properties);
        ServerSecurityServiceFactory serverSecurityServiceFactory = Utils.getObj(serverConfig.serverSecurityServiceFactory, properties, serverMetrics, registry);
        serverSecurityService = serverSecurityServiceFactory.getServerSecurityService();
        clusterMap = clusterAgentsFactory.getClusterMap();
        logger.info("Initialized clusterMap");
        registry = clusterMap.getMetricRegistry();
        serverMetrics = new ServerMetrics(registry, VcrServer.class, VcrServer.class);
        logger.info("Setting up JMX.");
        long startTime = SystemTime.getInstance().milliseconds();
        registry = clusterMap.getMetricRegistry();
        reporter = reporterFactory != null ? reporterFactory.apply(registry) : JmxReporter.forRegistry(registry).build();
        reporter.start();
        logger.info("creating configs");
        NetworkConfig networkConfig = new NetworkConfig(properties);
        StoreConfig storeConfig = new StoreConfig(properties);
        ReplicationConfig replicationConfig = new ReplicationConfig(properties);
        CloudConfig cloudConfig = new CloudConfig(properties);
        ConnectionPoolConfig connectionPoolConfig = new ConnectionPoolConfig(properties);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
        SSLConfig sslConfig = new SSLConfig(properties);
        // verify the configs
        properties.verify();
        // initialize cloud destination
        if (cloudDestinationFactory == null) {
            cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, properties, registry, clusterMap);
        }
        cloudDestination = cloudDestinationFactory.getCloudDestination();
        // TODO Make sure that config.updaterPollingIntervalMs value is large (~one day) for VCR.
        AccountServiceFactory accountServiceFactory = Utils.getObj(serverConfig.serverAccountServiceFactory, properties, registry);
        AccountService accountService = accountServiceFactory.getAccountService();
        vcrClusterParticipant = ((VcrClusterAgentsFactory) Utils.getObj(cloudConfig.vcrClusterAgentsFactoryClass, cloudConfig, clusterMapConfig, clusterMap, accountService, storeConfig, cloudDestination, registry)).getVcrClusterParticipant();
        scheduler = Utils.newScheduler(serverConfig.serverSchedulerNumOfthreads, false);
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        SSLFactory sslFactory = new NettySslHttp2Factory(sslConfig);
        if (clusterMapConfig.clusterMapEnableHttp2Replication) {
            connectionPool = new Http2BlockingChannelPool(sslFactory, new Http2ClientConfig(properties), new Http2ClientMetrics(registry));
            logger.info("Using http2 for VCR replication.");
        } else {
            connectionPool = new BlockingChannelConnectionPool(connectionPoolConfig, sslConfig, clusterMapConfig, registry);
            logger.info("Using blocking channel for VCR replication.");
        }
        connectionPool.start();
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, registry);
        VcrMetrics vcrMetrics = new VcrMetrics(registry);
        CloudStorageManager cloudStorageManager = new CloudStorageManager(properties, vcrMetrics, cloudDestination, clusterMap);
        vcrReplicationManager = new VcrReplicationManager(cloudConfig, replicationConfig, clusterMapConfig, storeConfig, cloudStorageManager, storeKeyFactory, clusterMap, vcrClusterParticipant, cloudDestination, scheduler, connectionPool, vcrMetrics, notificationSystem, storeKeyConverterFactory, serverConfig.serverMessageTransformer);
        vcrReplicationManager.start();
        DataNodeId currentNode = vcrClusterParticipant.getCurrentDataNodeId();
        ArrayList<Port> ports = new ArrayList<Port>();
        ports.add(new Port(networkConfig.port, PortType.PLAINTEXT));
        if (currentNode.hasSSLPort()) {
            ports.add(new Port(cloudConfig.vcrSslPort, PortType.SSL));
        }
        networkServer = new SocketServer(networkConfig, sslConfig, registry, ports);
        // todo fix enableDataPrefetch
        ServerMetrics serverMetrics = new ServerMetrics(registry, VcrRequests.class, VcrServer.class);
        VcrRequests requests = new VcrRequests(cloudStorageManager, networkServer.getRequestResponseChannel(), clusterMap, currentNode, registry, serverMetrics, new FindTokenHelper(storeKeyFactory, replicationConfig), notificationSystem, vcrReplicationManager, storeKeyFactory, storeKeyConverterFactory);
        requestHandlerPool = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, networkServer.getRequestResponseChannel(), requests);
        networkServer.start();
        // Start netty http2 server
        if (currentNode.hasHttp2Port()) {
            logger.info("Http2 port {} is enabled. Starting HTTP/2 service.", currentNode.getHttp2Port());
            NettyConfig nettyConfig = new NettyConfig(properties);
            NettyMetrics nettyMetrics = new NettyMetrics(registry);
            Http2ServerMetrics http2ServerMetrics = new Http2ServerMetrics(registry);
            Http2ClientConfig http2ClientConfig = new Http2ClientConfig(properties);
            NettyServerRequestResponseChannel requestResponseChannel = new NettyServerRequestResponseChannel(networkConfig.queuedMaxRequests, http2ServerMetrics);
            VcrRequests vcrRequestsForHttp2 = new VcrRequests(cloudStorageManager, requestResponseChannel, clusterMap, currentNode, registry, serverMetrics, new FindTokenHelper(storeKeyFactory, replicationConfig), notificationSystem, vcrReplicationManager, storeKeyFactory, storeKeyConverterFactory);
            requestHandlerPoolForHttp2 = new RequestHandlerPool(serverConfig.serverRequestHandlerNumOfThreads, requestResponseChannel, vcrRequestsForHttp2);
            NioServerFactory nioServerFactory = new StorageServerNettyFactory(currentNode.getHttp2Port(), requestResponseChannel, sslFactory, nettyConfig, http2ClientConfig, serverMetrics, nettyMetrics, http2ServerMetrics, serverSecurityService);
            nettyHttp2Server = nioServerFactory.getNioServer();
            nettyHttp2Server.start();
        }
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        logger.info("VCR startup time in Ms {}", processingTime);
    } catch (Exception e) {
        logger.error("Error during VCR startup", e);
        throw new InstantiationException("failure during VCR startup " + e);
    }
}

Example 7 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class StorageManagerTest method removeBlobStoreTest.

/**
 * Test remove blob store with given {@link PartitionId}
 * @throws Exception
 */
@Test
public void removeBlobStoreTest() throws Exception {
    MockDataNodeId dataNode = clusterMap.getDataNodes().get(0);
    List<ReplicaId> replicas = clusterMap.getReplicaIds(dataNode);
    List<MockDataNodeId> dataNodes = new ArrayList<>();
    dataNodes.add(dataNode);
    MockPartitionId invalidPartition = new MockPartitionId(Long.MAX_VALUE, MockClusterMap.DEFAULT_PARTITION_CLASS, dataNodes, 0);
    StorageManager storageManager = createStorageManager(dataNode, metricRegistry, null);
    storageManager.start();
    // Replica[1] will be used to test removing a started store. Replica[2] will be used to test a store with compaction enabled
    for (int i = 3; i < replicas.size(); i++) {
        ReplicaId replica = replicas.get(i);
        PartitionId id = replica.getPartitionId();
        assertTrue("Disable compaction should succeed", storageManager.controlCompactionForBlobStore(id, false));
        assertTrue("Shutdown should succeed on given store", storageManager.shutdownBlobStore(id));
        assertTrue("Removing store should succeed", storageManager.removeBlobStore(id));
        assertNull("The store should not exist", storageManager.getStore(id, false));
    }
    // test remove store that compaction is still enabled on it, even though it is shutdown
    PartitionId id = replicas.get(2).getPartitionId();
    assertTrue("Shutdown should succeed on given store", storageManager.shutdownBlobStore(id));
    assertFalse("Removing store should fail because compaction is enabled on this store", storageManager.removeBlobStore(id));
    // test remove store that is still started
    id = replicas.get(1).getPartitionId();
    assertFalse("Removing store should fail because store is still started", storageManager.removeBlobStore(id));
    // test remove store that the disk manager is not running
    id = replicas.get(0).getPartitionId();
    storageManager.getDiskManager(id).shutdown();
    assertFalse("Removing store should fail because disk manager is not running", storageManager.removeBlobStore(id));
    // test a store that doesn't exist
    assertFalse("Removing not-found store should return false", storageManager.removeBlobStore(invalidPartition));
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
    // test that remove store when compaction executor is not instantiated
    // by default, storeCompactionTriggers = "" which makes compaction executor = null during initialization
    VerifiableProperties vProps = new VerifiableProperties(new Properties());
    storageManager = new StorageManager(new StoreConfig(vProps), diskManagerConfig, Utils.newScheduler(1, false), metricRegistry, new MockIdFactory(), clusterMap, dataNode, new DummyMessageStoreHardDelete(), null, SystemTime.getInstance(), new DummyMessageStoreRecovery(), new InMemAccountService(false, false));
    storageManager.start();
    for (ReplicaId replica : replicas) {
        id = replica.getPartitionId();
        assertTrue("Disable compaction should succeed", storageManager.controlCompactionForBlobStore(id, false));
        assertTrue("Shutdown should succeed on given store", storageManager.shutdownBlobStore(id));
        assertTrue("Removing store should succeed", storageManager.removeBlobStore(id));
        assertNull("The store should not exist", storageManager.getStore(id, false));
    }
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
}

Example 8 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class StorageManagerTest method generateConfigs.

/**
 * Generates {@link StoreConfig} and {@link DiskManagerConfig} for use in tests.
 * @param segmentedLog {@code true} to set a segment capacity lower than total store capacity
 * @param updateInstanceConfig whether to update InstanceConfig in Helix
 */
private void generateConfigs(boolean segmentedLog, boolean updateInstanceConfig) {
    List<com.github.ambry.utils.TestUtils.ZkInfo> zkInfoList = new ArrayList<>();
    zkInfoList.add(new com.github.ambry.utils.TestUtils.ZkInfo(null, "DC0", (byte) 0, 2199, false));
    JSONObject zkJson = constructZkLayoutJSON(zkInfoList);
    Properties properties = new Properties();
    properties.put("disk.manager.enable.segment.pooling", "true");
    properties.put("store.compaction.triggers", "Periodic,Admin");
    properties.put("store.replica.status.delegate.enable", "true");
    properties.put("store.set.local.partition.state.enabled", "true");
    properties.setProperty("clustermap.host.name", "localhost");
    properties.setProperty("clustermap.port", "2200");
    properties.setProperty("clustermap.cluster.name", CLUSTER_NAME);
    properties.setProperty("clustermap.datacenter.name", "DC0");
    properties.setProperty("clustermap.dcs.zk.connect.strings", zkJson.toString(2));
    properties.setProperty("clustermap.update.datanode.info", Boolean.toString(updateInstanceConfig));
    if (segmentedLog) {
        long replicaCapacity = clusterMap.getAllPartitionIds(null).get(0).getReplicaIds().get(0).getCapacityInBytes();
        properties.put("store.segment.size.in.bytes", Long.toString(replicaCapacity / 2L));
    }
    VerifiableProperties vProps = new VerifiableProperties(properties);
    diskManagerConfig = new DiskManagerConfig(vProps);
    storeConfig = new StoreConfig(vProps);
    clusterMapConfig = new ClusterMapConfig(vProps);
}

Example 9 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class CompactionManagerTest method testCompactionExecutorHappyPath.

/**
 * Tests that compaction is triggered on all stores provided they do not misbehave. Also includes a store that is
 * not ready for compaction. Ensures that {@link BlobStore#maybeResumeCompaction(byte[])} is called before
 * {@link BlobStore#compact(CompactionDetails, byte[])} is called.
 * @throws Exception
 */
@Test
public void testCompactionExecutorHappyPath() throws Exception {
    int numStores = 5;
    List<BlobStore> stores = new ArrayList<>();
    // one store with nothing to compact isn't going to get compact calls.
    // since we are using mock time, wait for compact calls to arrive twice to ensure the time based scheduling works
    CountDownLatch compactCallsCountdown = new CountDownLatch(2 * (numStores - 1));
    properties.setProperty("store.compaction.triggers", ALL_COMPACTION_TRIGGERS);
    config = new StoreConfig(new VerifiableProperties(properties));
    MetricRegistry metricRegistry = new MetricRegistry();
    StoreMetrics metrics = new StoreMetrics(metricRegistry);
    MockBlobStore lastStore = null;
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = new MockBlobStore(config, metrics, time, compactCallsCountdown, null);
        // one store should not have any segments to compact
        store.details = i == 0 ? null : generateRandomCompactionDetails(i);
        stores.add(store);
        lastStore = store;
    }
    compactionManager = new CompactionManager(MOUNT_PATH, config, stores, new StorageManagerMetrics(metricRegistry), time);
    compactionManager.enable();
    assertNotNull("Compaction thread should be created", TestUtils.getThreadByThisName(CompactionManager.THREAD_NAME_PREFIX));
    assertTrue("Compaction calls did not come within the expected time", compactCallsCountdown.await(1, TimeUnit.SECONDS));
    assertTrue("Compaction Executor should be running", compactionManager.isCompactionExecutorRunning());
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = (MockBlobStore) stores.get(i);
        if (store.callOrderException != null) {
            throw store.callOrderException;
        }
        if (i > 0) {
            assertTrue("Compact was not called", store.compactCalled);
        } else {
            // should not call for i == 0 because there are no compaction details.
            assertFalse("Compact should not have been called", store.compactCalled);
        }
    }
    // test scheduleNextForCompaction()
    lastStore.compactCallsCountdown = new CountDownLatch(1);
    lastStore.compactCalled = false;
    assertTrue("Should schedule compaction", compactionManager.scheduleNextForCompaction(lastStore));
    assertTrue("Compaction call did not come within the expected time", lastStore.compactCallsCountdown.await(1, TimeUnit.HOURS));
    if (lastStore.callOrderException != null) {
        throw lastStore.callOrderException;
    }
    assertTrue("compact() should have been called", lastStore.compactCalled);
    compactionManager.disable();
    compactionManager.awaitTermination();
    assertFalse("Compaction thread should not be running", compactionManager.isCompactionExecutorRunning());
}

Example 10 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class CompactionManagerTest method doTestTrigger.

// testDifferentTriggers() helpers
/**
 * Does the compaction triggers test by running each trigger in {@code triggerRunners} after resetting state.
 * @param triggerRunners the code that triggers compaction.
 */
private void doTestTrigger(List<Runnable> triggerRunners) {
    blobStore.resumeCompactionCalled = false;
    StorageManagerMetrics metrics = new StorageManagerMetrics(new MetricRegistry());
    config = new StoreConfig(new VerifiableProperties(properties));
    compactionManager = new CompactionManager(MOUNT_PATH, config, Collections.singleton(blobStore), metrics, time);
    compactionManager.enable();
    if (config.storeCompactionTriggers[0].isEmpty()) {
        assertNull("Compaction thread should not be created", TestUtils.getThreadByThisName(CompactionManager.THREAD_NAME_PREFIX));
    } else {
        assertNotNull("Compaction thread should be created", TestUtils.getThreadByThisName(CompactionManager.THREAD_NAME_PREFIX));
    }
    for (Runnable triggerRunner : triggerRunners) {
        blobStore.compactCallsCountdown = new CountDownLatch(1);
        blobStore.compactCalled = false;
        triggerRunner.run();
    }
    compactionManager.disable();
    compactionManager.awaitTermination();
}