Search in sources :

Example 1 with StorageService

use of voldemort.server.storage.StorageService in project voldemort by voldemort.

the class ReadOnlyStoreManagementServlet method getReadOnlyStores.

private List<ReadOnlyStorageEngine> getReadOnlyStores(VoldemortServer server) {
    StorageService storage = (StorageService) Utils.notNull(server).getService(ServiceType.STORAGE);
    List<ReadOnlyStorageEngine> l = Lists.newArrayList();
    for (StorageEngine<ByteArray, byte[], byte[]> engine : storage.getStoreRepository().getStorageEnginesByClass(ReadOnlyStorageEngine.class)) {
        l.add((ReadOnlyStorageEngine) engine);
    }
    return l;
}
Also used : ReadOnlyStorageEngine(voldemort.store.readonly.ReadOnlyStorageEngine) ByteArray(voldemort.utils.ByteArray) StorageService(voldemort.server.storage.StorageService)

Example 2 with StorageService

use of voldemort.server.storage.StorageService in project voldemort by voldemort.

the class HintedHandoffSendHintTest method setUp.

@Before
public void setUp() throws Exception {
    if (logger.isDebugEnabled()) {
        logger.debug("Test Started: replication[" + REPLICATION_FACTOR + "], preferredW[" + P_WRITES + "], requiredW[" + R_WRITES + "]");
    }
    cluster = getNineNodeCluster();
    storeDef = getStoreDef();
    // create voldemort servers
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        SocketStoreFactory socketStoreFactory;
        socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
        List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
        stores.add(storeDef);
        VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
        config.setNioAdminConnectorSelectors(1);
        config.setNioConnectorSelectors(2);
        VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
        VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
        StorageService ss = (StorageService) vsrv;
        voldemortServers.put(nodeId, vs);
        slopStorageEngines.put(nodeId, ss.getStoreRepository().getSlopStore());
        slopStores.put(nodeId, SerializingStore.wrap(ss.getStoreRepository().getSlopStore(), new ByteArraySerializer(), new SlopSerializer(), new IdentitySerializer()));
        // wrap original store with force fail store
        Store<ByteArray, byte[], byte[]> store = ss.getStoreRepository().removeLocalStore(STORE_NAME);
        UnreachableStoreException exception = new UnreachableStoreException("Force failed");
        ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = new ForceFailStore<ByteArray, byte[], byte[]>(store, exception);
        forceFailStores.put(nodeId, forceFailStore);
        ss.getStoreRepository().addLocalStore(forceFailStore);
    }
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // create client socket stores and slop stores
    SocketStoreClientFactoryForTest clientSocketStoreFactory = new SocketStoreClientFactoryForTest(STORE_NAME, SLOP_STORE_NAME);
    Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
    Serializer<Slop> slopValueSerializer = new SlopSerializer();
    Map<Integer, Store<ByteArray, byte[], byte[]>> testStores = subStores;
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = new HashMap<Integer, Store<ByteArray, Slop, byte[]>>();
    for (Node node : cluster.getNodes()) {
        // test store
        SocketStore socketTestStore = clientSocketStoreFactory.getSocketTestStoreByNode(node);
        socketTestStores.put(node.getId(), socketTestStore);
        testStores.put(node.getId(), socketTestStore);
        // slop store
        SocketStore socketSlopStore = clientSocketStoreFactory.getSocketSlopStoreByNode(node);
        Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(socketSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        socketSlopStores.put(node.getId(), socketSlopStore);
        slopStores.put(node.getId(), slopStore);
    }
    // set failure detector
    if (failureDetector != null)
        failureDetector.destroy();
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig();
    failureDetectorConfig.setImplementationClassName(failureDetectorCls.getName());
    failureDetectorConfig.setThreshold(50);
    failureDetectorConfig.setCluster(cluster);
    failureDetectorConfig.setConnectionVerifier(MutableStoreConnectionVerifier.create(subStores));
    failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    // make routedStore
    RoutedStoreFactory factory = new RoutedStoreFactory();
    routedStore = factory.create(cluster, storeDef, testStores, socketTestStores, slopStores, socketSlopStores, failureDetector, new RoutedStoreConfig().setTimeoutConfig(new TimeoutConfig(1500L, false)));
    // generate the keys
    for (int i = 0; i < 5; i++) {
        Set<Integer> nodesCovered = Sets.newHashSet();
        while (nodesCovered.size() < NUM_NODES_TOTAL) {
            ByteArray randomKey = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
            byte[] randomValue = TestUtils.randomBytes(VALUE_LENGTH);
            if (randomKey.length() > 0 && randomValue.length > 0) {
                if (!keyList.contains(randomKey)) {
                    for (Node node : strategy.routeRequest(randomKey.get())) {
                        keysToNodes.put(randomKey, node.getId());
                        nodesCovered.add(node.getId());
                    }
                    logger.info("Inserting key [" + randomKey + "] to key list as id:" + keyList.size());
                    keyList.add(randomKey);
                    keyValues.put(randomKey, new ByteArray(randomValue));
                }
            }
        }
    }
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Node(voldemort.cluster.Node) ArrayList(java.util.ArrayList) SocketStore(voldemort.store.socket.SocketStore) Store(voldemort.store.Store) SerializingStore(voldemort.store.serialized.SerializingStore) ForceFailStore(voldemort.store.ForceFailStore) NonblockingStore(voldemort.store.nonblockingstore.NonblockingStore) Properties(java.util.Properties) VoldemortServer(voldemort.server.VoldemortServer) ByteArraySerializer(voldemort.serialization.ByteArraySerializer) VoldemortConfig(voldemort.server.VoldemortConfig) StorageService(voldemort.server.storage.StorageService) VoldemortService(voldemort.common.service.VoldemortService) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray) IdentitySerializer(voldemort.serialization.IdentitySerializer) UnreachableStoreException(voldemort.store.UnreachableStoreException) ForceFailStore(voldemort.store.ForceFailStore) TimeoutConfig(voldemort.client.TimeoutConfig) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) SocketStore(voldemort.store.socket.SocketStore) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) FailureDetectorConfig(voldemort.cluster.failuredetector.FailureDetectorConfig) SlopSerializer(voldemort.serialization.SlopSerializer) Slop(voldemort.store.slop.Slop) Before(org.junit.Before)

Example 3 with StorageService

use of voldemort.server.storage.StorageService in project voldemort by voldemort.

the class HintedHandoffTestEnvironment method startServer.

/**
     * Start a server How it works:
     * 
     * 1. create a server using test utilities
     * 
     * 2.Inject prepared test store and storage engine
     * 
     * 3. Inject prepared slop store and storage engine
     * 
     * @param nodeId The node of server to start
     * @throws IOException
     */
public void startServer(int nodeId) throws IOException {
    if (logger.isInfoEnabled())
        logger.info("Starting server of node [" + nodeId + "]");
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
    List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
    stores.add(storeDef);
    // start a voldemort server
    VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
    config.setNioAdminConnectorSelectors(1);
    config.setNioConnectorSelectors(5);
    config.setSlopFrequencyMs(DEFAULT_SLOP_PUSH_INTERVAL_S * 1000);
    config.setSlopStoreType("memory");
    config.setFailureDetectorAsyncRecoveryInterval(DEFAULT_ASYNC_RECOVERY_INTERVAL_S * 1000);
    VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
    socketStoreFactory.close();
    voldemortServers.put(nodeId, vs);
    VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
    StoreRepository sr = ((StorageService) vsrv).getStoreRepository();
    // storage engine injection
    sr.removeLocalStore(STORE_NAME);
    sr.addLocalStore(sleepyStores.get(nodeId));
    sr.removeStorageEngine(STORE_NAME);
    sr.addStorageEngine((StorageEngine<ByteArray, byte[], byte[]>) realStores.get(nodeId));
    // slop stores caching and injection
    if (!slopStorageEngines.containsKey(nodeId)) {
        SlopStorageEngine slopStorageEngine = sr.getSlopStore();
        slopStorageEngines.put(nodeId, slopStorageEngine);
    } else {
        sr.removeStorageEngine("slop");
        sr.removeLocalStore("slop");
        sr.addStorageEngine(slopStorageEngines.get(nodeId));
        sr.addLocalStore(slopStorageEngines.get(nodeId));
        sr.setSlopStore(slopStorageEngines.get(nodeId));
    }
}
Also used : ArrayList(java.util.ArrayList) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) StoreRepository(voldemort.server.StoreRepository) Properties(java.util.Properties) VoldemortServer(voldemort.server.VoldemortServer) VoldemortConfig(voldemort.server.VoldemortConfig) StorageService(voldemort.server.storage.StorageService) VoldemortService(voldemort.common.service.VoldemortService) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray)

Example 4 with StorageService

use of voldemort.server.storage.StorageService in project voldemort by voldemort.

the class ServerStoreConnectionVerifierTest method testMetadataStore.

@Test
public void testMetadataStore() throws Exception {
    for (Node node : cluster.getNodes()) {
        VoldemortServer voldemortServer = serverMap.get(node.getId());
        StorageService ss = (StorageService) voldemortServer.getService(ServiceType.STORAGE);
        ServerStoreConnectionVerifier ssv = new ServerStoreConnectionVerifier(ss.getSocketStoreFactory(), voldemortServer.getMetadataStore(), voldemortServer.getVoldemortConfig());
        for (Node siblingNodes : cluster.getNodes()) ssv.verifyConnection(siblingNodes);
    }
}
Also used : Node(voldemort.cluster.Node) VoldemortServer(voldemort.server.VoldemortServer) StorageService(voldemort.server.storage.StorageService) Test(org.junit.Test)

Example 5 with StorageService

use of voldemort.server.storage.StorageService in project voldemort by voldemort.

the class VoldemortServer method createBasicServices.

private List<VoldemortService> createBasicServices() {
    /* Services are given in the order they must be started */
    List<VoldemortService> services = new ArrayList<VoldemortService>();
    SchedulerService scheduler = new SchedulerService(voldemortConfig.getSchedulerThreads(), SystemTime.INSTANCE, voldemortConfig.canInterruptService());
    storageService = new StorageService(storeRepository, metadata, scheduler, voldemortConfig);
    asyncService = new AsyncOperationService(scheduler, ASYNC_REQUEST_CACHE_SIZE);
    jmxService = null;
    services.add(storageService);
    services.add(scheduler);
    services.add(asyncService);
    if (voldemortConfig.isAdminServerEnabled()) {
        Rebalancer rebalancer = null;
        if (voldemortConfig.isEnableRebalanceService()) {
            RebalancerService rebalancerService = new RebalancerService(storeRepository, metadata, voldemortConfig, asyncService, scheduler);
            services.add(rebalancerService);
            rebalancer = rebalancerService.getRebalancer();
        }
        SocketRequestHandlerFactory adminRequestHandlerFactory = new SocketRequestHandlerFactory(storageService, this.storeRepository, this.metadata, this.voldemortConfig, this.asyncService, scheduler, rebalancer, this);
        if (voldemortConfig.getUseNioConnector()) {
            logger.info("Using NIO Connector for Admin Service.");
            services.add(new NioSocketService(adminRequestHandlerFactory, identityNode.getAdminPort(), voldemortConfig.getAdminSocketBufferSize(), voldemortConfig.isNioAdminConnectorKeepAlive(), voldemortConfig.getNioAdminConnectorSelectors(), "admin-server", voldemortConfig.isJmxEnabled(), voldemortConfig.getNioAcceptorBacklog(), voldemortConfig.getNioSelectorMaxHeartBeatTimeMs()));
        } else {
            logger.info("Using BIO Connector for Admin Service.");
            services.add(new SocketService(adminRequestHandlerFactory, identityNode.getAdminPort(), voldemortConfig.getAdminCoreThreads(), voldemortConfig.getAdminMaxThreads(), voldemortConfig.getAdminSocketBufferSize(), "admin-server", voldemortConfig.isJmxEnabled()));
        }
    }
    if (voldemortConfig.isGossipEnabled()) {
        services.add(new GossipService(this.metadata, scheduler, voldemortConfig));
    }
    if (voldemortConfig.isJmxEnabled()) {
        jmxService = new JmxService(this, this.metadata.getCluster(), storeRepository, services);
        services.add(jmxService);
    }
    return ImmutableList.copyOf(services);
}
Also used : SchedulerService(voldemort.common.service.SchedulerService) Rebalancer(voldemort.server.rebalance.Rebalancer) JmxService(voldemort.server.jmx.JmxService) ArrayList(java.util.ArrayList) NioSocketService(voldemort.server.niosocket.NioSocketService) NioSocketService(voldemort.server.niosocket.NioSocketService) SocketService(voldemort.server.socket.SocketService) AsyncOperationService(voldemort.server.protocol.admin.AsyncOperationService) RebalancerService(voldemort.server.rebalance.RebalancerService) StorageService(voldemort.server.storage.StorageService) GossipService(voldemort.server.gossip.GossipService) VoldemortService(voldemort.common.service.VoldemortService) SocketRequestHandlerFactory(voldemort.server.protocol.SocketRequestHandlerFactory)

Aggregations

StorageService (voldemort.server.storage.StorageService)5 ArrayList (java.util.ArrayList)3 VoldemortService (voldemort.common.service.VoldemortService)3 VoldemortServer (voldemort.server.VoldemortServer)3 ByteArray (voldemort.utils.ByteArray)3 Properties (java.util.Properties)2 Node (voldemort.cluster.Node)2 VoldemortConfig (voldemort.server.VoldemortConfig)2 StoreDefinition (voldemort.store.StoreDefinition)2 SocketStoreFactory (voldemort.store.socket.SocketStoreFactory)2 ClientRequestExecutorPool (voldemort.store.socket.clientrequest.ClientRequestExecutorPool)2 HashMap (java.util.HashMap)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 Before (org.junit.Before)1 Test (org.junit.Test)1 TimeoutConfig (voldemort.client.TimeoutConfig)1 FailureDetectorConfig (voldemort.cluster.failuredetector.FailureDetectorConfig)1 SchedulerService (voldemort.common.service.SchedulerService)1 RoutingStrategyFactory (voldemort.routing.RoutingStrategyFactory)1 ByteArraySerializer (voldemort.serialization.ByteArraySerializer)1