Search in sources :

Example 6 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class BlockingSlopPusherTest method setUp.

@Override
protected void setUp() throws Exception {
    Cluster cluster = makeCluster(3);
    repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
    MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
    Props props = new Props();
    props.put("node.id", 0);
    props.put("voldemort.home", "/");
    pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}
Also used : BlockingSlopPusherJob(voldemort.server.scheduler.slop.BlockingSlopPusherJob) NoopFailureDetector(voldemort.cluster.failuredetector.NoopFailureDetector) ScanPermitWrapper(voldemort.server.storage.ScanPermitWrapper) Cluster(voldemort.cluster.Cluster) StoreRepository(voldemort.server.StoreRepository) Props(voldemort.utils.Props) VoldemortConfig(voldemort.server.VoldemortConfig) MetadataStore(voldemort.store.metadata.MetadataStore) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) ByteArray(voldemort.utils.ByteArray) SlopStorageEngine(voldemort.store.slop.SlopStorageEngine)

Example 7 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class RemoteStoreComparisonTest method main.

public static void main(String[] args) throws Exception {
    if (args.length != 2)
        Utils.croak("USAGE: java " + RemoteStoreComparisonTest.class.getName() + " numRequests numThreads [useNio]");
    int numRequests = Integer.parseInt(args[0]);
    int numThreads = Integer.parseInt(args[1]);
    boolean useNio = args.length > 2 ? args[2].equals("true") : false;
    /** * In memory test ** */
    final Store<byte[], byte[], byte[]> memStore = new InMemoryStorageEngine<byte[], byte[], byte[]>("test");
    PerformanceTest memWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            memStore.put(key, new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing memory write test.");
    memWriteTest.run(numRequests, numThreads);
    memWriteTest.printStats();
    System.out.println();
    PerformanceTest memReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                memStore.get(String.valueOf(i).getBytes(), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing memory read test.");
    memReadTest.run(numRequests, numThreads);
    memReadTest.printStats();
    System.out.println();
    System.out.println();
    /** * Do Socket tests ** */
    String storeName = "test";
    StoreRepository repository = new StoreRepository();
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    SocketStoreFactory storeFactory = new ClientRequestExecutorPool(10, 1000, 1000, 32 * 1024);
    final Store<ByteArray, byte[], byte[]> socketStore = storeFactory.create(storeName, "localhost", 6666, RequestFormatType.VOLDEMORT_V1, RequestRoutingType.NORMAL);
    RequestHandlerFactory factory = ServerTestUtils.getSocketRequestHandlerFactory(repository);
    AbstractSocketService socketService = ServerTestUtils.getSocketService(useNio, factory, 6666, 50, 50, 1000);
    socketService.start();
    PerformanceTest socketWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] bytes = String.valueOf(i).getBytes();
            ByteArray key = new ByteArray(bytes);
            socketStore.put(key, new Versioned<byte[]>(bytes), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing socket write test.");
    socketWriteTest.run(numRequests, numThreads);
    socketWriteTest.printStats();
    System.out.println();
    PerformanceTest socketReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                socketStore.get(TestUtils.toByteArray(String.valueOf(i)), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing socket read test.");
    socketReadTest.run(numRequests, 1);
    socketReadTest.printStats();
    System.out.println();
    System.out.println();
    socketStore.close();
    storeFactory.close();
    socketService.stop();
    /** * Do HTTP tests ** */
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    HttpService httpService = new HttpService(null, null, repository, RequestFormatType.VOLDEMORT_V0, numThreads, 8080);
    httpService.start();
    ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager(SchemeRegistryFactory.createDefault(), 10000, TimeUnit.MILLISECONDS);
    DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);
    HttpParams clientParams = httpClient.getParams();
    httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
    HttpClientParams.setCookiePolicy(clientParams, CookiePolicy.IGNORE_COOKIES);
    HttpProtocolParams.setUserAgent(clientParams, "test-agent");
    HttpProtocolParams.setVersion(clientParams, HttpVersion.HTTP_1_1);
    HttpConnectionParams.setConnectionTimeout(clientParams, 10000);
    connectionManager.setMaxTotal(numThreads);
    connectionManager.setDefaultMaxPerRoute(numThreads);
    HttpConnectionParams.setStaleCheckingEnabled(clientParams, false);
    final HttpStore httpStore = new HttpStore("test", "localhost", 8080, httpClient, new RequestFormatFactory().getRequestFormat(RequestFormatType.VOLDEMORT_V0), false);
    Thread.sleep(400);
    PerformanceTest httpWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            httpStore.put(new ByteArray(key), new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing HTTP write test.");
    httpWriteTest.run(numRequests, numThreads);
    httpWriteTest.printStats();
    System.out.println();
    PerformanceTest httpReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            httpStore.get(new ByteArray(String.valueOf(i).getBytes()), null);
        }
    };
    System.out.println("Performing HTTP read test.");
    httpReadTest.run(numRequests, numThreads);
    httpReadTest.printStats();
    httpService.stop();
    VoldemortIOUtils.closeQuietly(httpClient);
}
Also used : RequestFormatFactory(voldemort.client.protocol.RequestFormatFactory) RequestHandlerFactory(voldemort.server.protocol.RequestHandlerFactory) DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) StoreRepository(voldemort.server.StoreRepository) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) DefaultHttpClient(org.apache.http.impl.client.DefaultHttpClient) HttpParams(org.apache.http.params.HttpParams) ThreadSafeClientConnManager(org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) HttpService(voldemort.server.http.HttpService) HttpStore(voldemort.store.http.HttpStore) ByteArray(voldemort.utils.ByteArray) AbstractSocketService(voldemort.server.AbstractSocketService)

Example 8 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class HintedHandoffTestEnvironment method startServer.

/**
     * Start a server How it works:
     * 
     * 1. create a server using test utilities
     * 
     * 2.Inject prepared test store and storage engine
     * 
     * 3. Inject prepared slop store and storage engine
     * 
     * @param nodeId The node of server to start
     * @throws IOException
     */
public void startServer(int nodeId) throws IOException {
    if (logger.isInfoEnabled())
        logger.info("Starting server of node [" + nodeId + "]");
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
    List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
    stores.add(storeDef);
    // start a voldemort server
    VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
    config.setNioAdminConnectorSelectors(1);
    config.setNioConnectorSelectors(5);
    config.setSlopFrequencyMs(DEFAULT_SLOP_PUSH_INTERVAL_S * 1000);
    config.setSlopStoreType("memory");
    config.setFailureDetectorAsyncRecoveryInterval(DEFAULT_ASYNC_RECOVERY_INTERVAL_S * 1000);
    VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
    socketStoreFactory.close();
    voldemortServers.put(nodeId, vs);
    VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
    StoreRepository sr = ((StorageService) vsrv).getStoreRepository();
    // storage engine injection
    sr.removeLocalStore(STORE_NAME);
    sr.addLocalStore(sleepyStores.get(nodeId));
    sr.removeStorageEngine(STORE_NAME);
    sr.addStorageEngine((StorageEngine<ByteArray, byte[], byte[]>) realStores.get(nodeId));
    // slop stores caching and injection
    if (!slopStorageEngines.containsKey(nodeId)) {
        SlopStorageEngine slopStorageEngine = sr.getSlopStore();
        slopStorageEngines.put(nodeId, slopStorageEngine);
    } else {
        sr.removeStorageEngine("slop");
        sr.removeLocalStore("slop");
        sr.addStorageEngine(slopStorageEngines.get(nodeId));
        sr.addLocalStore(slopStorageEngines.get(nodeId));
        sr.setSlopStore(slopStorageEngines.get(nodeId));
    }
}
Also used : ArrayList(java.util.ArrayList) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) StoreRepository(voldemort.server.StoreRepository) Properties(java.util.Properties) VoldemortServer(voldemort.server.VoldemortServer) VoldemortConfig(voldemort.server.VoldemortConfig) StorageService(voldemort.server.storage.StorageService) VoldemortService(voldemort.common.service.VoldemortService) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray)

Example 9 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class DataCleanupJobTest method setUp.

@Before
public void setUp() throws Exception {
    time = new MockTime();
    storeDir = TestUtils.createTempDir();
    FileDeleteStrategy.FORCE.delete(storeDir);
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(1024 * 1024);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(storeDir.toURI().getPath());
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    StoreDefinition storeDef = getStoreDef(START_RETENTION);
    engine = bdbStorage.getStore(storeDef, TestUtils.makeSingleNodeRoutingStrategy());
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(0, "test-host", 1234, 1235, 1236, Arrays.asList(0)));
    Cluster cluster = new Cluster("cluster", nodes);
    StoreRepository repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, engine);
    metadataStore = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
}
Also used : InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) StoreDefinition(voldemort.store.StoreDefinition) Node(voldemort.cluster.Node) Cluster(voldemort.cluster.Cluster) StoreRepository(voldemort.server.StoreRepository) SlopStorageEngine(voldemort.store.slop.SlopStorageEngine) Props(voldemort.utils.Props) BdbStorageConfiguration(voldemort.store.bdb.BdbStorageConfiguration) MockTime(voldemort.MockTime) VoldemortConfig(voldemort.server.VoldemortConfig) Before(org.junit.Before)

Example 10 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class RepairJobTest method setUp.

public void setUp() {
    File temp = TestUtils.createTempDir();
    VoldemortConfig config = new VoldemortConfig(0, temp.getAbsolutePath());
    new File(config.getMetadataDirectory()).mkdir();
    this.serverMap = new HashMap<Integer, VoldemortServer>();
    this.scheduler = new SchedulerService(1, new MockTime());
    this.cluster = VoldemortTestConstants.getNineNodeCluster();
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    this.storeDefs = mapper.readStoreList(new StringReader((VoldemortTestConstants.getSingleStore322Xml())));
    this.storeRepository = new StoreRepository();
    this.metadataStore = ServerTestUtils.createMetadataStore(cluster, storeDefs);
    storage = new StorageService(storeRepository, metadataStore, scheduler, config);
    // Start the storage service
    storage.start();
    this.socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    String storeDefsString = mapper.writeStoreList(storeDefs);
    File file = null;
    try {
        file = ServerTestUtils.createTempFile("single-store-", ".xml");
        FileUtils.writeStringToFile(file, storeDefsString);
        String storeDefFile = file.getAbsolutePath();
        List<Integer> nodesToStart = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8);
        // Start the servers
        startServers(cluster, storeDefFile, nodesToStart, null);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : SchedulerService(voldemort.common.service.SchedulerService) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) StoreRepository(voldemort.server.StoreRepository) VoldemortServer(voldemort.server.VoldemortServer) VoldemortConfig(voldemort.server.VoldemortConfig) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) StringReader(java.io.StringReader) File(java.io.File) MockTime(voldemort.MockTime)

Aggregations

StoreRepository (voldemort.server.StoreRepository)14 InMemoryStorageEngine (voldemort.store.memory.InMemoryStorageEngine)6 ByteArray (voldemort.utils.ByteArray)6 ClientRequestExecutorPool (voldemort.store.socket.clientrequest.ClientRequestExecutorPool)5 Before (org.junit.Before)4 VoldemortConfig (voldemort.server.VoldemortConfig)4 VoldemortServer (voldemort.server.VoldemortServer)4 StoreDefinition (voldemort.store.StoreDefinition)4 File (java.io.File)3 Properties (java.util.Properties)3 MockTime (voldemort.MockTime)3 Cluster (voldemort.cluster.Cluster)3 RequestHandlerFactory (voldemort.server.protocol.RequestHandlerFactory)3 SocketStoreFactory (voldemort.store.socket.SocketStoreFactory)3 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)3 StringReader (java.io.StringReader)2 Node (voldemort.cluster.Node)2 SchedulerService (voldemort.common.service.SchedulerService)2 MetadataStore (voldemort.store.metadata.MetadataStore)2 SlopStorageEngine (voldemort.store.slop.SlopStorageEngine)2