Examples with ClientConfig voldemort.client.ClientConfig used on opensource projects

Example 46 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testProxyPutDuringRebalancing.

@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
    logger.info("Starting testProxyPutDuringRebalancing");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0 }, { 1, 3 }, { 2 } });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Lists.newArrayList(3));
        // start servers 0,1,2 only
        final List<Integer> serverList = Arrays.asList(0, 1, 2);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        // Its is imperative that we test in a single shot since multiple
        // batches would mean the proxy bridges being torn down and
        // established multiple times and we cannot test against the source
        // cluster topology then.
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
        final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
        // the plan would cause these partitions to move
        // Partition : Donor -> Stealer
        // p2 (SEC) : s1 -> s0
        // p3 (PRI) : s1 -> s2
        final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 1, rwStoreDefWithReplication.getName(), Arrays.asList(2, 3), 20);
        assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
        final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
        final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
        final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
        final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
        for (String key : baselineTuples.keySet()) {
            baselineVersions.put(key, new VectorClock());
        }
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                SocketStoreClientFactory factory = null;
                try {
                    // wait for the rebalancing to begin.
                    List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(0), serverMap.get(2));
                    while (!rebalancingComplete.get()) {
                        Iterator<VoldemortServer> serverIterator = serverList.iterator();
                        while (serverIterator.hasNext()) {
                            VoldemortServer server = serverIterator.next();
                            if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
                                logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
                                serverIterator.remove();
                            }
                        }
                        if (serverList.size() == 0) {
                            rebalancingStarted.set(true);
                            break;
                        }
                    }
                    if (!rebalancingComplete.get()) {
                        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
                        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
                        // zero vector clock
                        for (ByteArray movingKey : movingKeysList) {
                            try {
                                if (rebalancingComplete.get()) {
                                    break;
                                }
                                String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
                                String valStr = "proxy_write";
                                storeClientRW.put(keyStr, valStr);
                                baselineTuples.put(keyStr, valStr);
                                // all these keys will have [2:1] vector
                                // clock
                                // is node 2 is the pseudo master in both
                                // moves
                                baselineVersions.get(keyStr).incrementVersion(2, System.currentTimeMillis());
                                proxyWritesDone.set(true);
                            } catch (InvalidMetadataException e) {
                                // let this go
                                logger.error("Encountered an invalid metadata exception.. ", e);
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy put thread", e);
                    exceptions.add(e);
                } finally {
                    if (factory != null)
                        factory.close();
                    latch.countDown();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    rebalanceKit.rebalance();
                } catch (Exception e) {
                    logger.error("Error in rebalancing... ", e);
                    exceptions.add(e);
                } finally {
                    rebalancingComplete.set(true);
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
        assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
        checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(0, 1, 2), baselineTuples, baselineVersions);
        checkConsistentMetadata(finalCluster, serverList);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
        // check that the proxy writes were made to the original donor, node
        // 1
        List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
        for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
        VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
        for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
        adminClient.setAdminClientCluster(currentCluster);
        checkForTupleEquivalence(adminClient, 1, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
        // stop servers
        try {
            stopServer(serverList);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
        throw ae;
    }
}

Example 47 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class CompressingStoreTest method testPutGetWithSocketService.

@Test
public void testPutGetWithSocketService() throws Exception {
    int freePort = ServerTestUtils.findFreePort();
    String clusterXml = VoldemortTestConstants.getOneNodeClusterXml();
    clusterXml = clusterXml.replace("<socket-port>6666</socket-port>", "<socket-port>" + freePort + "</socket-port>");
    AbstractSocketService socketService = ServerTestUtils.getSocketService(useNio, clusterXml, VoldemortTestConstants.getCompressedStoreDefinitionsXml(), "test", freePort);
    socketService.start();
    Thread.sleep(1000);
    SocketStoreClientFactory storeClientFactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls("tcp://localhost:" + freePort).setMaxBootstrapRetries(10));
    StoreClient<String, String> storeClient = storeClientFactory.getStoreClient("test");
    storeClient.put("someKey", "someValue");
    assertEquals(storeClient.getValue("someKey"), "someValue");
    socketService.stop();
}

Example 48 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class AbstractZoneAffinityTest method setup.

@Before
public void setup() throws IOException {
    byte[] v1_bytes = { (byte) 'V', (byte) '1' };
    byte[] v2_bytes = { (byte) 'V', (byte) '2' };
    byte[] k1_bytes = { (byte) 'K', (byte) '1' };
    byte[] k2_bytes = { (byte) 'K', (byte) '2' };
    byte[] k3_bytes = { (byte) 'K', (byte) '3' };
    clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(cluster.getNodes().iterator().next().getSocketUrl().toString());
    clientConfig.setClientZoneId(clientZoneId);
    setupZoneAffinitySettings();
    SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
    for (Integer nodeId : cluster.getNodeIds()) {
        SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
        VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
        VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
        vservers.put(nodeId, vs);
        socketStoreFactories.put(nodeId, socketStoreFactory);
        Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getLocalStore(storeDef.getName());
        Node node = cluster.getNodeById(nodeId);
        VectorClock version1 = new VectorClock();
        version1.incrementVersion(0, System.currentTimeMillis());
        VectorClock version2 = version1.incremented(0, System.currentTimeMillis());
        if (node.getZoneId() == clientZoneId) {
            // local zone
            store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
            store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
        } else {
            // remote zone
            store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v2_bytes, version2), null);
            store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
            store.put(new ByteArray(k3_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
        }
    }
    client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
}

Example 49 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class TestClientShutdown method main.

public static void main(String[] args) throws Exception {
    if (args.length < 2 || args.length > 3)
        Utils.croak("USAGE: java VoldemortClientShell store_name bootstrap_url [command_file]");
    String storeName = args[0];
    String bootstrapUrl = args[1];
    StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setEnableLazy(false).setBootstrapUrls(bootstrapUrl));
    DefaultStoreClient<Object, Object> client = null;
    try {
        client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(storeName);
    } catch (Exception e) {
        Utils.croak("Could not connect to server: " + e.getMessage());
    }
    System.out.println("Established connection to " + storeName + " via " + bootstrapUrl);
    client.get("hello");
    System.out.println("Got value");
}

Example 50 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class RemoteDataGenerator method generateData.

/**
     * Populate the store with
     * 
     * @param requests key-value pairs with key of size, appending
     * @param postfix for uniqueness.
     * 
     * @param keySize bytes and value of size
     * @param valueSize bytes
     * @param requests How many key-value pairs to generate
     * @param keySize Size (in bytes) of the key
     * @param valueSize Size (in bytes) of the value
     * @param postfix Postfix to append (for uniqueness)
     */
public void generateData(int requests, int keySize, int valueSize, String postfix) {
    StoreClientFactory storeClientFactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(url).setMaxThreads(workers));
    StoreClient<String, String> client = storeClientFactory.getStoreClient(storeName);
    for (int i = 0; i < requests; i++) {
        StringBuilder keyBuilder = new StringBuilder(makeString(keySize)).append(i);
        StringBuilder valueBuilder = new StringBuilder(makeString(valueSize)).append(i);
        if (postfix != null) {
            keyBuilder.append(postfix);
        }
        try {
            client.put(keyBuilder.toString(), valueBuilder.toString());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}