Examples with SocketStoreClientFactory voldemort.client.SocketStoreClientFactory used on opensource projects

Example 6 with SocketStoreClientFactory

use of voldemort.client.SocketStoreClientFactory in project voldemort by voldemort.

the class ZoneShrinkageClientTest method testZoneAffinityClient.

@Test
public void testZoneAffinityClient() {
    ZoneAffinity zoneAffinity = new ZoneAffinity(true, true, true);
    ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(sourceCluster.getNodeById(0).getSocketUrl().toString()).setClientZoneId(DROP_ZONE_ID).setZoneAffinity(zoneAffinity);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(clientConfig);
    StoreClient<String, String> client = factory.getStoreClient(sourceStoreDefs.get(sourceStoreDefs.size() - 1).getName());
    // do some operations against the stores from zone 0.
    for (int i = 0; i < 10; i++) {
        try {
            client.put("key" + i, "val" + i);
            assertEquals("Must read value back", "val" + i, client.get("key" + i).getValue());
        } catch (Exception e) {
            fail("Should be not see any failures");
        }
    }
    // shrink the cluster, by dropping zone 0
    String clusterXmlString = new ClusterMapper().writeCluster(targetCluster);
    String storesXmlString = new StoreDefinitionsMapper().writeStoreList(targetStoreDefs);
    int[] remoteNodeList = new int[sourceCluster.getNumberOfNodes()];
    int ni = 0;
    for (Integer nodeId : sourceCluster.getNodeIds()) {
        remoteNodeList[ni++] = nodeId;
    }
    adminClient.metadataMgmtOps.updateRemoteMetadataPair(new ArrayList<Integer>(sourceCluster.getNodeIds()), "cluster.xml", new Versioned<String>(clusterXmlString, TestUtils.getClock(remoteNodeList)), "stores.xml", new Versioned<String>(storesXmlString, TestUtils.getClock(remoteNodeList)));
    try {
        Thread.sleep(clientConfig.getAsyncMetadataRefreshInMs() * 2);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    // do more operations, the client should fail.
    for (int i = 0; i < 10; i++) {
        long startMs = System.currentTimeMillis();
        try {
            client.put("key" + i, "val" + i);
            assertEquals("Must read value back", "val" + i, client.get("key" + i).getValue());
            fail("Should be not see any successes");
        } catch (Exception e) {
            e.printStackTrace();
            long elapsedMs = System.currentTimeMillis() - startMs;
            assertTrue("Operation took longer than timeout to fail :" + elapsedMs + " ms.", elapsedMs < clientConfig.getRoutingTimeout(TimeUnit.MILLISECONDS));
        }
    }
}

Example 7 with SocketStoreClientFactory

use of voldemort.client.SocketStoreClientFactory in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method testProxyPutDuringRebalancing.

@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
    logger.info("Starting testProxyPutDuringRebalancing");
    try {
        int[] zoneIds = new int[] { 1, 3 };
        int[][] nodesPerZone = new int[][] { { 3, 4, 5 }, { 9, 10, 11 } };
        int[][] partitionMap = new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } };
        Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 5, Lists.newArrayList(7));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 11, Lists.newArrayList(6));
        /**
             * Original partition map
             *
             * [s3 : p0] [s4 : p1, p6] [s5 : p2]
             *
             * [s9 : p3] [s10 : p4, p7] [s11 : p5]
             *
             * final server partition ownership
             *
             * [s3 : p0] [s4 : p1] [s5 : p2, p7]
             *
             * [s9 : p3] [s10 : p4] [s11 : p5, p6]
             *
             * Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
             *
             * Original server n-ary partition ownership
             *
             * [s3 : p0, p3-7] [s4 : p0-p7] [s5 : p1-2]
             *
             * [s9 : p0-3, p6-7] [s10 : p0-p7] [s11 : p4-5]
             *
             * final server n-ary partition ownership
             *
             * [s3 : p0, p2-7] [s4 : p0-1] [s5 : p1-p7]
             *
             * [s9 : p0-3, p5-7] [s10 : p0-4, p7] [s11 : p4-6]
             */
        List<Integer> serverList = Arrays.asList(3, 4, 5, 9, 10, 11);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        // Its is imperative that we test in a single shot since multiple batches would mean the proxy bridges
        // being torn down and established multiple times and we cannot test against the source
        // cluster topology then. getRebalanceKit uses batch size of infinite, so this should be fine.
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        populateData(currentCluster, rwStoreDefWithReplication);
        final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
        // the plan would cause these partitions to move:
        // Partition : Donor -> stealer
        //
        // p2 (Z-SEC) : s4 -> s3
        // p3-6 (Z-PRI) : s4 -> s5
        // p7 (Z-PRI) : s3 -> s5
        //
        // p5 (Z-SEC): s10 -> s9
        // p6 (Z-PRI): s10 -> s11
        //
        // Rebalancing will run on servers 3, 5, 9, & 11
        final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 4, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
        assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
        final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
        final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
        final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
        final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
        for (String key : baselineTuples.keySet()) {
            baselineVersions.put(key, new VectorClock());
        }
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                SocketStoreClientFactory factory = null;
                try {
                    // wait for the rebalancing to begin
                    List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(3), serverMap.get(5), serverMap.get(9), serverMap.get(11));
                    while (!rebalancingComplete.get()) {
                        Iterator<VoldemortServer> serverIterator = serverList.iterator();
                        while (serverIterator.hasNext()) {
                            VoldemortServer server = serverIterator.next();
                            if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
                                logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
                                serverIterator.remove();
                            }
                        }
                        if (serverList.size() == 0) {
                            rebalancingStarted.set(true);
                            break;
                        }
                    }
                    if (rebalancingStarted.get()) {
                        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(3));
                        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
                        // Initially, all data now with zero vector clock
                        for (ByteArray movingKey : movingKeysList) {
                            try {
                                String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
                                String valStr = "proxy_write";
                                storeClientRW.put(keyStr, valStr);
                                baselineTuples.put(keyStr, valStr);
                                baselineVersions.get(keyStr).incrementVersion(11, System.currentTimeMillis());
                                proxyWritesDone.set(true);
                                if (rebalancingComplete.get()) {
                                    break;
                                }
                            } catch (InvalidMetadataException e) {
                                logger.error("Encountered an invalid metadata exception.. ", e);
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy write thread..", e);
                    exceptions.add(e);
                } finally {
                    if (factory != null)
                        factory.close();
                    latch.countDown();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    rebalanceKit.rebalance();
                } catch (Exception e) {
                    logger.error("Error in rebalancing... ", e);
                    exceptions.add(e);
                } finally {
                    rebalancingComplete.set(true);
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
        assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
        checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(3, 4, 5, 9, 10, 11), baselineTuples, baselineVersions);
        checkConsistentMetadata(finalCluster, serverList);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
        // check that the proxy writes were made to the original donor, node 4
        List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
        for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
        VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
        for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
        adminClient.setAdminClientCluster(currentCluster);
        checkForTupleEquivalence(adminClient, 4, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
        // stop servers
        try {
            stopServer(serverList);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
        throw ae;
    }
}

Example 8 with SocketStoreClientFactory

use of voldemort.client.SocketStoreClientFactory in project voldemort by voldemort.

the class DynamicTimeoutStoreClientTest method setUp.

/**
     * Setup a one node Voldemort cluster with a 'slow' store
     * (SlowStorageEngine) with a delay of 500 ms for get and put.
     * 
     * @throws java.lang.Exception
     */
@Before
public void setUp() throws Exception {
    int numServers = 1;
    servers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 2, 4, 6, 1, 3, 5, 7 } };
    Properties props = new Properties();
    props.setProperty("storage.configs", "voldemort.store.bdb.BdbStorageConfiguration,voldemort.store.slow.SlowStorageConfiguration");
    props.setProperty("testing.slow.queueing.get.ms", SLOW_STORE_DELAY);
    props.setProperty("testing.slow.queueing.put.ms", SLOW_STORE_DELAY);
    cluster = ServerTestUtils.startVoldemortCluster(numServers, servers, partitionMap, socketStoreFactory, // useNio
    true, null, STORES_XML, props);
    socketUrl = servers[0].getIdentityNode().getSocketUrl().toString();
    String bootstrapUrl = socketUrl;
    ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(bootstrapUrl).setEnableCompressionLayer(false).setEnableSerializationLayer(false).enableDefaultClient(true).setEnableLazy(false);
    String storesXml = FileUtils.readFileToString(new File(STORES_XML), "UTF-8");
    ClusterMapper mapper = new ClusterMapper();
    this.dynamicTimeoutClient = new DynamicTimeoutStoreClient<ByteArray, byte[]>(STORE_NAME, new SocketStoreClientFactory(clientConfig), 1, storesXml, mapper.writeCluster(cluster));
}

Example 9 with SocketStoreClientFactory

use of voldemort.client.SocketStoreClientFactory in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testProxyGetDuringRebalancing.

@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
    logger.info("Starting testProxyGetDuringRebalancing");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(4, 2, new int[] { 0, 0, 1, 1 }, new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } });
        Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(2));
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 1, Lists.newArrayList(3));
        final List<Integer> serverList = Arrays.asList(0, 1, 2, 3);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithReplication);
            final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
            final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
            final CountDownLatch latch = new CountDownLatch(2);
            // start get operation.
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        List<String> keys = new ArrayList<String>(testEntries.keySet());
                        while (!rebalancingComplete.get()) {
                            // should always able to get values.
                            int index = (int) (Math.random() * keys.size());
                            // should get a valid value
                            try {
                                Versioned<String> value = storeClientRW.get(keys.get(index));
                                assertNotSame("StoreClient get() should not return null.", null, value);
                                assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                            } catch (Exception e) {
                                logger.error("Exception in proxy get thread", e);
                                e.printStackTrace();
                                exceptions.add(e);
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Exception in proxy get thread", e);
                        exceptions.add(e);
                    } finally {
                        factory.close();
                        latch.countDown();
                    }
                }
            });
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        Thread.sleep(500);
                        rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2, 3));
                        Thread.sleep(500);
                        rebalancingComplete.set(true);
                        checkConsistentMetadata(finalCluster, serverList);
                    } catch (Exception e) {
                        exceptions.add(e);
                    } finally {
                        // stop servers
                        try {
                            stopServer(serverList);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                        latch.countDown();
                    }
                }
            });
            latch.await();
            executors.shutdown();
            executors.awaitTermination(300, TimeUnit.SECONDS);
            // check No Exception
            if (exceptions.size() > 0) {
                for (Exception e : exceptions) {
                    e.printStackTrace();
                }
                fail("Should not see any exceptions.");
            }
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
        throw ae;
    }
}

Example 10 with SocketStoreClientFactory

use of voldemort.client.SocketStoreClientFactory in project voldemort by voldemort.

the class CoordinatorProxyService method getFatClientFactory.

/**
     * Create a @SocketStoreClientFactory from the given configPops
     * 
     * @param bootstrapURLs
     * @param configProps
     * @return
     */
private SocketStoreClientFactory getFatClientFactory(String[] bootstrapURLs, Properties configProps) {
    ClientConfig fatClientConfig = new ClientConfig(configProps);
    logger.info("Using config: " + fatClientConfig);
    fatClientConfig.setBootstrapUrls(bootstrapURLs).setEnableCompressionLayer(false).setEnableSerializationLayer(false).enableDefaultClient(true).setEnableLazy(false);
    return new SocketStoreClientFactory(fatClientConfig);
}