Search in sources :

Example 1 with DefaultStoreClient

use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testProxyGetDuringRebalancing.

@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
    logger.info("Starting testProxyGetDuringRebalancing");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(4, 2, new int[] { 0, 0, 1, 1 }, new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } });
        Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(2));
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 1, Lists.newArrayList(3));
        final List<Integer> serverList = Arrays.asList(0, 1, 2, 3);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithReplication);
            final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
            final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
            final CountDownLatch latch = new CountDownLatch(2);
            // start get operation.
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        List<String> keys = new ArrayList<String>(testEntries.keySet());
                        while (!rebalancingComplete.get()) {
                            // should always able to get values.
                            int index = (int) (Math.random() * keys.size());
                            // should get a valid value
                            try {
                                Versioned<String> value = storeClientRW.get(keys.get(index));
                                assertNotSame("StoreClient get() should not return null.", null, value);
                                assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                            } catch (Exception e) {
                                logger.error("Exception in proxy get thread", e);
                                e.printStackTrace();
                                exceptions.add(e);
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Exception in proxy get thread", e);
                        exceptions.add(e);
                    } finally {
                        factory.close();
                        latch.countDown();
                    }
                }
            });
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        Thread.sleep(500);
                        rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2, 3));
                        Thread.sleep(500);
                        rebalancingComplete.set(true);
                        checkConsistentMetadata(finalCluster, serverList);
                    } catch (Exception e) {
                        exceptions.add(e);
                    } finally {
                        // stop servers
                        try {
                            stopServer(serverList);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                        latch.countDown();
                    }
                }
            });
            latch.await();
            executors.shutdown();
            executors.awaitTermination(300, TimeUnit.SECONDS);
            // check No Exception
            if (exceptions.size() > 0) {
                for (Exception e : exceptions) {
                    e.printStackTrace();
                }
                fail("Should not see any exceptions.");
            }
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
        throw ae;
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) SocketStoreClientFactory(voldemort.client.SocketStoreClientFactory) List(java.util.List) ArrayList(java.util.ArrayList) ClientConfig(voldemort.client.ClientConfig) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) DefaultStoreClient(voldemort.client.DefaultStoreClient) Test(org.junit.Test)

Example 2 with DefaultStoreClient

use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method testProxyPutDuringRebalancing.

@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
    logger.info("Starting testProxyPutDuringRebalancing");
    try {
        int[] zoneIds = new int[] { 1, 3 };
        int[][] nodesPerZone = new int[][] { { 3, 4, 5 }, { 9, 10, 11 } };
        int[][] partitionMap = new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } };
        Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 5, Lists.newArrayList(7));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 11, Lists.newArrayList(6));
        /**
         * Original partition map
         *
         * [s3 : p0] [s4 : p1, p6] [s5 : p2]
         *
         * [s9 : p3] [s10 : p4, p7] [s11 : p5]
         *
         * final server partition ownership
         *
         * [s3 : p0] [s4 : p1] [s5 : p2, p7]
         *
         * [s9 : p3] [s10 : p4] [s11 : p5, p6]
         *
         * Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
         *
         * Original server n-ary partition ownership
         *
         * [s3 : p0, p3-7] [s4 : p0-p7] [s5 : p1-2]
         *
         * [s9 : p0-3, p6-7] [s10 : p0-p7] [s11 : p4-5]
         *
         * final server n-ary partition ownership
         *
         * [s3 : p0, p2-7] [s4 : p0-1] [s5 : p1-p7]
         *
         * [s9 : p0-3, p5-7] [s10 : p0-4, p7] [s11 : p4-6]
         */
        List<Integer> serverList = Arrays.asList(3, 4, 5, 9, 10, 11);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        // Its is imperative that we test in a single shot since multiple batches would mean the proxy bridges
        // being torn down and established multiple times and we cannot test against the source
        // cluster topology then. getRebalanceKit uses batch size of infinite, so this should be fine.
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        populateData(currentCluster, rwStoreDefWithReplication);
        final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
        // the plan would cause these partitions to move:
        // Partition : Donor -> stealer
        // 
        // p2 (Z-SEC) : s4 -> s3
        // p3-6 (Z-PRI) : s4 -> s5
        // p7 (Z-PRI) : s3 -> s5
        // 
        // p5 (Z-SEC): s10 -> s9
        // p6 (Z-PRI): s10 -> s11
        // 
        // Rebalancing will run on servers 3, 5, 9, & 11
        final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 4, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
        assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
        final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
        final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
        final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
        final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
        for (String key : baselineTuples.keySet()) {
            baselineVersions.put(key, new VectorClock());
        }
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                SocketStoreClientFactory factory = null;
                try {
                    // wait for the rebalancing to begin
                    List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(3), serverMap.get(5), serverMap.get(9), serverMap.get(11));
                    while (!rebalancingComplete.get()) {
                        Iterator<VoldemortServer> serverIterator = serverList.iterator();
                        while (serverIterator.hasNext()) {
                            VoldemortServer server = serverIterator.next();
                            if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
                                logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
                                serverIterator.remove();
                            }
                        }
                        if (serverList.size() == 0) {
                            rebalancingStarted.set(true);
                            break;
                        }
                    }
                    if (rebalancingStarted.get()) {
                        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(3));
                        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
                        // Initially, all data now with zero vector clock
                        for (ByteArray movingKey : movingKeysList) {
                            try {
                                String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
                                String valStr = "proxy_write";
                                storeClientRW.put(keyStr, valStr);
                                baselineTuples.put(keyStr, valStr);
                                baselineVersions.get(keyStr).incrementVersion(11, System.currentTimeMillis());
                                proxyWritesDone.set(true);
                                if (rebalancingComplete.get()) {
                                    break;
                                }
                            } catch (InvalidMetadataException e) {
                                logger.error("Encountered an invalid metadata exception.. ", e);
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy write thread..", e);
                    exceptions.add(e);
                } finally {
                    if (factory != null)
                        factory.close();
                    latch.countDown();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    rebalanceKit.rebalance();
                } catch (Exception e) {
                    logger.error("Error in rebalancing... ", e);
                    exceptions.add(e);
                } finally {
                    rebalancingComplete.set(true);
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
        assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
        checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(3, 4, 5, 9, 10, 11), baselineTuples, baselineVersions);
        checkConsistentMetadata(finalCluster, serverList);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
        // check that the proxy writes were made to the original donor, node 4
        List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
        for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
        VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
        for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
        adminClient.setAdminClientCluster(currentCluster);
        checkForTupleEquivalence(adminClient, 4, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
        // stop servers
        try {
            stopServer(serverList);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
        throw ae;
    }
}
Also used : DefaultStoreClient(voldemort.client.DefaultStoreClient) StoreClient(voldemort.client.StoreClient) HashMap(java.util.HashMap) InvalidMetadataException(voldemort.store.InvalidMetadataException) ArrayList(java.util.ArrayList) VoldemortServer(voldemort.server.VoldemortServer) SocketStoreClientFactory(voldemort.client.SocketStoreClientFactory) Iterator(java.util.Iterator) ByteArray(voldemort.utils.ByteArray) List(java.util.List) ArrayList(java.util.ArrayList) ClientConfig(voldemort.client.ClientConfig) VectorClock(voldemort.versioning.VectorClock) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) ClockEntry(voldemort.versioning.ClockEntry) AdminClient(voldemort.client.protocol.admin.AdminClient) Test(org.junit.Test)

Example 3 with DefaultStoreClient

use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method testProxyGetDuringRebalancing.

@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
    logger.info("Starting testProxyGetDuringRebalancing");
    try {
        int[] zoneIds = new int[] { 1, 3 };
        int[][] nodesPerZone = new int[][] { { 3, 4 }, { 9, 10 } };
        int[][] partitionMap = new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } };
        Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
        Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 10, Lists.newArrayList(2));
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 4, Lists.newArrayList(3));
        // start servers
        final List<Integer> serverList = Arrays.asList(3, 4, 9, 10);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithReplication);
            final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
            final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
            final CountDownLatch latch = new CountDownLatch(2);
            // start get operation.
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        List<String> keys = new ArrayList<String>(testEntries.keySet());
                        while (!rebalancingComplete.get()) {
                            // should always able to get values.
                            int index = (int) (Math.random() * keys.size());
                            // should get a valid value
                            try {
                                Versioned<String> value = storeClientRW.get(keys.get(index));
                                assertNotSame("StoreClient get() should not return null.", null, value);
                                assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                            } catch (Exception e) {
                                logger.error("Exception in proxy get thread", e);
                                e.printStackTrace();
                                exceptions.add(e);
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Exception in proxy get thread", e);
                        exceptions.add(e);
                    } finally {
                        factory.close();
                        latch.countDown();
                    }
                }
            });
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        Thread.sleep(500);
                        rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(3, 4, 9, 10));
                        Thread.sleep(500);
                        rebalancingComplete.set(true);
                        checkConsistentMetadata(finalCluster, serverList);
                    } catch (Exception e) {
                        exceptions.add(e);
                    } finally {
                        // stop servers
                        try {
                            stopServer(serverList);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                        latch.countDown();
                    }
                }
            });
            latch.await();
            executors.shutdown();
            executors.awaitTermination(300, TimeUnit.SECONDS);
            // check No Exception
            if (exceptions.size() > 0) {
                for (Exception e : exceptions) {
                    e.printStackTrace();
                }
                fail("Should not see any exceptions.");
            }
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
        throw ae;
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) SocketStoreClientFactory(voldemort.client.SocketStoreClientFactory) List(java.util.List) ArrayList(java.util.ArrayList) ClientConfig(voldemort.client.ClientConfig) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) DefaultStoreClient(voldemort.client.DefaultStoreClient) Test(org.junit.Test)

Example 4 with DefaultStoreClient

use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testProxyGetDuringRebalancing.

@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
    logger.info("Starting testProxyGetDuringRebalancing");
    try {
        final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        // start servers 0 , 1 only
        final List<Integer> serverList = Arrays.asList(0, 1);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        // Populate the two stores
        populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
        populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
        final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
        final StoreClient<String, String> storeClientRO = new DefaultStoreClient<String, String>(testStoreNameRO, null, factory, 3);
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    List<String> keys = new ArrayList<String>(testEntries.keySet());
                    while (!rebalancingComplete.get()) {
                        // should always able to get values.
                        int index = (int) (Math.random() * keys.size());
                        // should get a valid value
                        try {
                            Versioned<String> value = storeClientRW.get(keys.get(index));
                            assertNotSame("StoreClient get() should not return null.", null, value);
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                            value = storeClientRO.get(keys.get(index));
                            assertNotSame("StoreClient get() should not return null.", null, value);
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                        } catch (Exception e) {
                            logger.error("Exception in online thread", e);
                            exceptions.add(e);
                        } finally {
                            latch.countDown();
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy get thread", e);
                    exceptions.add(e);
                } finally {
                    factory.close();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
                    Thread.sleep(500);
                    rebalancingComplete.set(true);
                    checkConsistentMetadata(finalCluster, serverList);
                } catch (Exception e) {
                    exceptions.add(e);
                    logger.error("Exception in rebalancing thread", e);
                } finally {
                    // stop servers
                    try {
                        stopServer(serverList);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
        throw ae;
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) SocketStoreClientFactory(voldemort.client.SocketStoreClientFactory) List(java.util.List) ArrayList(java.util.ArrayList) ClientConfig(voldemort.client.ClientConfig) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) UnreachableStoreException(voldemort.store.UnreachableStoreException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) DefaultStoreClient(voldemort.client.DefaultStoreClient) Test(org.junit.Test)

Example 5 with DefaultStoreClient

use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testProxyPutDuringRebalancing.

@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
    logger.info("Starting testProxyPutDuringRebalancing");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(6, 2, new int[] { 0, 0, 0, 1, 1, 1 }, new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Lists.newArrayList(7));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 5, Lists.newArrayList(6));
        /**
         * Original partition map
         *
         * [s0 : p0] [s1 : p1, p6] [s2 : p2]
         *
         * [s3 : p3] [s4 : p4, p7] [s5 : p5]
         *
         * final server partition ownership
         *
         * [s0 : p0] [s1 : p1] [s2 : p2, p7]
         *
         * [s3 : p3] [s4 : p4] [s5 : p5, p6]
         *
         * Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
         *
         * Original server n-ary partition ownership
         *
         * [s0 : p0, p3-7] [s1 : p0-p7] [s2 : p1-2]
         *
         * [s3 : p0-3, p6-7] [s4 : p0-p7] [s5 : p4-5]
         *
         * final server n-ary partition ownership
         *
         * [s0 : p0, p2-7] [s1 : p0-1] [s2 : p1-p7]
         *
         * [s3 : p0-3, p5-7] [s4 : p0-4, p7] [s5 : p4-6]
         */
        List<Integer> serverList = Arrays.asList(0, 1, 2, 3, 4, 5);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        // Its is imperative that we test in a single shot since multiple
        // batches would mean the proxy bridges being torn down and
        // established multiple times and we cannot test against the source
        // cluster topology then. getRebalanceKit uses batch size of
        // infinite, so this should be fine.
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        populateData(currentCluster, rwStoreDefWithReplication);
        final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
        // the plan would cause these partitions to move:
        // Partition : Donor -> stealer
        // 
        // p2 (Z-SEC) : s1 -> s0
        // p3-6 (Z-PRI) : s1 -> s2
        // p7 (Z-PRI) : s0 -> s2
        // 
        // p5 (Z-SEC): s4 -> s3
        // p6 (Z-PRI): s4 -> s5
        // 
        // :. rebalancing will run on servers 0, 2, 3, & 5
        final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 1, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
        assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
        final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
        final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
        final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
        final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
        for (String key : baselineTuples.keySet()) {
            baselineVersions.put(key, new VectorClock());
        }
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                SocketStoreClientFactory factory = null;
                try {
                    // wait for the rebalancing to begin
                    List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(0), serverMap.get(2), serverMap.get(3), serverMap.get(5));
                    while (!rebalancingComplete.get()) {
                        Iterator<VoldemortServer> serverIterator = serverList.iterator();
                        while (serverIterator.hasNext()) {
                            VoldemortServer server = serverIterator.next();
                            if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
                                logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
                                serverIterator.remove();
                            }
                        }
                        if (serverList.size() == 0) {
                            rebalancingStarted.set(true);
                            break;
                        }
                    }
                    if (rebalancingStarted.get()) {
                        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(1));
                        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
                        // now with zero vector clock
                        for (ByteArray movingKey : movingKeysList) {
                            try {
                                String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
                                String valStr = "proxy_write";
                                storeClientRW.put(keyStr, valStr);
                                baselineTuples.put(keyStr, valStr);
                                // all these keys will have [5:1] vector
                                // clock is node 5 is the new pseudo master
                                baselineVersions.get(keyStr).incrementVersion(5, System.currentTimeMillis());
                                proxyWritesDone.set(true);
                                if (rebalancingComplete.get()) {
                                    break;
                                }
                            } catch (InvalidMetadataException e) {
                                // let this go
                                logger.error("Encountered an invalid metadata exception.. ", e);
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy write thread..", e);
                    exceptions.add(e);
                } finally {
                    if (factory != null)
                        factory.close();
                    latch.countDown();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    rebalanceKit.rebalance();
                } catch (Exception e) {
                    logger.error("Error in rebalancing... ", e);
                    exceptions.add(e);
                } finally {
                    rebalancingComplete.set(true);
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
        assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
        checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(0, 1, 2, 3, 4, 5), baselineTuples, baselineVersions);
        checkConsistentMetadata(finalCluster, serverList);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
        // check that the proxy writes were made to the original donor, node
        // 1
        List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
        for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
        VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
        for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
        adminClient.setAdminClientCluster(currentCluster);
        checkForTupleEquivalence(adminClient, 1, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
        // stop servers
        try {
            stopServer(serverList);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
        throw ae;
    }
}
Also used : DefaultStoreClient(voldemort.client.DefaultStoreClient) StoreClient(voldemort.client.StoreClient) HashMap(java.util.HashMap) InvalidMetadataException(voldemort.store.InvalidMetadataException) ArrayList(java.util.ArrayList) VoldemortServer(voldemort.server.VoldemortServer) SocketStoreClientFactory(voldemort.client.SocketStoreClientFactory) Iterator(java.util.Iterator) ByteArray(voldemort.utils.ByteArray) List(java.util.List) ArrayList(java.util.ArrayList) ClientConfig(voldemort.client.ClientConfig) VectorClock(voldemort.versioning.VectorClock) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) ClockEntry(voldemort.versioning.ClockEntry) AdminClient(voldemort.client.protocol.admin.AdminClient) Test(org.junit.Test)

Aggregations

IOException (java.io.IOException)6 ArrayList (java.util.ArrayList)6 HashMap (java.util.HashMap)6 List (java.util.List)6 CountDownLatch (java.util.concurrent.CountDownLatch)6 ExecutorService (java.util.concurrent.ExecutorService)6 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 Test (org.junit.Test)6 ClusterTestUtils (voldemort.ClusterTestUtils)6 ClientConfig (voldemort.client.ClientConfig)6 DefaultStoreClient (voldemort.client.DefaultStoreClient)6 SocketStoreClientFactory (voldemort.client.SocketStoreClientFactory)6 Cluster (voldemort.cluster.Cluster)6 InvalidMetadataException (voldemort.store.InvalidMetadataException)6 ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)6 Iterator (java.util.Iterator)3 StoreClient (voldemort.client.StoreClient)3 AdminClient (voldemort.client.protocol.admin.AdminClient)3 VoldemortServer (voldemort.server.VoldemortServer)3 ByteArray (voldemort.utils.ByteArray)3