Examples with InvalidMetadataException voldemort.store.InvalidMetadataException used on opensource projects

Example 1 with InvalidMetadataException

use of voldemort.store.InvalidMetadataException in project voldemort by voldemort.

the class AdminServiceBasicTest method testQuery.

@Test
public void testQuery() {
    HashMap<ByteArray, byte[]> belongToAndInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToAndInsideServer1 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongServer0ButInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToServer0ButOutsideBoth = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongToServer0AndOutsideBoth = new HashMap<ByteArray, byte[]>();
    Store<ByteArray, byte[], byte[]> store0 = getStore(0, testStoreName);
    Store<ByteArray, byte[], byte[]> store1 = getStore(1, testStoreName);
    HashMap<ByteArray, byte[]> entrySet = null;
    Iterator<ByteArray> keys = null;
    RoutingStrategy strategy = servers[0].getMetadataStore().getRoutingStrategy(testStoreName);
    while (true) {
        ByteArray key;
        byte[] value;
        if (keys == null || !keys.hasNext()) {
            entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
            keys = entrySet.keySet().iterator();
        }
        key = keys.next();
        value = entrySet.get(key);
        List<Node> routedNodes = strategy.routeRequest(key.get());
        boolean keyShouldBeInNode0 = false;
        boolean keyShouldBeInNode1 = false;
        for (Node node : routedNodes) {
            keyShouldBeInNode0 = keyShouldBeInNode0 || (node.getId() == 0);
            keyShouldBeInNode1 = keyShouldBeInNode1 || (node.getId() == 1);
        }
        if (belongToAndInsideServer0.size() < 10) {
            if (keyShouldBeInNode0) {
                belongToAndInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToAndInsideServer1.size() < 10) {
            if (keyShouldBeInNode1) {
                belongToAndInsideServer1.put(key, value);
                store1.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (notBelongServer0ButInsideServer0.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongServer0ButInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToServer0ButOutsideBoth.size() < 5) {
            if (keyShouldBeInNode0) {
                belongToServer0ButOutsideBoth.put(key, value);
            }
        } else if (notBelongToServer0AndOutsideBoth.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongToServer0AndOutsideBoth.put(key, value);
            }
        } else {
            break;
        }
    }
    ArrayList<ByteArray> belongToAndInsideServer0Keys = new ArrayList<ByteArray>(belongToAndInsideServer0.keySet());
    ArrayList<ByteArray> belongToAndInsideServer1Keys = new ArrayList<ByteArray>(belongToAndInsideServer1.keySet());
    ArrayList<ByteArray> notBelongServer0ButInsideServer0Keys = new ArrayList<ByteArray>(notBelongServer0ButInsideServer0.keySet());
    ArrayList<ByteArray> belongToServer0ButOutsideBothKeys = new ArrayList<ByteArray>(belongToServer0ButOutsideBoth.keySet());
    ArrayList<ByteArray> notBelongToServer0AndOutsideBothKeys = new ArrayList<ByteArray>(notBelongToServer0AndOutsideBoth.keySet());
    List<ByteArray> queryKeys;
    Iterator<QueryKeyResult> results;
    QueryKeyResult entry;
    // test one key on store 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertEquals(queryKeys.get(0), entry.getKey());
    assertNull("There should not be exception in response", entry.getException());
    assertEquals("There should be only 1 value in versioned list", 1, entry.getValues().size());
    assertEquals("Two byte[] should be equal", 0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertFalse("There should be only one result", results.hasNext());
    // test one key belongs to but not exists in server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertNotNull("Response should be non-null", entry.getValues());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    assertNull("There should not be exception", entry.getException());
    // test one key not exist and does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key that exists on server 0 but does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key deleted
    store0.delete(belongToAndInsideServer0Keys.get(4), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(4));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    // test empty request
    queryKeys = new ArrayList<ByteArray>();
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertFalse("Results should be empty", results.hasNext());
    // test null key
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(null);
    assertEquals(1, queryKeys.size());
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be IllegalArgumentException exception", entry.getException() instanceof IllegalArgumentException);
    // test multiple keys (3) on store 1
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer1Keys.get(0));
    queryKeys.add(belongToAndInsideServer1Keys.get(1));
    queryKeys.add(belongToAndInsideServer1Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(1, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    Map<ByteArray, List<Versioned<byte[]>>> entries = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    int resultCount = 0;
    while (results.hasNext()) {
        resultCount++;
        entry = results.next();
        assertNull("There should not be exception in response", entry.getException());
        assertNotNull("Value should not be null for Key: ", entry.getValues());
        entries.put(entry.getKey(), entry.getValues());
    }
    assertEquals("There should 3 and only 3 results", 3, resultCount);
    for (ByteArray key : queryKeys) {
        // this loop and the count ensure one-to-one mapping
        assertNotNull("This key should exist in the results: " + key, entries.get(key));
        assertEquals("Two byte[] should be equal for key: " + key, 0, ByteUtils.compare(belongToAndInsideServer1.get(key), entries.get(key).get(0).getValue()));
    }
    // test multiple keys, mixed situation
    // key 0: Exists and belongs to
    // key 1: Exists but does not belong to
    // key 2: Does not exist but belongs to
    // key 3: Does not belong and not exist
    // key 4: Same situation with key0
    // key 5: Deleted
    // key 6: Same situation with key2
    store0.delete(belongToAndInsideServer0Keys.get(5), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(2));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(1));
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(1));
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(1));
    queryKeys.add(belongToAndInsideServer0Keys.get(3));
    queryKeys.add(belongToAndInsideServer0Keys.get(5));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    // key 0
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(0).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 1
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(1).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 2
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(2).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 3
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(3).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 4
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(4).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(4)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 5
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(5).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 6
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(6).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // no more keys
    assertFalse(results.hasNext());
}

Example 2 with InvalidMetadataException

use of voldemort.store.InvalidMetadataException in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method testProxyPutDuringRebalancing.

@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
    logger.info("Starting testProxyPutDuringRebalancing");
    try {
        int[] zoneIds = new int[] { 1, 3 };
        int[][] nodesPerZone = new int[][] { { 3, 4, 5 }, { 9, 10, 11 } };
        int[][] partitionMap = new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } };
        Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 5, Lists.newArrayList(7));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 11, Lists.newArrayList(6));
        /**
             * Original partition map
             *
             * [s3 : p0] [s4 : p1, p6] [s5 : p2]
             *
             * [s9 : p3] [s10 : p4, p7] [s11 : p5]
             *
             * final server partition ownership
             *
             * [s3 : p0] [s4 : p1] [s5 : p2, p7]
             *
             * [s9 : p3] [s10 : p4] [s11 : p5, p6]
             *
             * Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
             *
             * Original server n-ary partition ownership
             *
             * [s3 : p0, p3-7] [s4 : p0-p7] [s5 : p1-2]
             *
             * [s9 : p0-3, p6-7] [s10 : p0-p7] [s11 : p4-5]
             *
             * final server n-ary partition ownership
             *
             * [s3 : p0, p2-7] [s4 : p0-1] [s5 : p1-p7]
             *
             * [s9 : p0-3, p5-7] [s10 : p0-4, p7] [s11 : p4-6]
             */
        List<Integer> serverList = Arrays.asList(3, 4, 5, 9, 10, 11);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        // Its is imperative that we test in a single shot since multiple batches would mean the proxy bridges
        // being torn down and established multiple times and we cannot test against the source
        // cluster topology then. getRebalanceKit uses batch size of infinite, so this should be fine.
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        populateData(currentCluster, rwStoreDefWithReplication);
        final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
        // the plan would cause these partitions to move:
        // Partition : Donor -> stealer
        //
        // p2 (Z-SEC) : s4 -> s3
        // p3-6 (Z-PRI) : s4 -> s5
        // p7 (Z-PRI) : s3 -> s5
        //
        // p5 (Z-SEC): s10 -> s9
        // p6 (Z-PRI): s10 -> s11
        //
        // Rebalancing will run on servers 3, 5, 9, & 11
        final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 4, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
        assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
        final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
        final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
        final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
        final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
        for (String key : baselineTuples.keySet()) {
            baselineVersions.put(key, new VectorClock());
        }
        final CountDownLatch latch = new CountDownLatch(2);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                SocketStoreClientFactory factory = null;
                try {
                    // wait for the rebalancing to begin
                    List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(3), serverMap.get(5), serverMap.get(9), serverMap.get(11));
                    while (!rebalancingComplete.get()) {
                        Iterator<VoldemortServer> serverIterator = serverList.iterator();
                        while (serverIterator.hasNext()) {
                            VoldemortServer server = serverIterator.next();
                            if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
                                logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
                                serverIterator.remove();
                            }
                        }
                        if (serverList.size() == 0) {
                            rebalancingStarted.set(true);
                            break;
                        }
                    }
                    if (rebalancingStarted.get()) {
                        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(3));
                        final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
                        // Initially, all data now with zero vector clock
                        for (ByteArray movingKey : movingKeysList) {
                            try {
                                String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
                                String valStr = "proxy_write";
                                storeClientRW.put(keyStr, valStr);
                                baselineTuples.put(keyStr, valStr);
                                baselineVersions.get(keyStr).incrementVersion(11, System.currentTimeMillis());
                                proxyWritesDone.set(true);
                                if (rebalancingComplete.get()) {
                                    break;
                                }
                            } catch (InvalidMetadataException e) {
                                logger.error("Encountered an invalid metadata exception.. ", e);
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Exception in proxy write thread..", e);
                    exceptions.add(e);
                } finally {
                    if (factory != null)
                        factory.close();
                    latch.countDown();
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    rebalanceKit.rebalance();
                } catch (Exception e) {
                    logger.error("Error in rebalancing... ", e);
                    exceptions.add(e);
                } finally {
                    rebalancingComplete.set(true);
                    latch.countDown();
                }
            }
        });
        latch.await();
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
        assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
        checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(3, 4, 5, 9, 10, 11), baselineTuples, baselineVersions);
        checkConsistentMetadata(finalCluster, serverList);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions.");
        }
        // check that the proxy writes were made to the original donor, node 4
        List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
        for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
        VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
        for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
        adminClient.setAdminClientCluster(currentCluster);
        checkForTupleEquivalence(adminClient, 4, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
        // stop servers
        try {
            stopServer(serverList);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
        throw ae;
    }
}

Example 3 with InvalidMetadataException

use of voldemort.store.InvalidMetadataException in project voldemort by voldemort.

the class InvalidMetadataCheckingStoreTest method testAddingPartition.

/**
     * NOTE: the total number of partitions should remain same for hash
     * consistency
     */
public void testAddingPartition() {
    StoreDefinition storeDef = ServerTestUtils.getStoreDefs(1).get(0);
    Cluster cluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10 } });
    MetadataStore metadata = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
    InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0, new DoNothingStore<ByteArray, byte[], byte[]>(storeDef.getName()), metadata);
    try {
        // add partitions to node 0 on client side.
        Cluster updatedCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3, 4, 5, 10 }, { 6, 7 }, { 8, 9 } });
        MetadataStore updatedMetadata = ServerTestUtils.createMetadataStore(updatedCluster, Arrays.asList(storeDef));
        doOperations(0, store, updatedMetadata, storeDef);
        fail("Should see InvalidMetadataExceptions");
    } catch (InvalidMetadataException e) {
    // ignore
    }
}

Example 4 with InvalidMetadataException

use of voldemort.store.InvalidMetadataException in project voldemort by voldemort.

the class InvalidMetadataCheckingStoreTest method testRemovingPartition.

public void testRemovingPartition() {
    StoreDefinition storeDef = ServerTestUtils.getStoreDefs(1).get(0);
    Cluster cluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10 } });
    MetadataStore metadata = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
    InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0, new DoNothingStore<ByteArray, byte[], byte[]>(storeDef.getName()), metadata);
    try {
        // remove partitions to node 0 on client side.
        Cluster updatedCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 4, 5, 6, 7 }, { 3, 8, 9, 10 } });
        MetadataStore updatedMetadata = ServerTestUtils.createMetadataStore(updatedCluster, Arrays.asList(storeDef));
        doOperations(0, store, updatedMetadata, storeDef);
    } catch (InvalidMetadataException e) {
        throw new RuntimeException("Should not see any InvalidMetaDataException", e);
    }
}

Example 5 with InvalidMetadataException

use of voldemort.store.InvalidMetadataException in project voldemort by voldemort.

the class DefaultStoreClient method getAll.

public Map<K, Versioned<V>> getAll(Iterable<K> keys, Map<K, Object> transforms) {
    Map<K, List<Versioned<V>>> items = null;
    for (int attempts = 0; ; attempts++) {
        if (attempts >= this.metadataRefreshAttempts)
            throw new VoldemortException(this.metadataRefreshAttempts + " metadata refresh attempts failed.");
        try {
            items = store.getAll(keys, transforms);
            break;
        } catch (InvalidMetadataException e) {
            logger.info("Received invalid metadata exception during getAll [  " + e.getMessage() + " ] on store '" + storeName + "'. Rebootstrapping");
            bootStrap();
        }
    }
    Map<K, Versioned<V>> result = Maps.newHashMapWithExpectedSize(items.size());
    for (Entry<K, List<Versioned<V>>> mapEntry : items.entrySet()) {
        Versioned<V> value = getItemOrThrow(mapEntry.getKey(), null, mapEntry.getValue());
        result.put(mapEntry.getKey(), value);
    }
    return result;
}