Examples with Pair voldemort.utils.Pair used on opensource projects

Example 11 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class ROTestUtils method getNodeIdToAllPartitions.

/**
     * For a particular cluster creates a mapping of node id to their
     * corresponding list of [ replicaType, partition ] tuple
     * 
     * @param cluster The cluster metadata
     * @param storeDef The store definition
     * @param includePrimary Include the primary partition?
     * @return Map of node id to set of [ replicaType, partition ] tuple
     */
public static Map<Integer, Set<Pair<Integer, Integer>>> getNodeIdToAllPartitions(final Cluster cluster, final StoreDefinition storeDef, boolean includePrimary) {
    final RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    final Map<Integer, Set<Pair<Integer, Integer>>> nodeIdToReplicas = new HashMap<Integer, Set<Pair<Integer, Integer>>>();
    final Map<Integer, Integer> partitionToNodeIdMap = cluster.getPartitionIdToNodeIdMap();
    // Map initialization.
    for (Node node : cluster.getNodes()) {
        nodeIdToReplicas.put(node.getId(), new HashSet<Pair<Integer, Integer>>());
    }
    // Track how many zones actually have partitions (and so replica types)
    // in them.
    int zonesWithPartitions = 0;
    for (Integer zoneId : cluster.getZoneIds()) {
        if (cluster.getNumberOfPartitionsInZone(zoneId) > 0) {
            zonesWithPartitions++;
        }
    }
    // Loops through all nodes
    for (Node node : cluster.getNodes()) {
        // Gets the partitions that this node was configured with.
        for (Integer primary : node.getPartitionIds()) {
            // Gets the list of replicating partitions.
            List<Integer> replicaPartitionList = routingStrategy.getReplicatingPartitionList(primary);
            if ((replicaPartitionList.size() % zonesWithPartitions != 0) || ((replicaPartitionList.size() / zonesWithPartitions) != (storeDef.getReplicationFactor() / cluster.getNumberOfZones()))) {
                // For zone expansion & shrinking, this warning is expected
                // in some cases. For other use cases (shuffling, cluster
                // expansion), this warning indicates that something
                // is wrong between the clusters and store defs.
                logger.warn("Number of replicas returned (" + replicaPartitionList.size() + ") does not make sense given the replication factor (" + storeDef.getReplicationFactor() + ") and that there are " + cluster.getNumberOfZones() + " zones of which " + zonesWithPartitions + " have partitions (and of which " + (cluster.getNumberOfZones() - zonesWithPartitions) + " are empty).");
            }
            int replicaType = 0;
            if (!includePrimary) {
                replicaPartitionList.remove(primary);
                replicaType = 1;
            }
            // Get the node that this replicating partition belongs to.
            for (Integer replicaPartition : replicaPartitionList) {
                Integer replicaNodeId = partitionToNodeIdMap.get(replicaPartition);
                // The replicating node will have a copy of primary.
                nodeIdToReplicas.get(replicaNodeId).add(Pair.create(replicaType, primary));
                replicaType++;
            }
        }
    }
    return nodeIdToReplicas;
}

Example 12 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AdminCommandStream method readEntriesBinary.

private static Iterator<Pair<ByteArray, Versioned<byte[]>>> readEntriesBinary(File inputDir, String storeName) throws IOException {
    File inputFile = new File(inputDir, storeName + ".entries");
    if (!inputFile.exists()) {
        throw new FileNotFoundException("File " + inputFile.getAbsolutePath() + " does not exist!");
    }
    final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(inputFile)));
    return new AbstractIterator<Pair<ByteArray, Versioned<byte[]>>>() {

        @Override
        protected Pair<ByteArray, Versioned<byte[]>> computeNext() {
            try {
                int length = dis.readInt();
                byte[] keyBytes = new byte[length];
                ByteUtils.read(dis, keyBytes);
                length = dis.readInt();
                byte[] versionBytes = new byte[length];
                ByteUtils.read(dis, versionBytes);
                length = dis.readInt();
                byte[] valueBytes = new byte[length];
                ByteUtils.read(dis, valueBytes);
                ByteArray key = new ByteArray(keyBytes);
                VectorClock version = new VectorClock(versionBytes);
                Versioned<byte[]> value = new Versioned<byte[]>(valueBytes, version);
                return new Pair<ByteArray, Versioned<byte[]>>(key, value);
            } catch (EOFException e) {
                try {
                    dis.close();
                } catch (IOException ie) {
                    ie.printStackTrace();
                }
                return endOfData();
            } catch (IOException e) {
                try {
                    dis.close();
                } catch (IOException ie) {
                    ie.printStackTrace();
                }
                throw new VoldemortException("Error reading from input file ", e);
            }
        }
    };
}

Example 13 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class ReadOnlyStorageEngineTest method testIteration.

@SuppressWarnings("unchecked")
@Test
public void testIteration() throws Exception {
    ReadOnlyStorageEngineTestInstance testData = ReadOnlyStorageEngineTestInstance.create(strategy, dir, TOTAL_NUMBER_OF_RECORDS, numberOfNodes, replicationFactor, serDef, serDef, storageType, partitionMap);
    ListMultimap<Integer, Pair<String, String>> nodeToEntries = ArrayListMultimap.create();
    for (Map.Entry<String, String> entry : testData.getData().entrySet()) {
        for (Node node : testData.routeRequest(entry.getKey())) {
            nodeToEntries.put(node.getId(), Pair.create(entry.getKey(), entry.getValue()));
        }
    }
    SerializerFactory factory = new DefaultSerializerFactory();
    Serializer<String> serializer = (Serializer<String>) factory.getSerializer(serDef);
    for (Map.Entry<Integer, ReadOnlyStorageEngine> storeEntry : testData.getReadOnlyStores().entrySet()) {
        List<Pair<String, String>> entries = Lists.newArrayList(nodeToEntries.get(storeEntry.getKey()));
        ClosableIterator<ByteArray> keyIterator = null;
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = null;
        try {
            keyIterator = storeEntry.getValue().keys();
            entryIterator = storeEntry.getValue().entries();
        } catch (Exception e) {
            if (storageType.compareTo(ReadOnlyStorageFormat.READONLY_V2) == 0) {
                fail("Should not have thrown exception since this version supports iteration");
            } else {
                return;
            }
        }
        // Generate keys from entries
        List<String> keys = Lists.newArrayList();
        Iterator<Pair<String, String>> pairIterator = entries.iterator();
        while (pairIterator.hasNext()) {
            keys.add(pairIterator.next().getFirst());
        }
        // Test keys
        int keyCount = 0;
        while (keyIterator.hasNext()) {
            String key = serializer.toObject(keyIterator.next().get());
            Assert.assertEquals(keys.contains(key), true);
            keyCount++;
        }
        Assert.assertEquals(keyCount, entries.size());
        // Test entries
        int entriesCount = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Pair<String, String> stringEntry = Pair.create(serializer.toObject(entry.getFirst().get()), serializer.toObject(entry.getSecond().getValue()));
            Assert.assertEquals(entries.contains(stringEntry), true);
            entriesCount++;
        }
        Assert.assertEquals(entriesCount, entries.size());
    }
}

Example 14 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AbstractStorageEngineTest method testTruncate.

@Test
public void testTruncate() throws Exception {
    StorageEngine<ByteArray, byte[], byte[]> engine = getStorageEngine();
    Versioned<byte[]> v1 = new Versioned<byte[]>(new byte[] { 1 });
    Versioned<byte[]> v2 = new Versioned<byte[]>(new byte[] { 2 });
    Versioned<byte[]> v3 = new Versioned<byte[]>(new byte[] { 3 });
    ByteArray key1 = new ByteArray((byte) 3);
    ByteArray key2 = new ByteArray((byte) 4);
    ByteArray key3 = new ByteArray((byte) 5);
    engine.put(key1, v1, null);
    engine.put(key2, v2, null);
    engine.put(key3, v3, null);
    engine.truncate();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> it = null;
    try {
        it = engine.entries();
        while (it.hasNext()) {
            fail("There shouldn't be any entries in this store.");
        }
    } finally {
        if (it != null) {
            it.close();
        }
    }
    // Verify that the store is still useable after the truncate.
    assertEquals(0, engine.get(key1, null).size());
    engine.put(key1, v1, null);
    assertEquals(1, engine.get(key1, null).size());
}

Example 15 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AbstractStorageEngineTest method testIterationWithSerialization.

@Test
public void testIterationWithSerialization() {
    StorageEngine<ByteArray, byte[], byte[]> store = getStorageEngine();
    StorageEngine<String, String, String> stringStore = SerializingStorageEngine.wrap(store, new StringSerializer(), new StringSerializer(), new StringSerializer());
    Map<String, String> vals = ImmutableMap.of("a", "a", "b", "b", "c", "c", "d", "d", "e", "e");
    for (Map.Entry<String, String> entry : vals.entrySet()) stringStore.put(entry.getKey(), new Versioned<String>(entry.getValue()), null);
    ClosableIterator<Pair<String, Versioned<String>>> iter = stringStore.entries();
    int count = 0;
    while (iter.hasNext()) {
        Pair<String, Versioned<String>> keyAndVal = iter.next();
        assertTrue(vals.containsKey(keyAndVal.getFirst()));
        assertEquals(vals.get(keyAndVal.getFirst()), keyAndVal.getSecond().getValue());
        count++;
    }
    assertEquals(count, vals.size());
    iter.close();
}