Examples with Pair voldemort.utils.Pair used on opensource projects

Example 1 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class HadoopStoreBuilderCollisionTest method testCollisionWithParams.

@SuppressWarnings({ "unchecked" })
public void testCollisionWithParams(int totalElements, int maxCollisions) throws Exception {
    assertEquals(totalElements % maxCollisions, 0);
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    List<String> valuesLeft = Lists.newArrayList();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < totalElements; i++) {
        values.put(Integer.toString(i), Integer.toString(i));
        valuesLeft.add(Integer.toString(i));
    }
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    byte[] currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
    int entryId = 0;
    for (Map.Entry<String, String> entry : values.entrySet()) {
        if (entryId % maxCollisions == 0) {
            currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
        }
        contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
        byte[] oldMd5 = ByteUtils.copy(ByteUtils.md5(serializer.toBytes(entry.getKey())), 0, 2 * ByteUtils.SIZE_OF_INT);
        oldMd5ToNewMd5.put(new ByteArray(oldMd5), currentMd5);
        entryId++;
    }
    FileUtils.writeStringToFile(inputFile, contents.toString());
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testCollisionWithParams", new Props(), new JobConf(), CollidingTextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, true, false, 1024 * 1024 * 1024, false, null, false);
    builder.build();
    File nodeFile = new File(outputDir, "node-0");
    File versionDir = new File(storeDir, "version-0");
    HdfsFetcher fetcher = new HdfsFetcher();
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    // Test if we work in the normal collision scenario open store
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, new CustomBinarySearchStrategy(), new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        List<Versioned<Object>> found = store.get(entry.getKey(), null);
        Assert.assertEquals("Incorrect number of results", 1, found.size());
        Assert.assertEquals(entry.getValue(), found.get(0).getValue());
    }
    // also check the iterator - first key iterator...
    List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
    ClosableIterator<ByteArray> keyIterator = engine.keys();
    int numElements = 0;
    while (keyIterator.hasNext()) {
        Object object = serializer.toObject(keyIterator.next().get());
        assertEquals(valuesLeft.remove(object), true);
        Assert.assertTrue(values.containsKey(object));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft.size(), 0);
    // ... and entry iterator
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
    numElements = 0;
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
        Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft2.size(), 0);
}

Example 2 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class KratiStorageEngine method entries.

@Override
public ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries() {
    List<Pair<ByteArray, Versioned<byte[]>>> returnedList = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    DataArray array = datastore.getDataArray();
    for (int index = 0; index < array.length(); index++) {
        byte[] returnedBytes = array.get(index);
        if (returnedBytes != null) {
            // Extract the key value pair from this
            // TODO: Move to DynamicDataStore code
            ByteBuffer bb = ByteBuffer.wrap(returnedBytes);
            int cnt = bb.getInt();
            // Loop over all keys at this index
            for (int i = 0; i < cnt; i++) {
                int keyLen = bb.getInt();
                byte[] key = new byte[keyLen];
                bb.get(key);
                int valueLen = bb.getInt();
                byte[] value = new byte[valueLen];
                bb.get(value);
                List<Versioned<byte[]>> versions;
                try {
                    versions = disassembleValues(value);
                } catch (IOException e) {
                    versions = null;
                }
                if (versions != null) {
                    Iterator<Versioned<byte[]>> iterVersions = versions.iterator();
                    while (iterVersions.hasNext()) {
                        Versioned<byte[]> currentVersion = iterVersions.next();
                        returnedList.add(Pair.create(new ByteArray(key), currentVersion));
                    }
                }
            }
        }
    }
    return new KratiClosableIterator(returnedList);
}

Example 3 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testDeleteStreamWithFilter.

@Test
public void testDeleteStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
    VoldemortFilter filter = new VoldemortFilterImpl();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0))
            store.put(pair.getFirst(), pair.getSecond(), null);
    }
    // make delete stream call with filter
    getAdminClient().storeMntOps.deletePartitions(0, testStoreName, Lists.newArrayList(0, 1), filter);
    // assert none of the filtered entries are returned.
    for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(entry.getFirst().get())).contains(0)) {
            if (filter.accept(entry.getFirst(), entry.getSecond())) {
                assertEquals("All entries should be deleted except the filtered ones.", 0, store.get(entry.getFirst(), null).size());
            } else {
                assertNotSame("filtered entry should be still present.", 0, store.get(entry.getFirst(), null).size());
                assertEquals("values should match", new String(entry.getSecond().getValue()), new String(store.get(entry.getFirst(), null).get(0).getValue()));
            }
        }
    }
}

Example 4 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testUpdateAsStreamWithFilter.

@Test
public void testUpdateAsStreamWithFilter() {
    VoldemortFilter filter = new VoldemortFilterImpl();
    Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
    // make update stream call with filter
    getAdminClient().streamingOps.updateEntries(0, testStoreName, entrySet.iterator(), filter);
    // assert none of the filtered entries are updated.
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
        if (filter.accept(entry.getFirst(), entry.getSecond())) {
            assertEquals("Store should have this key/value pair", 1, store.get(entry.getFirst(), null).size());
            assertEquals("Store should have this key/value pair", entry.getSecond(), store.get(entry.getFirst(), null).get(0));
        } else {
            assertEquals("Store should Not have this key/value pair", 0, store.get(entry.getFirst(), null).size());
        }
    }
}

Example 5 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testFetchAsStreamWithFilter.

@Test
public void testFetchAsStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    VoldemortFilter filter = new VoldemortFilterImpl();
    int shouldFilterCount = 0;
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : createEntries()) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0)) {
            store.put(pair.getFirst(), pair.getSecond(), null);
            if (!filter.accept(pair.getFirst(), pair.getSecond())) {
                shouldFilterCount++;
            }
        }
    }
    // make fetch stream call with filter
    Iterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = getAdminClient().bulkFetchOps.fetchEntries(0, testStoreName, getCluster().getNodeById(0).getPartitionIds(), filter, false);
    // assert none of the filtered entries are returned.
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        if (!filter.accept(entry.getFirst(), entry.getSecond())) {
            fail();
        }
    }
}