Examples with ByteArray voldemort.utils.ByteArray used on opensource projects

Example 86 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class ServerTestUtils method getStores.

public static StoreRepository getStores(String storeName, String clusterXml, String storesXml) {
    StoreRepository repository = new StoreRepository();
    Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName);
    repository.addLocalStore(store);
    repository.addRoutedStore(store);
    // create new metadata store.
    MetadataStore metadata = createMetadataStore(new ClusterMapper().readCluster(new StringReader(clusterXml)), new StoreDefinitionsMapper().readStoreList(new StringReader(storesXml)));
    repository.addLocalStore(metadata);
    return repository;
}

Example 87 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class ServerTestUtils method waitForSlopDrain.

public static void waitForSlopDrain(Map<Integer, VoldemortServer> vservers, Long slopDrainTimoutMs) throws InterruptedException {
    long timeStart = System.currentTimeMillis();
    boolean allSlopsEmpty = false;
    while (System.currentTimeMillis() < timeStart + slopDrainTimoutMs) {
        allSlopsEmpty = true;
        for (Integer nodeId : vservers.keySet()) {
            VoldemortServer vs = vservers.get(nodeId);
            SlopStorageEngine sse = vs.getStoreRepository().getSlopStore();
            ClosableIterator<ByteArray> keys = sse.keys();
            long count = 0;
            while (keys.hasNext()) {
                keys.next();
                count++;
            }
            keys.close();
            if (count > 0) {
                allSlopsEmpty = false;
                logger.info(String.format("Slop engine for node %d is not yet empty with %d slops\n", nodeId, count));
            }
        }
        if (allSlopsEmpty) {
            break;
        }
        Thread.sleep(1000);
    }
    if (!allSlopsEmpty) {
        throw new RuntimeException("Timeout while waiting for all slops to drain");
    }
}

Example 88 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class ReadOnlyStorageEngineTest method testIteration.

@SuppressWarnings("unchecked")
@Test
public void testIteration() throws Exception {
    ReadOnlyStorageEngineTestInstance testData = ReadOnlyStorageEngineTestInstance.create(strategy, dir, TOTAL_NUMBER_OF_RECORDS, numberOfNodes, replicationFactor, serDef, serDef, storageType, partitionMap);
    ListMultimap<Integer, Pair<String, String>> nodeToEntries = ArrayListMultimap.create();
    for (Map.Entry<String, String> entry : testData.getData().entrySet()) {
        for (Node node : testData.routeRequest(entry.getKey())) {
            nodeToEntries.put(node.getId(), Pair.create(entry.getKey(), entry.getValue()));
        }
    }
    SerializerFactory factory = new DefaultSerializerFactory();
    Serializer<String> serializer = (Serializer<String>) factory.getSerializer(serDef);
    for (Map.Entry<Integer, ReadOnlyStorageEngine> storeEntry : testData.getReadOnlyStores().entrySet()) {
        List<Pair<String, String>> entries = Lists.newArrayList(nodeToEntries.get(storeEntry.getKey()));
        ClosableIterator<ByteArray> keyIterator = null;
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = null;
        try {
            keyIterator = storeEntry.getValue().keys();
            entryIterator = storeEntry.getValue().entries();
        } catch (Exception e) {
            if (storageType.compareTo(ReadOnlyStorageFormat.READONLY_V2) == 0) {
                fail("Should not have thrown exception since this version supports iteration");
            } else {
                return;
            }
        }
        // Generate keys from entries
        List<String> keys = Lists.newArrayList();
        Iterator<Pair<String, String>> pairIterator = entries.iterator();
        while (pairIterator.hasNext()) {
            keys.add(pairIterator.next().getFirst());
        }
        // Test keys
        int keyCount = 0;
        while (keyIterator.hasNext()) {
            String key = serializer.toObject(keyIterator.next().get());
            Assert.assertEquals(keys.contains(key), true);
            keyCount++;
        }
        Assert.assertEquals(keyCount, entries.size());
        // Test entries
        int entriesCount = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Pair<String, String> stringEntry = Pair.create(serializer.toObject(entry.getFirst().get()), serializer.toObject(entry.getSecond().getValue()));
            Assert.assertEquals(entries.contains(stringEntry), true);
            entriesCount++;
        }
        Assert.assertEquals(entriesCount, entries.size());
    }
}

Example 89 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}

Example 90 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class RedirectingStoreTest method testProxyPuts.

@Test
public void testProxyPuts() {
    List<ByteArray> testPrimaryKeys = new ArrayList<ByteArray>(this.proxyPutTestPrimaryEntries.keySet());
    List<ByteArray> testSecondaryKeys = new ArrayList<ByteArray>(this.proxyPutTestSecondaryEntries.keySet());
    final RedirectingStore redirectingStoreNode2 = getRedirectingStore(2, servers[2].getMetadataStore(), "test");
    final RedirectingStore redirectingStoreNode0 = getRedirectingStore(0, servers[0].getMetadataStore(), "test");
    final Store<ByteArray, byte[], byte[]> socketStoreNode2 = redirectingStoreNode2.getRedirectingSocketStore("test", 2);
    final Store<ByteArray, byte[], byte[]> socketStoreNode0 = redirectingStoreNode0.getRedirectingSocketStore("test", 0);
    // 1. Make sure the vector clocks make sense.. Read through Node 2 and
    // proxy getting from Node 0 and issue a write based off that,
    // incrementing the clock for Node 2 and make sure there is no
    // ObsoleteVersionException at both Node 0 and
    // Node 2.
    ByteArray secondaryKey = testSecondaryKeys.get(0);
    VectorClock clock1 = ((VectorClock) redirectingStoreNode2.getVersions(secondaryKey).get(0)).incremented(2, System.currentTimeMillis());
    try {
        redirectingStoreNode2.put(secondaryKey, Versioned.value("write-through".getBytes("UTF-8"), clock1), null);
    } catch (Exception e) {
        fail("Unexpected error in testing write through proxy put");
        e.printStackTrace();
    }
    waitForProxyPutsToDrain(redirectingStoreNode2);
    assertTrue("Unexpected failures in proxy put", redirectingStoreNode2.getProxyPutStats().getNumProxyPutFailures() == 0);
    assertEquals("Unexpected value in Node 2", "write-through", new String(socketStoreNode2.get(secondaryKey, null).get(0).getValue()));
    assertTrue("Proxy write not seen on proxy node 0", "write-through".equals(new String(socketStoreNode0.get(secondaryKey, null).get(0).getValue())));
    // Also test that if put fails locally, proxy put is not attempted.
    try {
        redirectingStoreNode2.put(secondaryKey, Versioned.value("write-through-updated".getBytes("UTF-8"), clock1), null);
        fail("Should have thrown OVE");
    } catch (ObsoleteVersionException ove) {
    // Expected
    } catch (Exception e) {
        fail("Unexpected error in testing write through proxy put");
        e.printStackTrace();
    }
    waitForProxyPutsToDrain(redirectingStoreNode2);
    assertFalse("Proxy write not seen on proxy node 0", "write-through-updated".equals(new String(socketStoreNode0.get(secondaryKey, null).get(0).getValue())));
    // 2. Make sure if the proxy node is still a replica, we don't issue
    // proxy puts. Node 2 -> Node 0 on partition 0, for which Node 0 is
    // still a replica
    ByteArray primaryKey = testPrimaryKeys.get(0);
    VectorClock clock2 = ((VectorClock) redirectingStoreNode2.getVersions(primaryKey).get(0)).incremented(2, System.currentTimeMillis());
    try {
        redirectingStoreNode2.put(primaryKey, Versioned.value("write-through".getBytes("UTF-8"), clock2), null);
    } catch (Exception e) {
        fail("Unexpected error in testing write through proxy put");
        e.printStackTrace();
    }
    waitForProxyPutsToDrain(redirectingStoreNode2);
    assertEquals("Unexpected value in Node 2", "write-through", new String(socketStoreNode2.get(primaryKey, null).get(0).getValue()));
    assertFalse("Proxy write seen on proxy node which is a replica", "write-through".equals(new String(socketStoreNode0.get(primaryKey, null).get(0).getValue())));
    // generate OVE.
    try {
        redirectingStoreNode2.put(primaryKey, Versioned.value("write-through".getBytes("UTF-8"), clock2), null);
        fail("Should have thrown OVE");
    } catch (ObsoleteVersionException ove) {
    // Expected
    } catch (Exception e) {
        fail("Unexpected error in testing write through proxy put");
        e.printStackTrace();
    }
}