Examples with FnvHashFunction voldemort.utils.FnvHashFunction used on opensource projects

Example 1 with FnvHashFunction

use of voldemort.utils.FnvHashFunction in project voldemort by voldemort.

the class FnvHashFunctionTester method main.

public static void main(String[] argv) throws Exception {
    if (argv.length != 1) {
        System.err.println("USAGE: java voldemort.partition.FnvHashFunctionTester filename");
        System.exit(1);
    }
    FnvHashFunction hash = new FnvHashFunction();
    BufferedReader reader = new BufferedReader(new FileReader(argv[0]));
    while (true) {
        String line = reader.readLine();
        if (line == null)
            break;
        System.out.println(hash.hash(line.trim().getBytes()));
    }
    reader.close();
}

Example 2 with FnvHashFunction

use of voldemort.utils.FnvHashFunction in project voldemort by voldemort.

the class BdbConvertBaseToPidScan method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    byte[] prevKey = null;
    List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
    HashFunction hash = new FnvHashFunction();
    int totalPartitions = cluster.getNumberOfPartitions();
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        scanCount++;
        if (scanCount % 1000000 == 0)
            logger.info("Converted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
        // read the value as a versioned Object
        VectorClock clock = new VectorClock(valueEntry.getData());
        byte[] bytes = ByteUtils.copy(valueEntry.getData(), clock.sizeInBytes(), valueEntry.getData().length);
        Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
        byte[] key = keyEntry.getData();
        if (prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
            // there is a new key; write out the buffered values and
            // previous key
            int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            vals = new ArrayList<Versioned<byte[]>>();
            keyCount++;
        }
        vals.add(value);
        prevKey = key;
    }
    if (vals.size() > 0) {
        int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        keyCount++;
    }
    logger.info("Completed " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}

Example 3 with FnvHashFunction

use of voldemort.utils.FnvHashFunction in project voldemort by voldemort.

the class ConsistentRoutingStrategyTest method testLoadBalancing.

public void testLoadBalancing(int numNodes, int tagsPerNode, int numRequests, int replicationFactor) {
    List<Integer> tags = new ArrayList<Integer>();
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes * tagsPerNode; i++) tags.add(i);
    for (int i = 0; i < numNodes; i++) nodes.add(new Node(i, "host", 8080, 6666, 6667, tags.subList(tagsPerNode * i, tagsPerNode * (i + 1))));
    // use a seed so that this test is repeatable
    Random random = new Random(2158745224L);
    Collections.shuffle(nodes, random);
    ConsistentRoutingStrategy router = new ConsistentRoutingStrategy(new FnvHashFunction(), nodes, replicationFactor);
    for (Node n : nodes) assertEquals(tagsPerNode, router.getPartitionsByNode(n).size());
    // do some requests and test the load balancing
    Multiset<Integer> masters = HashMultiset.create();
    Multiset<Integer> counts = HashMultiset.create();
    byte[] key = new byte[16];
    for (int i = 0; i < numRequests; i++) {
        random.nextBytes(key);
        List<Node> routed = router.routeRequest(key);
        assertEquals(replicationFactor, routed.size());
        masters.add(routed.get(0).getId());
        for (Node n : routed) counts.add(n.getId());
    }
    System.out.println("numNodes = " + numNodes + ", tagsPerNode = " + tagsPerNode + ", numRequests = " + numRequests);
    System.out.println("master node distribution:");
    assertWellBalanced(numNodes, masters);
    System.out.println();
    System.out.println("storage node distribution:");
    assertWellBalanced(numNodes, counts);
    System.out.println();
}

Example 4 with FnvHashFunction

use of voldemort.utils.FnvHashFunction in project voldemort by voldemort.

the class ConsistentRoutingStrategyTest method testTagAssignment.

public void testTagAssignment() {
    List<Node> nodes = getTestNodes();
    ConsistentRoutingStrategy router = getRouter(new FnvHashFunction(), 3);
    for (Node n : nodes) for (Integer tag : n.getPartitionIds()) assertEquals(router.getNodeByPartition(tag), n);
    for (int i = 0; i < nodes.size(); i++) assertEquals("Unexpected tag assignment for tag " + i + ": ", new HashSet<Integer>(nodes.get(i).getPartitionIds()), router.getPartitionsByNode(nodes.get(i)));
}

Example 5 with FnvHashFunction

use of voldemort.utils.FnvHashFunction in project voldemort by voldemort.

the class BdbConvertNewDupToPidScan method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    HashFunction hash = new FnvHashFunction();
    int totalPartitions = cluster.getNumberOfPartitions();
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        scanCount += vals.size();
        int partition = BdbConvertData.abs(hash.hash(keyEntry.getData())) % (Math.max(1, totalPartitions));
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(keyEntry.getData(), partition)), valueEntry);
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Converted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}