Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 11 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ChunkedFileSetTest method getTempStrategy.

private RoutingStrategy getTempStrategy() {
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < 1; i++) {
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, Arrays.asList(0)));
    }
    Cluster cluster = new Cluster("test", nodes);
    RoutingStrategy router = new ConsistentRoutingStrategy(cluster, 1);
    return router;
}

Example 12 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class DefaultStoreClient method getResponsibleNodes.

public List<Node> getResponsibleNodes(K key) {
    RoutingStrategy strategy = (RoutingStrategy) store.getCapability(StoreCapabilityType.ROUTING_STRATEGY);
    @SuppressWarnings("unchecked") Serializer<K> keySerializer = (Serializer<K>) store.getCapability(StoreCapabilityType.KEY_SERIALIZER);
    return strategy.routeRequest(keySerializer.toBytes(key));
}

Example 13 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class BaseStreamingClient method addStoreToSession.

/**
     * Add another store destination to an existing streaming session
     * 
     * 
     * @param store the name of the store to stream to
     */
@SuppressWarnings({ "unchecked", "rawtypes" })
protected void addStoreToSession(String store) {
    Exception initializationException = null;
    storeNames.add(store);
    for (Node node : nodesToStream) {
        SocketDestination destination = null;
        SocketAndStreams sands = null;
        try {
            destination = new SocketDestination(node.getHost(), node.getAdminPort(), RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
            sands = streamingSocketPool.checkout(destination);
            DataOutputStream outputStream = sands.getOutputStream();
            DataInputStream inputStream = sands.getInputStream();
            nodeIdStoreToSocketRequest.put(new Pair(store, node.getId()), destination);
            nodeIdStoreToOutputStreamRequest.put(new Pair(store, node.getId()), outputStream);
            nodeIdStoreToInputStreamRequest.put(new Pair(store, node.getId()), inputStream);
            nodeIdStoreToSocketAndStreams.put(new Pair(store, node.getId()), sands);
            nodeIdStoreInitialized.put(new Pair(store, node.getId()), false);
            remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(node.getId()).getValue();
        } catch (Exception e) {
            logger.error(e);
            try {
                close(sands.getSocket());
                streamingSocketPool.checkin(destination, sands);
            } catch (Exception ioE) {
                logger.error(ioE);
            }
            if (!faultyNodes.contains(node.getId()))
                faultyNodes.add(node.getId());
            initializationException = e;
        }
    }
    if (initializationException != null)
        throw new VoldemortException(initializationException);
    if (store.equals("slop"))
        return;
    boolean foundStore = false;
    for (StoreDefinition remoteStoreDef : remoteStoreDefs) {
        if (remoteStoreDef.getName().equals(store)) {
            RoutingStrategyFactory factory = new RoutingStrategyFactory();
            RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(remoteStoreDef, adminClient.getAdminClientCluster());
            storeToRoutingStrategy.put(store, storeRoutingStrategy);
            validateSufficientNodesAvailable(blackListedNodes, remoteStoreDef);
            foundStore = true;
            break;
        }
    }
    if (!foundStore) {
        logger.error("Store Name not found on the cluster");
        throw new VoldemortException("Store Name not found on the cluster");
    }
}

Example 14 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ImportTextDumpToBDB method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    String inputPath = (String) options.valueOf("input");
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String clusterXmlPath = (String) options.valueOf("cluster-xml");
    String storesXmlPath = (String) options.valueOf("stores-xml");
    Integer nodeId = (Integer) options.valueOf("node-id");
    File input = new File(inputPath);
    List<File> dataFiles = new ArrayList<File>();
    if (input.isDirectory()) {
        File[] files = input.listFiles();
        if (files != null)
            Collections.addAll(dataFiles, files);
    } else if (input.isFile()) {
        dataFiles.add(input);
    } else {
        System.err.println(inputPath + "is not file or directory");
    }
    File storeBdbFolder = new File(storeBdbFolderPath);
    final String storeName = storeBdbFolder.getName();
    Cluster cluster = new ClusterMapper().readCluster(new File(clusterXmlPath));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlPath));
    StoreDefinition storeDef = null;
    for (StoreDefinition sd : storeDefs) {
        if (sd.getName() != null && sd.getName().equals(storeName)) {
            storeDef = sd;
        }
    }
    if (storeDef == null) {
        throw new VoldemortException("StoreNotfound: " + storeName);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition mockStoreDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(mockStoreDef, routingStrategy);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    long lastInserted = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    long inserted = 0;
    for (File f : dataFiles) {
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(f), READER_BUFFER_SIZE);
            engine.beginBatchModifications();
            while (true) {
                String line = bufferedReader.readLine();
                if (line == null) {
                    break;
                }
                Pair<ByteArray, Versioned<byte[]>> entry;
                try {
                    entry = lineToEntry(line);
                } catch (Exception e) {
                    System.err.println("Skipping line: " + line);
                    e.printStackTrace();
                    continue;
                }
                ByteArray key = entry.getFirst();
                List<Node> nodeList = routingStrategy.routeRequest(key.get());
                for (Node node : nodeList) {
                    if (nodeId == node.getId()) {
                        try {
                            engine.put(key, entry.getSecond(), null);
                            inserted++;
                        } catch (ObsoleteVersionException e) {
                            e.printStackTrace();
                        }
                        break;
                    }
                }
                count++;
                final Long countObject = count;
                final Long insertedObject = inserted;
                Boolean reported = rp.tryReport(new Callable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                        System.out.print(String.format("Imported %15d entries; Inserted %15d entries", countObject, insertedObject));
                        return true;
                    }
                });
                if (reported != null) {
                    long importSpeed = (count - lastCount) / (reportIntervalMs / 1000);
                    long insertSpeed = (inserted - lastInserted) / (reportIntervalMs / 1000);
                    System.out.println(String.format("; ImportSpeed: %8d/s; InsertSpeed: %8d/s ", importSpeed, insertSpeed));
                    lastCount = count;
                    lastInserted = inserted;
                }
            }
            bufferedReader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            engine.endBatchModifications();
        }
    }
    engine.close();
    System.out.println(String.format("Finished importing %d entries (%d inserted, rest discarded)", count, inserted));
}

Example 15 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class RoutedStoreTest method testPutDeleteZoneRouting.

@Test
public void testPutDeleteZoneRouting() throws Exception {
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 2);
    Versioned<byte[]> versioned = new Versioned<byte[]>(new byte[] { 1 });
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    Set<Integer> sleepy = Sets.newHashSet(4, 5, 6, 7);
    for (Node n : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> subStore = null;
        if (sleepy != null && sleepy.contains(n.getId()))
            subStore = new SleepyStore<ByteArray, byte[], byte[]>(SLEEPY_TIME, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        else
            subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
    }
    setFailureDetector(subStores);
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 1, 1, 1, 1, 0, 0, zoneReplicationFactor, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    routedStoreThreadPool = Executors.newFixedThreadPool(8);
    RoutedStoreFactory routedStoreFactory = createFactory();
    Store<ByteArray, byte[], byte[]> s1 = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(OPERATION_TIMEOUT));
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    List<Node> nodesRoutedTo = routingStrategy.routeRequest("test".getBytes());
    long start = System.nanoTime(), elapsed;
    try {
        s1.put(new ByteArray("test".getBytes()), versioned, null);
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).get(0), versioned);
    }
    // make sure the failure detector adds back any previously failed nodes
    Thread.sleep(BANNAGE_PERIOD + 100);
    start = System.nanoTime();
    try {
        s1.delete(new ByteArray("test".getBytes()), versioned.getVersion());
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).size(), 0);
    }
}