Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 6 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testFetchAsStreamWithFilter.

@Test
public void testFetchAsStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    VoldemortFilter filter = new VoldemortFilterImpl();
    int shouldFilterCount = 0;
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : createEntries()) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0)) {
            store.put(pair.getFirst(), pair.getSecond(), null);
            if (!filter.accept(pair.getFirst(), pair.getSecond())) {
                shouldFilterCount++;
            }
        }
    }
    // make fetch stream call with filter
    Iterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = getAdminClient().bulkFetchOps.fetchEntries(0, testStoreName, getCluster().getNodeById(0).getPartitionIds(), filter, false);
    // assert none of the filtered entries are returned.
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        if (!filter.accept(entry.getFirst(), entry.getSecond())) {
            fail();
        }
    }
}

Example 7 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class StorageService method openStore.

public StorageEngine<ByteArray, byte[], byte[]> openStore(StoreDefinition storeDef) {
    logger.info("Opening store '" + storeDef.getName() + "' (" + storeDef.getType() + ").");
    StorageConfiguration config = storageConfigs.get(storeDef.getType());
    if (config == null)
        throw new ConfigurationException("Attempt to open store " + storeDef.getName() + " but " + storeDef.getType() + " storage engine has not been enabled.");
    boolean isReadOnly = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, metadata.getCluster());
    final StorageEngine<ByteArray, byte[], byte[]> engine = config.getStore(storeDef, routingStrategy);
    // Update the routing strategy + add listener to metadata
    if (storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) {
        metadata.addMetadataStoreListener(storeDef.getName(), new MetadataStoreListener() {

            public void updateRoutingStrategy(RoutingStrategy updatedRoutingStrategy) {
                ((ReadOnlyStorageEngine) engine).setRoutingStrategy(updatedRoutingStrategy);
            }

            public void updateStoreDefinition(StoreDefinition storeDef) {
                return;
            }
        });
    }
    // openStore() should have atomic semantics
    try {
        registerEngine(engine, isReadOnly, storeDef.getType(), storeDef);
        if (voldemortConfig.isServerRoutingEnabled())
            registerNodeStores(storeDef, metadata.getCluster(), voldemortConfig.getNodeId());
        if (storeDef.hasRetentionPeriod())
            scheduleCleanupJob(storeDef, engine);
    } catch (Exception e) {
        removeEngine(engine, isReadOnly, storeDef.getType(), false);
        throw new VoldemortException(e);
    }
    return engine;
}

Example 8 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method populateData.

protected void populateData(Cluster cluster, StoreDefinition storeDef, AdminClient adminClient, boolean isReadOnly) throws Exception {
    // Populate Read write stores
    if (!isReadOnly) {
        // Create SocketStores for each Node first
        Map<Integer, Store<ByteArray, byte[], byte[]>> storeMap = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
        for (Node node : cluster.getNodes()) {
            storeMap.put(node.getId(), getSocketStore(storeDef.getName(), node.getHost(), node.getSocketPort()));
        }
        BaseStoreRoutingPlan storeInstance = new BaseStoreRoutingPlan(cluster, storeDef);
        for (Entry<String, String> entry : testEntries.entrySet()) {
            ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
            List<Integer> preferenceNodes = storeInstance.getReplicationNodeList(keyBytes.get());
            // Go over every node
            for (int nodeId : preferenceNodes) {
                try {
                    storeMap.get(nodeId).put(keyBytes, new Versioned<byte[]>(ByteUtils.getBytes(entry.getValue(), "UTF-8")), null);
                } catch (ObsoleteVersionException e) {
                    logger.info("Why are we seeing this at all here ?? ");
                    e.printStackTrace();
                }
            }
        }
        // close all socket stores
        for (Store<ByteArray, byte[], byte[]> store : storeMap.values()) {
            store.close();
        }
    } else {
        // Populate Read only stores
        File baseDir = TestUtils.createTempDir();
        JsonReader reader = ReadOnlyStorageEngineTestInstance.makeTestDataReader(testEntries, baseDir);
        RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        File outputDir = TestUtils.createTempDir(baseDir);
        JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testEntries.size() / 5, 1, NUM_RO_CHUNKS_PER_BUCKET, 10000, false);
        storeBuilder.build(ReadOnlyStorageFormat.READONLY_V2);
        AdminStoreSwapper swapper = new AdminStoreSwapper(Executors.newFixedThreadPool(cluster.getNumberOfNodes()), adminClient, 100000);
        swapper.fetchAndSwapStoreData(testStoreNameRO, outputDir.getAbsolutePath(), 1L);
    }
}

Example 9 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ROTestUtils method getNodeIdToAllPartitions.

/**
     * For a particular cluster creates a mapping of node id to their
     * corresponding list of [ replicaType, partition ] tuple
     * 
     * @param cluster The cluster metadata
     * @param storeDef The store definition
     * @param includePrimary Include the primary partition?
     * @return Map of node id to set of [ replicaType, partition ] tuple
     */
public static Map<Integer, Set<Pair<Integer, Integer>>> getNodeIdToAllPartitions(final Cluster cluster, final StoreDefinition storeDef, boolean includePrimary) {
    final RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    final Map<Integer, Set<Pair<Integer, Integer>>> nodeIdToReplicas = new HashMap<Integer, Set<Pair<Integer, Integer>>>();
    final Map<Integer, Integer> partitionToNodeIdMap = cluster.getPartitionIdToNodeIdMap();
    // Map initialization.
    for (Node node : cluster.getNodes()) {
        nodeIdToReplicas.put(node.getId(), new HashSet<Pair<Integer, Integer>>());
    }
    // Track how many zones actually have partitions (and so replica types)
    // in them.
    int zonesWithPartitions = 0;
    for (Integer zoneId : cluster.getZoneIds()) {
        if (cluster.getNumberOfPartitionsInZone(zoneId) > 0) {
            zonesWithPartitions++;
        }
    }
    // Loops through all nodes
    for (Node node : cluster.getNodes()) {
        // Gets the partitions that this node was configured with.
        for (Integer primary : node.getPartitionIds()) {
            // Gets the list of replicating partitions.
            List<Integer> replicaPartitionList = routingStrategy.getReplicatingPartitionList(primary);
            if ((replicaPartitionList.size() % zonesWithPartitions != 0) || ((replicaPartitionList.size() / zonesWithPartitions) != (storeDef.getReplicationFactor() / cluster.getNumberOfZones()))) {
                // For zone expansion & shrinking, this warning is expected
                // in some cases. For other use cases (shuffling, cluster
                // expansion), this warning indicates that something
                // is wrong between the clusters and store defs.
                logger.warn("Number of replicas returned (" + replicaPartitionList.size() + ") does not make sense given the replication factor (" + storeDef.getReplicationFactor() + ") and that there are " + cluster.getNumberOfZones() + " zones of which " + zonesWithPartitions + " have partitions (and of which " + (cluster.getNumberOfZones() - zonesWithPartitions) + " are empty).");
            }
            int replicaType = 0;
            if (!includePrimary) {
                replicaPartitionList.remove(primary);
                replicaType = 1;
            }
            // Get the node that this replicating partition belongs to.
            for (Integer replicaPartition : replicaPartitionList) {
                Integer replicaNodeId = partitionToNodeIdMap.get(replicaPartition);
                // The replicating node will have a copy of primary.
                nodeIdToReplicas.get(replicaNodeId).add(Pair.create(replicaType, primary));
                replicaType++;
            }
        }
    }
    return nodeIdToReplicas;
}

Example 10 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}