Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 21 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class JsonStoreBuilder method buildVersion2.

public void buildVersion2() throws IOException {
    logger.info("Building store " + storeDefinition.getName() + " for " + cluster.getNumberOfPartitions() + " partitions, " + storeDefinition.getReplicationFactor() + " replica types, " + numChunks + " chunks per partitions per replica type and type " + ReadOnlyStorageFormat.READONLY_V2);
    // Initialize files
    DataOutputStream[][] indexes = new DataOutputStream[cluster.getNumberOfPartitions()][];
    DataOutputStream[][] datas = new DataOutputStream[cluster.getNumberOfPartitions()][];
    int[][] positions = new int[cluster.getNumberOfPartitions()][];
    File tempDirectory = new File(Utils.notNull(System.getProperty("java.io.tmpdir")), "tempDir-" + Integer.toString(new Random().nextInt()));
    Utils.mkdirs(tempDirectory);
    for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
        indexes[partitionId] = new DataOutputStream[storeDefinition.getReplicationFactor() * numChunks];
        datas[partitionId] = new DataOutputStream[storeDefinition.getReplicationFactor() * numChunks];
        positions[partitionId] = new int[storeDefinition.getReplicationFactor() * numChunks];
        int globalChunkId = 0;
        for (int repType = 0; repType < storeDefinition.getReplicationFactor(); repType++) {
            for (int chunk = 0; chunk < numChunks; chunk++) {
                File indexFile = new File(tempDirectory, Integer.toString(partitionId) + "_" + Integer.toString(repType) + "_" + Integer.toString(chunk) + ".index");
                File dataFile = new File(tempDirectory, Integer.toString(partitionId) + "_" + Integer.toString(repType) + "_" + Integer.toString(chunk) + ".data");
                positions[partitionId][globalChunkId] = 0;
                indexes[partitionId][globalChunkId] = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(indexFile), ioBufferSize));
                datas[partitionId][globalChunkId] = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(dataFile), ioBufferSize));
                globalChunkId++;
            }
        }
    }
    logger.info("Reading items...");
    ExternalSorter<KeyValuePair> sorter = new ExternalSorter<KeyValuePair>(new KeyValuePairSerializer(), new KeyMd5Comparator(), internalSortSize, tempDir.getAbsolutePath(), ioBufferSize, numThreads, gzipIntermediate);
    JsonObjectIterator iter = new JsonObjectIterator(reader, storeDefinition);
    int count = 0;
    HashMap<Pair<Integer, Integer>, Pair<byte[], byte[]>> previousElements = Maps.newHashMap();
    for (KeyValuePair currentElement : sorter.sorted(iter)) {
        List<Integer> partitionIds = this.routingStrategy.getPartitionList(currentElement.getKey());
        int masterPartition = partitionIds.get(0);
        int localChunkId = ReadOnlyUtils.chunk(currentElement.getKeyMd5(), numChunks);
        for (int replicaType = 0; replicaType < partitionIds.size(); replicaType++) {
            int globalChunkId = (replicaType * numChunks) + localChunkId;
            Pair<Integer, Integer> key = Pair.create(masterPartition, globalChunkId);
            if (!previousElements.containsKey(key)) {
                // First element, lets write it to map
                previousElements.put(key, Pair.create(ByteUtils.copy(currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT), generateFirstElement(currentElement)));
            } else {
                Pair<byte[], byte[]> previousElement = previousElements.get(key);
                // append it...
                if (ByteUtils.compare(previousElement.getFirst(), currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT) == 0) {
                    short numKeys = ByteUtils.readShort(previousElement.getSecond(), 0);
                    ByteArrayOutputStream stream = new ByteArrayOutputStream();
                    DataOutputStream valueStream = new DataOutputStream(stream);
                    valueStream.writeShort(numKeys + 1);
                    // Append the previous tuples
                    valueStream.write(ByteUtils.copy(previousElement.getSecond(), ByteUtils.SIZE_OF_SHORT, previousElement.getSecond().length));
                    valueStream.writeInt(currentElement.getKey().length);
                    valueStream.writeInt(currentElement.getValue().length);
                    valueStream.write(currentElement.getKey());
                    valueStream.write(currentElement.getValue());
                    valueStream.flush();
                    previousElements.put(key, Pair.create(previousElement.getFirst(), stream.toByteArray()));
                } else {
                    // ...else, flush the previous element to disk
                    indexes[masterPartition][globalChunkId].write(previousElement.getFirst());
                    indexes[masterPartition][globalChunkId].writeInt(positions[masterPartition][globalChunkId]);
                    datas[masterPartition][globalChunkId].write(previousElement.getSecond());
                    positions[masterPartition][globalChunkId] += previousElement.getSecond().length;
                    // ...and add current element as previous element
                    previousElements.put(key, Pair.create(ByteUtils.copy(currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT), generateFirstElement(currentElement)));
                }
            }
        }
        count++;
    }
    logger.info(count + " items read.");
    // files
    for (Entry<Pair<Integer, Integer>, Pair<byte[], byte[]>> entry : previousElements.entrySet()) {
        int partitionId = entry.getKey().getFirst();
        int globalChunkId = entry.getKey().getSecond();
        byte[] keyMd5 = entry.getValue().getFirst();
        byte[] value = entry.getValue().getSecond();
        indexes[partitionId][globalChunkId].write(keyMd5);
        indexes[partitionId][globalChunkId].writeInt(positions[partitionId][globalChunkId]);
        datas[partitionId][globalChunkId].write(value);
    }
    // Create node folders
    Map<Integer, File> nodeDirs = new HashMap<Integer, File>(cluster.getNumberOfNodes());
    for (Node node : cluster.getNodes()) {
        int nodeId = node.getId();
        // Create data directory
        File nodeDir = new File(outputDir, "node-" + Integer.toString(nodeId));
        nodeDir.mkdirs();
        // Add the data directory to the array
        nodeDirs.put(node.getId(), nodeDir);
        // Create metadata file
        BufferedWriter writer = new BufferedWriter(new FileWriter(new File(nodeDir, ".metadata")));
        ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
        metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
        writer.write(metadata.toJsonString());
        writer.close();
    }
    // Close everything
    logger.info("Closing all store files.");
    for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
        for (int chunk = 0; chunk < numChunks * storeDefinition.getReplicationFactor(); chunk++) {
            indexes[partitionId][chunk].close();
            datas[partitionId][chunk].close();
        }
    }
    // Start moving files over to their correct node
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    Map<Integer, Integer> replicaMapping = cluster.getPartitionIdToNodeIdMap();
    for (File file : tempDirectory.listFiles()) {
        String fileName = file.getName();
        if (fileName.matches("^[\\d]+_[\\d]+_[\\d]+\\.(data|index)")) {
            String[] props = fileName.split("_");
            int partitionId = Integer.parseInt(props[0]);
            int replicaType = Integer.parseInt(props[1]);
            int nodeId = replicaMapping.get(strategy.getReplicatingPartitionList(partitionId).get(replicaType));
            Utils.move(file, new File(nodeDirs.get(nodeId), fileName));
        }
    }
}

Example 22 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class JsonStoreBuilder method main.

/**
     * Main method to run on a input text file
     * 
     * @param args see USAGE for details
     * @throws IOException
     */
public static void main(String[] args) throws IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("cluster", "[REQUIRED] path to cluster xml config file").withRequiredArg().describedAs("cluster.xml");
    parser.accepts("stores", "[REQUIRED] path to stores xml config file").withRequiredArg().describedAs("stores.xml");
    parser.accepts("name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("buffer", "[REQUIRED] number of key/value pairs to buffer in memory").withRequiredArg().ofType(Integer.class);
    parser.accepts("input", "[REQUIRED] input file to read from").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] directory to output stores to").withRequiredArg().describedAs("output directory");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("chunks", "number of chunks [per node, per partition, per partition + replica]").withRequiredArg().ofType(Integer.class);
    parser.accepts("io-buffer-size", "size of i/o buffers in bytes").withRequiredArg().ofType(Integer.class);
    parser.accepts("temp-dir", "temporary directory for sorted file pieces").withRequiredArg().describedAs("temp dir");
    parser.accepts("gzip", "compress intermediate chunk files");
    parser.accepts("format", "read-only store format [" + ReadOnlyStorageFormat.READONLY_V0.getCode() + "," + ReadOnlyStorageFormat.READONLY_V1.getCode() + "," + ReadOnlyStorageFormat.READONLY_V2.getCode() + "]").withRequiredArg().ofType(String.class);
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "cluster", "stores", "name", "buffer", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    String clusterFile = (String) options.valueOf("cluster");
    String storeDefFile = (String) options.valueOf("stores");
    String storeName = (String) options.valueOf("name");
    int sortBufferSize = (Integer) options.valueOf("buffer");
    String inputFile = (String) options.valueOf("input");
    File outputDir = new File((String) options.valueOf("output"));
    int numThreads = CmdUtils.valueOf(options, "threads", 2);
    int chunks = CmdUtils.valueOf(options, "chunks", 2);
    int ioBufferSize = CmdUtils.valueOf(options, "io-buffer-size", 1000000);
    ReadOnlyStorageFormat storageFormat = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "format", ReadOnlyStorageFormat.READONLY_V2.getCode()));
    boolean gzipIntermediate = options.has("gzip");
    File tempDir = new File(CmdUtils.valueOf(options, "temp-dir", System.getProperty("java.io.tmpdir")));
    try {
        JsonReader reader = new JsonReader(new BufferedReader(new FileReader(inputFile), ioBufferSize));
        Cluster cluster = new ClusterMapper().readCluster(new BufferedReader(new FileReader(clusterFile)));
        StoreDefinition storeDef = null;
        List<StoreDefinition> stores = new StoreDefinitionsMapper().readStoreList(new BufferedReader(new FileReader(storeDefFile)));
        for (StoreDefinition def : stores) {
            if (def.getName().equals(storeName))
                storeDef = def;
        }
        if (storeDef == null)
            Utils.croak("No store found with name \"" + storeName + "\"");
        if (!outputDir.exists())
            Utils.croak("Directory \"" + outputDir.getAbsolutePath() + "\" does not exist.");
        RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        new JsonStoreBuilder(reader, cluster, storeDef, routingStrategy, outputDir, tempDir, sortBufferSize, numThreads, chunks, ioBufferSize, gzipIntermediate).build(storageFormat);
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 23 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ConfigureNodesTest method testConfigureNodesNotEnoughNodes.

@Test(expected = InsufficientOperationalNodesException.class)
public void testConfigureNodesNotEnoughNodes() throws Exception {
    RoutingStrategy routingStrategy = new RouteToAllStrategy(cluster.getNodesShuffled());
    BasicPipelineData<byte[]> pipelineData = new BasicPipelineData<byte[]>();
    ConfigureNodes<byte[], BasicPipelineData<byte[]>> action = new ConfigureNodes<byte[], BasicPipelineData<byte[]>>(pipelineData, Event.COMPLETED, failureDetector, cluster.getNodes().size() + 1, routingStrategy, aKey, null);
    Pipeline pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, action);
    pipeline.addEvent(Event.STARTED);
    pipeline.execute();
    if (pipelineData.getFatalError() != null)
        throw pipelineData.getFatalError();
    else
        fail();
}

Example 24 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ConfigureNodesTest method testConfigureNodes.

@Test
public void testConfigureNodes() throws Exception {
    RoutingStrategy routingStrategy = new RouteToAllStrategy(cluster.getNodesShuffled());
    BasicPipelineData<byte[]> pipelineData = new BasicPipelineData<byte[]>();
    ConfigureNodes<byte[], BasicPipelineData<byte[]>> action = new ConfigureNodes<byte[], BasicPipelineData<byte[]>>(pipelineData, Event.COMPLETED, failureDetector, 1, routingStrategy, aKey, null);
    Pipeline pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, action);
    pipeline.addEvent(Event.STARTED);
    pipeline.execute();
    if (pipelineData.getFatalError() != null)
        throw pipelineData.getFatalError();
    assertEquals(cluster.getNodes().size(), pipelineData.getNodes().size());
}

Example 25 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class GetAllConfigureNodesTest method testConfigureNodesNotEnoughNodes.

@Test(expected = InsufficientOperationalNodesException.class)
public void testConfigureNodesNotEnoughNodes() throws Exception {
    for (Node node : cluster.getNodes()) failureDetector.recordException(node, 0, new UnreachableStoreException("Test for " + getClass().getName()));
    RoutingStrategy routingStrategy = new RouteToAllStrategy(cluster.getNodesShuffled());
    GetAllPipelineData pipelineData = new GetAllPipelineData();
    GetAllConfigureNodes action = new GetAllConfigureNodes(pipelineData, Event.COMPLETED, failureDetector, 1, 1, routingStrategy, Arrays.asList(aKey), null, null, null);
    Pipeline pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, action);
    pipeline.addEvent(Event.STARTED);
    pipeline.execute();
    throw pipelineData.getFatalError();
}