Examples with RoutingStrategyFactory voldemort.routing.RoutingStrategyFactory used on opensource projects

Example 31 with RoutingStrategyFactory

use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.

the class JsonStoreBuilder method main.

/**
     * Main method to run on a input text file
     * 
     * @param args see USAGE for details
     * @throws IOException
     */
public static void main(String[] args) throws IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("cluster", "[REQUIRED] path to cluster xml config file").withRequiredArg().describedAs("cluster.xml");
    parser.accepts("stores", "[REQUIRED] path to stores xml config file").withRequiredArg().describedAs("stores.xml");
    parser.accepts("name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("buffer", "[REQUIRED] number of key/value pairs to buffer in memory").withRequiredArg().ofType(Integer.class);
    parser.accepts("input", "[REQUIRED] input file to read from").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] directory to output stores to").withRequiredArg().describedAs("output directory");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("chunks", "number of chunks [per node, per partition, per partition + replica]").withRequiredArg().ofType(Integer.class);
    parser.accepts("io-buffer-size", "size of i/o buffers in bytes").withRequiredArg().ofType(Integer.class);
    parser.accepts("temp-dir", "temporary directory for sorted file pieces").withRequiredArg().describedAs("temp dir");
    parser.accepts("gzip", "compress intermediate chunk files");
    parser.accepts("format", "read-only store format [" + ReadOnlyStorageFormat.READONLY_V0.getCode() + "," + ReadOnlyStorageFormat.READONLY_V1.getCode() + "," + ReadOnlyStorageFormat.READONLY_V2.getCode() + "]").withRequiredArg().ofType(String.class);
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "cluster", "stores", "name", "buffer", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    String clusterFile = (String) options.valueOf("cluster");
    String storeDefFile = (String) options.valueOf("stores");
    String storeName = (String) options.valueOf("name");
    int sortBufferSize = (Integer) options.valueOf("buffer");
    String inputFile = (String) options.valueOf("input");
    File outputDir = new File((String) options.valueOf("output"));
    int numThreads = CmdUtils.valueOf(options, "threads", 2);
    int chunks = CmdUtils.valueOf(options, "chunks", 2);
    int ioBufferSize = CmdUtils.valueOf(options, "io-buffer-size", 1000000);
    ReadOnlyStorageFormat storageFormat = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "format", ReadOnlyStorageFormat.READONLY_V2.getCode()));
    boolean gzipIntermediate = options.has("gzip");
    File tempDir = new File(CmdUtils.valueOf(options, "temp-dir", System.getProperty("java.io.tmpdir")));
    try {
        JsonReader reader = new JsonReader(new BufferedReader(new FileReader(inputFile), ioBufferSize));
        Cluster cluster = new ClusterMapper().readCluster(new BufferedReader(new FileReader(clusterFile)));
        StoreDefinition storeDef = null;
        List<StoreDefinition> stores = new StoreDefinitionsMapper().readStoreList(new BufferedReader(new FileReader(storeDefFile)));
        for (StoreDefinition def : stores) {
            if (def.getName().equals(storeName))
                storeDef = def;
        }
        if (storeDef == null)
            Utils.croak("No store found with name \"" + storeName + "\"");
        if (!outputDir.exists())
            Utils.croak("Directory \"" + outputDir.getAbsolutePath() + "\" does not exist.");
        RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        new JsonStoreBuilder(reader, cluster, storeDef, routingStrategy, outputDir, tempDir, sortBufferSize, numThreads, chunks, ioBufferSize, gzipIntermediate).build(storageFormat);
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 32 with RoutingStrategyFactory

use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.

the class HadoopStoreBuilderTest method testHadoopBuild.

@Test
public void testHadoopBuild() throws Exception {
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
    File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < 200; i++) values.put(Integer.toString(i), Integer.toBinaryString(i));
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
    FileUtils.writeStringToFile(inputFile, contents.toString());
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    // Test backwards compatibility
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir2.getAbsolutePath()), new Path(outputDir2.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    // Check if checkSum is generated in outputDir
    File nodeFile = new File(outputDir, "node-0");
    // Check if metadata file exists
    File metadataFile = new File(nodeFile, ".metadata");
    Assert.assertTrue("Metadata file should exist!", metadataFile.exists());
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata(metadataFile);
    if (saveKeys)
        Assert.assertEquals("In saveKeys mode, the metadata format should be READONLY_V2!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V2.getCode());
    else
        Assert.assertEquals("In legacy mode (saveKeys==false), the metadata format should be READONLY_V1!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V1.getCode());
    Assert.assertEquals("Checksum type should be MD5!", metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE), CheckSum.toString(CheckSumType.MD5));
    // Check contents of checkSum file
    byte[] md5 = Hex.decodeHex(((String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM)).toCharArray());
    byte[] checkSumBytes = CheckSumTests.calculateCheckSum(nodeFile.listFiles(), CheckSumType.MD5);
    Assert.assertEquals("Checksum is not as excepted!", 0, ByteUtils.compare(checkSumBytes, md5));
    // check if fetching works
    HdfsFetcher fetcher = new HdfsFetcher();
    // Fetch to version directory
    File versionDir = new File(storeDir, "version-0");
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    Assert.assertTrue("Version directory should exist!", versionDir.exists());
    // open store
    @SuppressWarnings("unchecked") Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, searchStrategy, new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        String key = entry.getKey();
        try {
            List<Versioned<Object>> found = store.get(key, null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        } catch (VoldemortException e) {
            throw new VoldemortException("Got an exception while trying to get key '" + key + "'.", e);
        }
    }
    // also check the iterator - first key iterator...
    try {
        ClosableIterator<ByteArray> keyIterator = engine.keys();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (keyIterator.hasNext()) {
            Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
    // ... and entry iterator
    try {
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
}