Examples with StoreDefinitionsMapper voldemort.xml.StoreDefinitionsMapper used on opensource projects

Example 51 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class MetaOperationsTest method testMetaSet.

@Test
public void testMetaSet() throws Exception {
    // check if all old metadata are the same
    Set<Object> storesValues = Sets.newHashSet();
    for (Node node : adminClient.getAdminClientCluster().getNodes()) {
        Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), MetadataStore.STORES_KEY);
        assertTrue(versioned != null && versioned.getValue() != null);
        storesValues.add(new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue())));
    }
    assertTrue(storesValues.size() == 1);
    // create new metadata object
    final String newStoreXMLFilePrefix = "updated.stores";
    final String newStoreXMLFileSuffix = "xml";
    List<StoreDefinition> newStoresToUpload = new ArrayList<StoreDefinition>();
    for (StoreDefinition storeDef : stores) {
        StoreDefinitionBuilder sb = AdminToolTestUtils.storeDefToBuilder(storeDef);
        sb.setRoutingPolicy(sb.getRoutingPolicy().equals(RoutingTier.CLIENT) ? RoutingTier.SERVER : RoutingTier.CLIENT);
        sb.setPreferredReads(sb.getPreferredReads() == null ? sb.getRequiredReads() : null);
        newStoresToUpload.add(sb.build());
    }
    // create file of new metadata
    File newStoresXMLFolder = TestUtils.createTempDir();
    File newStoreXMLFile = File.createTempFile(newStoreXMLFilePrefix, newStoreXMLFileSuffix, newStoresXMLFolder);
    FileWriter fwriter = new FileWriter(newStoreXMLFile);
    fwriter.write(new StoreDefinitionsMapper().writeStoreList(newStoresToUpload));
    fwriter.close();
    // set new metadata from file
    AdminCommand.executeCommand(new String[] { "meta", "set", MetadataStore.STORES_KEY + "=" + newStoreXMLFile.getAbsolutePath(), "-u", bsURL, "--confirm" });
    // fetch new metadata from node
    List<StoreDefinition> newStoresToVerify = adminClient.metadataMgmtOps.getRemoteStoreDefList(adminClient.getAdminClientCluster().getNodes().iterator().next().getId()).getValue();
    System.out.println("Uploades stores" + new StoreDefinitionsMapper().writeStoreList(newStoresToUpload));
    System.out.println("Retrieved stores" + new StoreDefinitionsMapper().writeStoreList(newStoresToVerify));
    // check if new metadata is the same as the source
    assertTrue("Old store should not equal to new stores", !newStoresToVerify.equals(stores));
    assertTrue("Uploaded stores and retrieved stores are different", newStoresToVerify.equals(newStoresToUpload));
    // check if all new metadata are the same
    storesValues.clear();
    for (Node node : adminClient.getAdminClientCluster().getNodes()) {
        Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), MetadataStore.STORES_KEY);
        assertTrue(versioned != null && versioned.getValue() != null);
        storesValues.add(new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue())));
    }
    System.out.println(storesValues.size());
    assertTrue(storesValues.size() == 1);
}

Example 52 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class SetMetadataTest method testSetMetadataStoreXmlVerifyVersionUpdate.

@Test
public void testSetMetadataStoreXmlVerifyVersionUpdate() throws Exception {
    // setup new stores xml
    final String newStoreXMLFilePrefix = "updated.stores";
    final String newStoreXMLFileSuffix = "xml";
    List<StoreDefinition> newStores = new ArrayList<StoreDefinition>();
    for (StoreDefinition storeDef : ClusterTestUtils.getZZZStoreDefsInMemory()) {
        StoreDefinitionBuilder sb = storeDefToBuilder(storeDef);
        Map<Integer, Integer> zrf = sb.getZoneReplicationFactor();
        Integer zone0RepFactor = zrf.get(0);
        zrf.remove(0);
        sb.setReplicationFactor(sb.getReplicationFactor() - zone0RepFactor);
        newStores.add(sb.build());
    }
    File newStoresXMLFolder = TestUtils.createTempDir();
    File newStoreXMLFile = File.createTempFile(newStoreXMLFilePrefix, newStoreXMLFileSuffix, newStoresXMLFolder);
    FileWriter fwriter = new FileWriter(newStoreXMLFile);
    fwriter.write(new StoreDefinitionsMapper().writeStoreList(newStores));
    fwriter.close();
    // check version
    String sysStoreName = SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name();
    ByteArray metadataKey = new ByteArray(ByteUtils.getBytes(SystemStoreConstants.VERSIONS_METADATA_KEY, "utf-8"));
    for (VoldemortServer vs : vservers.values()) {
        List<Versioned<byte[]>> result = vs.getStoreRepository().getLocalStore(sysStoreName).get(metadataKey, null);
        String versionInfo = new String(result.get(0).getValue());
        System.out.format("[INITIAL]Version values on node [%d] is: \n %s\n", vs.getIdentityNode().getId(), versionInfo);
        Properties props = new Properties();
        props.load(new ByteArrayInputStream(versionInfo.getBytes()));
        for (StoreDefinition sd : oldStores) {
            if (!props.getProperty(sd.getName()).equals("0")) {
                Assert.fail("Initial version of key [" + sd.getName() + "] on node [" + vs.getIdentityNode().getId() + "] is expected to be 0 but not");
            }
        }
        if (!props.getProperty("cluster.xml").equals("0")) {
            Assert.fail("Final version of key [stores.xml] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
        }
    }
    // update the stores xml
    VoldemortAdminTool.main(new String[] { "--auto", "--set-metadata", MetadataStore.STORES_KEY, "--set-metadata-value", newStoreXMLFile.getAbsolutePath(), "--url", bsURL });
    // properly
    try {
        Thread.sleep(10000);
    } catch (Exception e) {
    }
    // check version
    for (VoldemortServer vs : vservers.values()) {
        List<Versioned<byte[]>> result = vs.getStoreRepository().getLocalStore(sysStoreName).get(metadataKey, null);
        String versionInfo = new String(result.get(0).getValue());
        System.out.format("[FINAL]Version values on node [%d] is: \n %s\n", vs.getIdentityNode().getId(), versionInfo);
        Properties props = new Properties();
        props.load(new ByteArrayInputStream(versionInfo.getBytes()));
        for (StoreDefinition sd : oldStores) {
            if (!(Long.parseLong(props.getProperty(sd.getName())) > 0)) {
                Assert.fail("Final version of key [" + sd.getName() + "] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
            }
        }
        if (!(Long.parseLong(props.getProperty("stores.xml")) > 0)) {
            Assert.fail("Final version of key [stores.xml] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
        }
    }
}

Example 53 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class JsonStoreBuilder method main.

/**
     * Main method to run on a input text file
     * 
     * @param args see USAGE for details
     * @throws IOException
     */
public static void main(String[] args) throws IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("cluster", "[REQUIRED] path to cluster xml config file").withRequiredArg().describedAs("cluster.xml");
    parser.accepts("stores", "[REQUIRED] path to stores xml config file").withRequiredArg().describedAs("stores.xml");
    parser.accepts("name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("buffer", "[REQUIRED] number of key/value pairs to buffer in memory").withRequiredArg().ofType(Integer.class);
    parser.accepts("input", "[REQUIRED] input file to read from").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] directory to output stores to").withRequiredArg().describedAs("output directory");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("chunks", "number of chunks [per node, per partition, per partition + replica]").withRequiredArg().ofType(Integer.class);
    parser.accepts("io-buffer-size", "size of i/o buffers in bytes").withRequiredArg().ofType(Integer.class);
    parser.accepts("temp-dir", "temporary directory for sorted file pieces").withRequiredArg().describedAs("temp dir");
    parser.accepts("gzip", "compress intermediate chunk files");
    parser.accepts("format", "read-only store format [" + ReadOnlyStorageFormat.READONLY_V0.getCode() + "," + ReadOnlyStorageFormat.READONLY_V1.getCode() + "," + ReadOnlyStorageFormat.READONLY_V2.getCode() + "]").withRequiredArg().ofType(String.class);
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "cluster", "stores", "name", "buffer", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    String clusterFile = (String) options.valueOf("cluster");
    String storeDefFile = (String) options.valueOf("stores");
    String storeName = (String) options.valueOf("name");
    int sortBufferSize = (Integer) options.valueOf("buffer");
    String inputFile = (String) options.valueOf("input");
    File outputDir = new File((String) options.valueOf("output"));
    int numThreads = CmdUtils.valueOf(options, "threads", 2);
    int chunks = CmdUtils.valueOf(options, "chunks", 2);
    int ioBufferSize = CmdUtils.valueOf(options, "io-buffer-size", 1000000);
    ReadOnlyStorageFormat storageFormat = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "format", ReadOnlyStorageFormat.READONLY_V2.getCode()));
    boolean gzipIntermediate = options.has("gzip");
    File tempDir = new File(CmdUtils.valueOf(options, "temp-dir", System.getProperty("java.io.tmpdir")));
    try {
        JsonReader reader = new JsonReader(new BufferedReader(new FileReader(inputFile), ioBufferSize));
        Cluster cluster = new ClusterMapper().readCluster(new BufferedReader(new FileReader(clusterFile)));
        StoreDefinition storeDef = null;
        List<StoreDefinition> stores = new StoreDefinitionsMapper().readStoreList(new BufferedReader(new FileReader(storeDefFile)));
        for (StoreDefinition def : stores) {
            if (def.getName().equals(storeName))
                storeDef = def;
        }
        if (storeDef == null)
            Utils.croak("No store found with name \"" + storeName + "\"");
        if (!outputDir.exists())
            Utils.croak("Directory \"" + outputDir.getAbsolutePath() + "\" does not exist.");
        RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        new JsonStoreBuilder(reader, cluster, storeDef, routingStrategy, outputDir, tempDir, sortBufferSize, numThreads, chunks, ioBufferSize, gzipIntermediate).build(storageFormat);
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 54 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ClusterForkLiftToolTest method setUpClusters.

@Before
public void setUpClusters() {
    // setup four nodes with one store and one partition
    final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    try {
        int[][] srcPartitionMap = { { 0 }, { 1 }, { 2 }, { 3 } };
        srcServers = new VoldemortServer[4];
        srcCluster = ServerTestUtils.startVoldemortCluster(4, srcServers, srcPartitionMap, socketStoreFactory, true, null, SRC_STORES_XML, new Properties());
        Node node = srcCluster.getNodeById(0);
        srcBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
        int[][] dstPartitionMap = { { 0 }, { 1 }, { 2 } };
        dstServers = new VoldemortServer[3];
        dstCluster = ServerTestUtils.startVoldemortCluster(3, dstServers, dstPartitionMap, socketStoreFactory, true, null, DST_STORES_XML, new Properties());
        node = dstCluster.getNodeById(0);
        dstBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
        kvPairs = ServerTestUtils.createRandomKeyValueString(100);
        int keyCount = 0;
        for (String key : kvPairs.keySet()) {
            if (keyCount == 0)
                firstKey = key;
            if (keyCount == kvPairs.size() - 1)
                lastKey = key;
            if (keyCount == kvPairs.size() / 2)
                conflictKey = key;
            keyCount++;
        }
        srcAdminClient = new AdminClient(srcCluster);
        List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(SRC_STORES_XML));
        primaryResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, PRIMARY_RESOLVING_STORE_NAME);
        globallyResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, GLOBALLY_RESOLVING_STORE_NAME);
        nonResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, MULTIPLE_VERSIONS_STORE_NAME);
        srcfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(srcBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
        srcPrimaryResolvingStoreClient = srcfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
        srcGloballyResolvingStoreClient = srcfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
        dstfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(dstBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
        dstPrimaryResolvingStoreClient = dstfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
        dstGloballyResolvingStoreClient = dstfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
    } catch (IOException e) {
        e.printStackTrace();
        fail("Unexpected exception");
    }
}

Example 55 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AvroAddStoreTest method testAddAvroSchema.

@Test
public void testAddAvroSchema() throws Exception {
    // backwards incompatible schema should fail
    try {
        logger.info("Now inserting stores with non backward compatible schema. Should see exception");
        adminClient.storeMgmtOps.addStore(new StoreDefinitionsMapper().readStore(new StringReader(storeXmlWithBackwardIncompatibleSchema)));
        Assert.fail("Did not throw exception");
    } catch (VoldemortException e) {
    }
    // invalid key schema should fail
    try {
        logger.info("Now inserting stores with int32 avro key. Should see exception");
        adminClient.storeMgmtOps.addStore(new StoreDefinitionsMapper().readStore(new StringReader(storeXmlWithInvalidAvroKeySchema)));
        Assert.fail("Did not throw exception for invalid key schema");
    } catch (VoldemortException e) {
        logger.error("As expected", e);
    }
    // invalid value schema should fail
    try {
        logger.info("Now inserting stores with html characters in avro value schema");
        adminClient.storeMgmtOps.addStore(new StoreDefinitionsMapper().readStore(new StringReader(storeXmlWithInvalidAvroValueSchema)));
        Assert.fail("Did not throw exception for invalid value schema");
    } catch (VoldemortException e) {
        logger.error("As expected", e);
    }
    for (VoldemortServer vs : vservers.values()) {
        assertNull(vs.getStoreRepository().getLocalStore("test"));
    }
    logger.info("Now inserting stores with backward compatible schema. Should not see exception");
    adminClient.storeMgmtOps.addStore(new StoreDefinitionsMapper().readStore(new StringReader(storeXmlWithBackwardCompatibleSchema)));
    for (VoldemortServer vs : vservers.values()) {
        assertNotNull(vs.getStoreRepository().getLocalStore("test"));
    }
}