Examples with StoreDefinitionsMapper voldemort.xml.StoreDefinitionsMapper used on opensource projects

Example 71 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ReplaceNodeCLI method modifyTopology.

private void modifyTopology() {
    List<StoreDefinition> newStoreDefinitions = this.newAdminClient.metadataMgmtOps.getRemoteStoreDefList().getValue();
    String updatedClusterXML = updateClusterXML();
    String storesXML = new StoreDefinitionsMapper().writeStoreList(storeDefinitions);
    List<Integer> newNodeIdAsList = new ArrayList<Integer>();
    newNodeIdAsList.add(newNodeId);
    newAdminClient.metadataMgmtOps.updateRemoteMetadata(newNodeIdAsList, MetadataStore.CLUSTER_KEY, updatedClusterXML);
    if (newStoreDefinitions.size() == 0) {
        for (StoreDefinition def : storeDefinitions) {
            newAdminClient.storeMgmtOps.addStore(def, newNodeIdAsList);
        }
    }
    newAdminClient.metadataMgmtOps.updateRemoteMetadata(newNodeIdAsList, MetadataStore.STORES_KEY, storesXML);
    newAdminClient.metadataMgmtOps.updateRemoteMetadata(newNodeIdAsList, MetadataStore.NODE_ID_KEY, Integer.toString(nodeId));
    List<Integer> oldNodeIds = new ArrayList<Integer>(cluster.getNodeIds());
    oldNodeIds.remove(nodeId);
    adminClient.metadataMgmtOps.updateRemoteMetadata(oldNodeIds, MetadataStore.CLUSTER_KEY, updatedClusterXML);
    init();
}

Example 72 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ValidateSchemaForAvroStores method validate.

public static void validate(File storesXMLFile) {
    StoreDefinitionsMapper storeDefsMapper = new StoreDefinitionsMapper();
    List<StoreDefinition> newStoreDefs = null;
    // parse the store definitions from the xml
    try {
        newStoreDefs = storeDefsMapper.readStoreList(storesXMLFile);
    } catch (Exception e) {
        e.printStackTrace();
        return;
    }
    // validate schema for each store as needed
    for (StoreDefinition storeDefinition : newStoreDefs) {
        try {
            StoreDefinitionUtils.validateSchemaAsNeeded(storeDefinition);
        } catch (VoldemortException ex) {
            logger.error("A VoldemortException occurred");
            ex.printStackTrace();
        // Continue with the loop so we catch all invalid schemas at
        // once.
        }
    }
}

Example 73 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ZoneClipperCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    int dropZoneId = CmdUtils.valueOf(options, "drop-zoneid", Zone.UNSET_ZONE_ID);
    String outputDir = null;
    if (options.has("output-dir")) {
        outputDir = (String) options.valueOf("output-dir");
    }
    /*
         * A. Generate the clipped cluster.xml
         */
    String initialClusterXML = (String) options.valueOf("current-cluster");
    Cluster initialCluster = new ClusterMapper().readCluster(new File(initialClusterXML));
    // Create a list of current partition ids. We will use this set to
    // compare partitions ids in final cluster
    Set<Integer> originalPartitions = new HashSet<Integer>();
    for (Integer zoneId : initialCluster.getZoneIds()) {
        originalPartitions.addAll(initialCluster.getPartitionIdsInZone(zoneId));
    }
    // Get an intermediate cluster where partitions that belong to the zone
    // that is being dropped have been moved to the existing zones
    Cluster intermediateCluster = RebalanceUtils.vacateZone(initialCluster, dropZoneId);
    Cluster finalCluster = RebalanceUtils.dropZone(intermediateCluster, dropZoneId);
    // Make sure everything is fine
    if (initialCluster.getNumberOfPartitions() != finalCluster.getNumberOfPartitions()) {
        logger.error("The number of partitions in the initial and the final cluster is not equal \n");
    }
    Set<Integer> finalPartitions = new HashSet<Integer>();
    for (Integer zoneId : finalCluster.getZoneIds()) {
        finalPartitions.addAll(finalCluster.getPartitionIdsInZone(zoneId));
    }
    // Compare to original partition ids list
    if (!originalPartitions.equals(finalPartitions)) {
        logger.error("The list of partition ids in the initial and the final cluster doesn't match \n ");
    }
    // Finally write the final cluster to a xml file
    RebalanceUtils.dumpClusterToFile(outputDir, RebalanceUtils.finalClusterFileName, finalCluster);
    /*
         * B. Generate the clipped stores.xml
         */
    logger.info("Generating the adjusted stores.xml..");
    String initialStoresXML = (String) options.valueOf("current-stores");
    List<StoreDefinition> initialStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(initialStoresXML));
    List<StoreDefinition> finalStoreDefs = RebalanceUtils.dropZone(initialStoreDefs, dropZoneId);
    RebalanceUtils.dumpStoreDefsToFile(outputDir, RebalanceUtils.finalStoresFileName, finalStoreDefs);
}

Example 74 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class MetadataStore method addStoreDefinition.

/**
     * Function to add a new Store to the Metadata store. This involves
     * 
     * 1. Create a new entry in the ConfigurationStorageEngine for STORES.
     * 
     * 2. Update the metadata cache.
     * 
     * 3. Re-create the 'stores.xml' key
     * 
     * @param storeDef defines the new store to be created
     */
public void addStoreDefinition(StoreDefinition storeDef) {
    // acquire write lock
    writeLock.lock();
    try {
        // Check if store already exists
        if (this.storeNames.contains(storeDef.getName())) {
            throw new VoldemortException("Store already exists !");
        }
        // Check for backwards compatibility
        StoreDefinitionUtils.validateSchemaAsNeeded(storeDef);
        // Otherwise add to the STORES directory
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        String storeDefStr = mapper.writeStore(storeDef);
        Versioned<String> versionedValueStr = new Versioned<String>(storeDefStr);
        this.storeDefinitionsStorageEngine.put(storeDef.getName(), versionedValueStr, null);
        // Update the metadata cache
        this.metadataCache.put(storeDef.getName(), new Versioned<Object>(storeDefStr));
        // Re-initialize the store definitions. This is primarily required
        // to re-create the value for key: 'stores.xml'. This is necessary
        // for backwards compatibility.
        initStoreDefinitions(null);
        updateRoutingStrategies(getCluster(), getStoreDefList());
    } finally {
        writeLock.unlock();
    }
}

Example 75 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class PartitionPrefixedBdbStorageEngineTest method testHashConsistencyAcrossRoutingStrategies.

@Test
public void testHashConsistencyAcrossRoutingStrategies() {
    // check that as long as the cluster.xml is the same, a key will hash to
    // the same partition, immaterial of whether it is zone or consistent
    // routing strategy
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getTwoStoresWithZonesXml()));
    StoreDefinition consistentStore = storeDefs.get(0);
    StoreDefinition zoneStore = storeDefs.get(1);
    assertEquals(consistentStore.getName(), "cstore");
    assertEquals(zoneStore.getName(), "zstore");
    Cluster cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    RoutingStrategy cStrategy = new RoutingStrategyFactory().updateRoutingStrategy(consistentStore, cluster);
    RoutingStrategy zStrategy = new RoutingStrategyFactory().updateRoutingStrategy(zoneStore, cluster);
    BdbStorageEngine cPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(consistentStore, cStrategy);
    BdbStorageEngine zPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(zoneStore, zStrategy);
    HashMap<ByteArray, byte[]> kvpairs = ServerTestUtils.createRandomKeyValuePairs(10000);
    for (ByteArray key : kvpairs.keySet()) {
        assertEquals(cStrategy.getPartitionList(key.get()).get(0), zStrategy.getPartitionList(key.get()).get(0));
        cPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
        zPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
    }
    for (ByteArray key : kvpairs.keySet()) {
        assertEquals("Values read back does not match up", 0, ByteUtils.compare(cPrefixedBdbStore.get(key, null).get(0).getValue(), zPrefixedBdbStore.get(key, null).get(0).getValue()));
    }
    cPrefixedBdbStore.close();
    zPrefixedBdbStore.close();
}