Examples with StoreDefinition voldemort.store.StoreDefinition used on opensource projects

Example 76 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RepartitionerCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    // Required args
    String currentClusterXML = (String) options.valueOf("current-cluster");
    String currentStoresXML = (String) options.valueOf("current-stores");
    String interimClusterXML = new String(currentClusterXML);
    if (options.has("interim-cluster")) {
        interimClusterXML = (String) options.valueOf("interim-cluster");
    }
    String finalStoresXML = new String(currentStoresXML);
    if (options.has("final-stores")) {
        finalStoresXML = (String) options.valueOf("final-stores");
    }
    Cluster currentCluster = new ClusterMapper().readCluster(new File(currentClusterXML));
    List<StoreDefinition> currentStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(currentStoresXML));
    RebalanceUtils.validateClusterStores(currentCluster, currentStoreDefs);
    Cluster interimCluster = new ClusterMapper().readCluster(new File(interimClusterXML));
    List<StoreDefinition> finalStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(finalStoresXML));
    RebalanceUtils.validateClusterStores(interimCluster, finalStoreDefs);
    RebalanceUtils.validateCurrentInterimCluster(currentCluster, interimCluster);
    // Optional administrivia args
    int attempts = CmdUtils.valueOf(options, "attempts", Repartitioner.DEFAULT_REPARTITION_ATTEMPTS);
    String outputDir = null;
    if (options.has("output-dir")) {
        outputDir = (String) options.valueOf("output-dir");
    }
    // Optional repartitioning args
    boolean disableNodeBalancing = options.has("disable-node-balancing");
    boolean disableZoneBalancing = options.has("disable-zone-balancing");
    boolean enableRandomSwaps = options.has("enable-random-swaps");
    int randomSwapAttempts = CmdUtils.valueOf(options, "random-swap-attempts", Repartitioner.DEFAULT_RANDOM_SWAP_ATTEMPTS);
    int randomSwapSuccesses = CmdUtils.valueOf(options, "random-swap-successes", Repartitioner.DEFAULT_RANDOM_SWAP_SUCCESSES);
    List<Integer> randomSwapZoneIds = CmdUtils.valuesOf(options, "random-swap-zoneids", Repartitioner.DEFAULT_RANDOM_SWAP_ZONE_IDS);
    boolean enableGreedySwaps = options.has("enable-greedy-swaps");
    int greedySwapAttempts = CmdUtils.valueOf(options, "greedy-swap-attempts", Repartitioner.DEFAULT_GREEDY_SWAP_ATTEMPTS);
    int greedyMaxPartitionsPerNode = CmdUtils.valueOf(options, "greedy-max-partitions-per-node", Repartitioner.DEFAULT_GREEDY_MAX_PARTITIONS_PER_NODE);
    int greedyMaxPartitionsPerZone = CmdUtils.valueOf(options, "greedy-max-partitions-per-zone", Repartitioner.DEFAULT_GREEDY_MAX_PARTITIONS_PER_ZONE);
    List<Integer> greedySwapZoneIds = CmdUtils.valuesOf(options, "greedy-swap-zoneids", Repartitioner.DEFAULT_GREEDY_SWAP_ZONE_IDS);
    int maxContiguousPartitionsPerZone = CmdUtils.valueOf(options, "max-contiguous-partitions", Repartitioner.DEFAULT_MAX_CONTIGUOUS_PARTITIONS);
    // Sanity check optional repartitioning args
    if (disableNodeBalancing && !enableRandomSwaps && !enableGreedySwaps && maxContiguousPartitionsPerZone == 0) {
        printUsageAndDie("Did not enable any forms for repartitioning.");
    }
    if ((options.has("random-swap-attempts") || options.has("random-swap-successes")) && !enableRandomSwaps) {
        printUsageAndDie("Provided arguments for generate random swaps but did not enable the feature");
    }
    if ((options.has("greedy-swap-attempts") || options.has("greedy-max-partitions-per-node") || options.has("greedy-max-partitions-per-zone")) && !enableGreedySwaps) {
        printUsageAndDie("Provided arguments for generate greedy swaps but did not enable the feature");
    }
    Repartitioner.repartition(currentCluster, currentStoreDefs, interimCluster, finalStoreDefs, outputDir, attempts, disableNodeBalancing, disableZoneBalancing, enableRandomSwaps, randomSwapAttempts, randomSwapSuccesses, randomSwapZoneIds, enableGreedySwaps, greedySwapAttempts, greedyMaxPartitionsPerNode, greedyMaxPartitionsPerZone, greedySwapZoneIds, maxContiguousPartitionsPerZone);
}

Example 77 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class ZoneCLipperTest method testDropZoneIdWorks.

@Test
public void testDropZoneIdWorks() {
    // Create a list of current partition ids. We will use this set to
    // compare partitions ids with the interim cluster
    Set<Integer> originalPartitions = new HashSet<Integer>();
    for (Integer zoneId : initialCluster.getZoneIds()) {
        originalPartitions.addAll(initialCluster.getPartitionIdsInZone(zoneId));
    }
    // Get an intermediate cluster where partitions that belong to the zone
    // that is being dropped have been moved to the existing zones
    Cluster interimCluster = RebalanceUtils.vacateZone(initialCluster, dropZoneId);
    // Make sure that the intermediate cluster should have same number of
    // partitions
    RebalanceUtils.validateClusterPartitionCounts(initialCluster, interimCluster);
    // Make sure that the intermediate cluster should have same number of
    // nodes
    RebalanceUtils.validateClusterNodeCounts(initialCluster, interimCluster);
    // Make sure that the intermediate cluster doesn't have any partitions in
    // the dropped zone
    assertTrue("Zone being dropped has partitions. ZoneClipper didn't work properly", interimCluster.getPartitionIdsInZone(dropZoneId).isEmpty());
    // Make sure that the nodes being dropped don't host any partitions
    for (Integer nodeId : interimCluster.getNodeIdsInZone(dropZoneId)) {
        assertTrue("Nodes in the zone being dropped don't have empty partitions list", interimCluster.getNodeById(nodeId).getPartitionIds().isEmpty());
    }
    Set<Integer> finalPartitions = new HashSet<Integer>();
    for (Integer zoneId : interimCluster.getZoneIds()) {
        finalPartitions.addAll(interimCluster.getPartitionIdsInZone(zoneId));
    }
    // Compare to original partition ids list
    assertTrue("Original and interm partition ids don't match", originalPartitions.equals(finalPartitions));
    // Make sure that there is no data movement
    RebalancePlan rebalancePlan = ClusterTestUtils.makePlan(initialCluster, storeDefs, interimCluster, storeDefs);
    // Make sure we have a plan
    assertEquals(rebalancePlan.getPlan().size(), 1);
    // Make sure there is no cross zones between zones in the plan
    assertEquals(rebalancePlan.getPartitionStoresMovedXZone(), 0);
    // Make sure there is no data movement between nodes
    assertEquals(rebalancePlan.getPartitionStoresMoved(), 0);
    for (Integer nodeId : interimCluster.getNodeIds()) {
        Set<Integer> remainingNodes = Sets.symmetricDifference(interimCluster.getNodeIds(), Sets.newHashSet(nodeId));
        for (Integer otherNodeId : remainingNodes) {
            assertTrue("Something went wrong as there is data movement between nodes", rebalancePlan.getNodeMoveMap().get(nodeId, otherNodeId) == 0);
        }
    }
    // Also get the adjusted store definitions, with the zone dropped
    Cluster finalCluster = RebalanceUtils.dropZone(interimCluster, dropZoneId);
    List<StoreDefinition> clippedStoreDefs = RebalanceUtils.dropZone(storeDefs, dropZoneId);
    for (StoreDefinition storeDef : clippedStoreDefs) {
        StoreDefinition orgStoreDef = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, storeDef.getName());
        assertFalse("Clipped storedef has replication for dropped zone", storeDef.getZoneReplicationFactor().containsKey(dropZoneId));
        assertEquals("Clipped storedef has incorrect number of zones", initialCluster.getZones().size() - 1, storeDef.getZoneReplicationFactor().size());
        assertEquals("Clipped storedef has incorrect total repfactor", orgStoreDef.getReplicationFactor() - orgStoreDef.getZoneReplicationFactor().get(dropZoneId), storeDef.getReplicationFactor());
    }
    // Confirm that we would not route to any of the dropped nodes for any
    // store.
    Set<Integer> dropNodes = interimCluster.getNodeIdsInZone(dropZoneId);
    for (StoreDefinition storeDef : clippedStoreDefs) {
        StoreRoutingPlan routingPlan = new StoreRoutingPlan(finalCluster, storeDef);
        Node[] partitionToNode = finalCluster.getPartitionIdToNodeArray();
        for (int p = 0; p < partitionToNode.length; p++) {
            List<Integer> replicaNodes = routingPlan.getReplicationNodeList(p);
            assertFalse("Should not be routing to any dropped nodes", replicaNodes.removeAll(dropNodes));
        }
    }
}

Example 78 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetaOperationsTest method testMetaGet.

@Test
public void testMetaGet() throws Exception {
    Integer nodeId = 0;
    String tempDir = System.getProperty("java.io.tmpdir") + "temp" + System.currentTimeMillis();
    // get metadata and write to files
    AdminCommand.executeCommand(new String[] { "meta", "get", MetadataStore.CLUSTER_KEY + "," + MetadataStore.STORES_KEY, "-u", bsURL, "-n", nodeId.toString(), "-d", tempDir });
    // load metadata files
    String clusterFile = tempDir + "/" + MetadataStore.CLUSTER_KEY + "_" + nodeId;
    String storesFile = tempDir + "/" + MetadataStore.STORES_KEY + "_" + nodeId;
    assertTrue(Utils.isReadableFile(clusterFile));
    assertTrue(Utils.isReadableFile(storesFile));
    ClusterMapper clusterMapper = new ClusterMapper();
    StoreDefinitionsMapper storeDefsMapper = new StoreDefinitionsMapper();
    Cluster newCluster = clusterMapper.readCluster(new File(clusterFile));
    List<StoreDefinition> newStores = storeDefsMapper.readStoreList(new File(storesFile));
    // compare metadata objects
    assertTrue(newCluster.equals(cluster));
    assertTrue(newStores.size() == stores.size());
    for (StoreDefinition store : stores) {
        assertTrue(newStores.contains(store));
    }
}

Example 79 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetaOperationsTest method testMetaSet.

@Test
public void testMetaSet() throws Exception {
    // check if all old metadata are the same
    Set<Object> storesValues = Sets.newHashSet();
    for (Node node : adminClient.getAdminClientCluster().getNodes()) {
        Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), MetadataStore.STORES_KEY);
        assertTrue(versioned != null && versioned.getValue() != null);
        storesValues.add(new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue())));
    }
    assertTrue(storesValues.size() == 1);
    // create new metadata object
    final String newStoreXMLFilePrefix = "updated.stores";
    final String newStoreXMLFileSuffix = "xml";
    List<StoreDefinition> newStoresToUpload = new ArrayList<StoreDefinition>();
    for (StoreDefinition storeDef : stores) {
        StoreDefinitionBuilder sb = AdminToolTestUtils.storeDefToBuilder(storeDef);
        sb.setRoutingPolicy(sb.getRoutingPolicy().equals(RoutingTier.CLIENT) ? RoutingTier.SERVER : RoutingTier.CLIENT);
        sb.setPreferredReads(sb.getPreferredReads() == null ? sb.getRequiredReads() : null);
        newStoresToUpload.add(sb.build());
    }
    // create file of new metadata
    File newStoresXMLFolder = TestUtils.createTempDir();
    File newStoreXMLFile = File.createTempFile(newStoreXMLFilePrefix, newStoreXMLFileSuffix, newStoresXMLFolder);
    FileWriter fwriter = new FileWriter(newStoreXMLFile);
    fwriter.write(new StoreDefinitionsMapper().writeStoreList(newStoresToUpload));
    fwriter.close();
    // set new metadata from file
    AdminCommand.executeCommand(new String[] { "meta", "set", MetadataStore.STORES_KEY + "=" + newStoreXMLFile.getAbsolutePath(), "-u", bsURL, "--confirm" });
    // fetch new metadata from node
    List<StoreDefinition> newStoresToVerify = adminClient.metadataMgmtOps.getRemoteStoreDefList(adminClient.getAdminClientCluster().getNodes().iterator().next().getId()).getValue();
    System.out.println("Uploades stores" + new StoreDefinitionsMapper().writeStoreList(newStoresToUpload));
    System.out.println("Retrieved stores" + new StoreDefinitionsMapper().writeStoreList(newStoresToVerify));
    // check if new metadata is the same as the source
    assertTrue("Old store should not equal to new stores", !newStoresToVerify.equals(stores));
    assertTrue("Uploaded stores and retrieved stores are different", newStoresToVerify.equals(newStoresToUpload));
    // check if all new metadata are the same
    storesValues.clear();
    for (Node node : adminClient.getAdminClientCluster().getNodes()) {
        Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), MetadataStore.STORES_KEY);
        assertTrue(versioned != null && versioned.getValue() != null);
        storesValues.add(new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue())));
    }
    System.out.println(storesValues.size());
    assertTrue(storesValues.size() == 1);
}

Example 80 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class SetMetadataTest method testSetMetadataStoreXmlVerifyVersionUpdate.

@Test
public void testSetMetadataStoreXmlVerifyVersionUpdate() throws Exception {
    // setup new stores xml
    final String newStoreXMLFilePrefix = "updated.stores";
    final String newStoreXMLFileSuffix = "xml";
    List<StoreDefinition> newStores = new ArrayList<StoreDefinition>();
    for (StoreDefinition storeDef : ClusterTestUtils.getZZZStoreDefsInMemory()) {
        StoreDefinitionBuilder sb = storeDefToBuilder(storeDef);
        Map<Integer, Integer> zrf = sb.getZoneReplicationFactor();
        Integer zone0RepFactor = zrf.get(0);
        zrf.remove(0);
        sb.setReplicationFactor(sb.getReplicationFactor() - zone0RepFactor);
        newStores.add(sb.build());
    }
    File newStoresXMLFolder = TestUtils.createTempDir();
    File newStoreXMLFile = File.createTempFile(newStoreXMLFilePrefix, newStoreXMLFileSuffix, newStoresXMLFolder);
    FileWriter fwriter = new FileWriter(newStoreXMLFile);
    fwriter.write(new StoreDefinitionsMapper().writeStoreList(newStores));
    fwriter.close();
    // check version
    String sysStoreName = SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name();
    ByteArray metadataKey = new ByteArray(ByteUtils.getBytes(SystemStoreConstants.VERSIONS_METADATA_KEY, "utf-8"));
    for (VoldemortServer vs : vservers.values()) {
        List<Versioned<byte[]>> result = vs.getStoreRepository().getLocalStore(sysStoreName).get(metadataKey, null);
        String versionInfo = new String(result.get(0).getValue());
        System.out.format("[INITIAL]Version values on node [%d] is: \n %s\n", vs.getIdentityNode().getId(), versionInfo);
        Properties props = new Properties();
        props.load(new ByteArrayInputStream(versionInfo.getBytes()));
        for (StoreDefinition sd : oldStores) {
            if (!props.getProperty(sd.getName()).equals("0")) {
                Assert.fail("Initial version of key [" + sd.getName() + "] on node [" + vs.getIdentityNode().getId() + "] is expected to be 0 but not");
            }
        }
        if (!props.getProperty("cluster.xml").equals("0")) {
            Assert.fail("Final version of key [stores.xml] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
        }
    }
    // update the stores xml
    VoldemortAdminTool.main(new String[] { "--auto", "--set-metadata", MetadataStore.STORES_KEY, "--set-metadata-value", newStoreXMLFile.getAbsolutePath(), "--url", bsURL });
    // properly
    try {
        Thread.sleep(10000);
    } catch (Exception e) {
    }
    // check version
    for (VoldemortServer vs : vservers.values()) {
        List<Versioned<byte[]>> result = vs.getStoreRepository().getLocalStore(sysStoreName).get(metadataKey, null);
        String versionInfo = new String(result.get(0).getValue());
        System.out.format("[FINAL]Version values on node [%d] is: \n %s\n", vs.getIdentityNode().getId(), versionInfo);
        Properties props = new Properties();
        props.load(new ByteArrayInputStream(versionInfo.getBytes()));
        for (StoreDefinition sd : oldStores) {
            if (!(Long.parseLong(props.getProperty(sd.getName())) > 0)) {
                Assert.fail("Final version of key [" + sd.getName() + "] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
            }
        }
        if (!(Long.parseLong(props.getProperty("stores.xml")) > 0)) {
            Assert.fail("Final version of key [stores.xml] on node [" + vs.getIdentityNode().getId() + "] is expected to greater than 0 but not");
        }
    }
}