Search in sources :

Example 26 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class ClusterTestUtils method getZZZ111StoreDefs.

public static List<StoreDefinition> getZZZ111StoreDefs(String storageType) {
    List<StoreDefinition> storeDefs = new LinkedList<StoreDefinition>();
    HashMap<Integer, Integer> zoneRep111 = new HashMap<Integer, Integer>();
    zoneRep111.put(0, 1);
    zoneRep111.put(1, 1);
    zoneRep111.put(2, 1);
    StoreDefinition storeDef111 = new StoreDefinitionBuilder().setName("ZZ111").setType(storageType).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setReplicationFactor(3).setZoneReplicationFactor(zoneRep111).setRequiredReads(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).build();
    storeDefs.add(storeDef111);
    return storeDefs;
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) HashMap(java.util.HashMap) StoreDefinition(voldemort.store.StoreDefinition) LinkedList(java.util.LinkedList) SerializerDefinition(voldemort.serialization.SerializerDefinition)

Example 27 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class ServerTestUtils method getStoreDefs.

public static List<StoreDefinition> getStoreDefs(int numStores) {
    List<StoreDefinition> defs = new ArrayList<StoreDefinition>();
    SerializerDefinition serDef = new SerializerDefinition("string");
    for (int i = 0; i < numStores; i++) defs.add(new StoreDefinitionBuilder().setName("test" + i).setType(InMemoryStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.SERVER).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build());
    return defs;
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) StoreDefinition(voldemort.store.StoreDefinition) ArrayList(java.util.ArrayList) SerializerDefinition(voldemort.serialization.SerializerDefinition)

Example 28 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class StoreDefinitionsMapper method readView.

private StoreDefinition readView(Element store, List<StoreDefinition> stores) {
    String name = store.getChildText(STORE_NAME_ELMT);
    String targetName = store.getChildText(VIEW_TARGET_ELMT);
    String description = store.getChildText(STORE_DESCRIPTION_ELMT);
    String ownerText = store.getChildText(STORE_OWNERS_ELMT);
    List<String> owners = Lists.newArrayList();
    if (ownerText != null) {
        for (String owner : Utils.COMMA_SEP.split(ownerText.trim())) if (owner.trim().length() > 0)
            owners.add(owner);
    }
    StoreDefinition target = StoreUtils.getStoreDef(stores, targetName);
    if (target == null)
        throw new MappingException("View \"" + name + "\" has target store \"" + targetName + "\" but no such store exists");
    int requiredReads = getChildWithDefault(store, STORE_REQUIRED_READS_ELMT, target.getRequiredReads());
    int preferredReads = getChildWithDefault(store, STORE_PREFERRED_READS_ELMT, target.getRequiredReads());
    int requiredWrites = getChildWithDefault(store, STORE_REQUIRED_WRITES_ELMT, target.getRequiredReads());
    int preferredWrites = getChildWithDefault(store, STORE_PREFERRED_WRITES_ELMT, target.getRequiredReads());
    Integer zoneCountReads = getChildWithDefault(store, STORE_ZONE_COUNT_READS, target.getZoneCountReads());
    Integer zoneCountWrites = getChildWithDefault(store, STORE_ZONE_COUNT_WRITES, target.getZoneCountWrites());
    String viewSerializerFactoryName = null;
    if (store.getChildText(VIEW_SERIALIZER_FACTORY_ELMT) != null) {
        viewSerializerFactoryName = store.getChild(VIEW_SERIALIZER_FACTORY_ELMT).getText();
    }
    SerializerDefinition keySerializer = target.getKeySerializer();
    SerializerDefinition valueSerializer = target.getValueSerializer();
    if (store.getChild(STORE_VALUE_SERIALIZER_ELMT) != null)
        valueSerializer = readSerializer(store.getChild(STORE_VALUE_SERIALIZER_ELMT));
    SerializerDefinition transformSerializer = target.getTransformsSerializer();
    if (store.getChild(STORE_TRANSFORM_SERIALIZER_ELMT) != null)
        transformSerializer = readSerializer(store.getChild(STORE_TRANSFORM_SERIALIZER_ELMT));
    RoutingTier routingTier = null;
    if (store.getChildText(STORE_ROUTING_TIER_ELMT) != null) {
        routingTier = RoutingTier.fromDisplay(store.getChildText(STORE_ROUTING_TIER_ELMT));
    } else {
        routingTier = target.getRoutingPolicy();
    }
    String viewClass = store.getChildText(VIEW_TRANS_ELMT);
    return new StoreDefinitionBuilder().setName(name).setViewOf(targetName).setType(ViewStorageConfiguration.TYPE_NAME).setDescription(description).setOwners(owners).setRoutingPolicy(routingTier).setRoutingStrategyType(target.getRoutingStrategyType()).setKeySerializer(keySerializer).setValueSerializer(valueSerializer).setTransformsSerializer(transformSerializer).setReplicationFactor(target.getReplicationFactor()).setZoneReplicationFactor(target.getZoneReplicationFactor()).setPreferredReads(preferredReads).setRequiredReads(requiredReads).setPreferredWrites(preferredWrites).setRequiredWrites(requiredWrites).setZoneCountReads(zoneCountReads).setZoneCountWrites(zoneCountWrites).setView(viewClass).setSerializerFactory(viewSerializerFactoryName).build();
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) RoutingTier(voldemort.client.RoutingTier) StoreDefinition(voldemort.store.StoreDefinition) SerializerDefinition(voldemort.serialization.SerializerDefinition)

Example 29 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class StoreDefinitionsMapper method readStore.

@SuppressWarnings("unchecked")
private static StoreDefinition readStore(Element store) {
    String name = store.getChildText(STORE_NAME_ELMT);
    String storeType = store.getChildText(STORE_PERSISTENCE_ELMT);
    String description = store.getChildText(STORE_DESCRIPTION_ELMT);
    String ownerText = store.getChildText(STORE_OWNERS_ELMT);
    List<String> owners = Lists.newArrayList();
    if (ownerText != null) {
        for (String owner : Utils.COMMA_SEP.split(ownerText.trim())) if (owner.trim().length() > 0)
            owners.add(owner);
    }
    int replicationFactor = Integer.parseInt(store.getChildText(STORE_REPLICATION_FACTOR_ELMT));
    HashMap<Integer, Integer> zoneReplicationFactor = null;
    Element zoneReplicationFactorNode = store.getChild(STORE_ZONE_REPLICATION_FACTOR_ELMT);
    if (zoneReplicationFactorNode != null) {
        zoneReplicationFactor = new HashMap<Integer, Integer>();
        for (Element node : (List<Element>) zoneReplicationFactorNode.getChildren(STORE_REPLICATION_FACTOR_ELMT)) {
            int zone = Integer.parseInt(node.getAttribute(STORE_ZONE_ID_ELMT).getValue());
            int repFactor = Integer.parseInt(node.getText());
            zoneReplicationFactor.put(zone, repFactor);
        }
    }
    String zoneCountReadsStr = store.getChildText(STORE_ZONE_COUNT_READS);
    Integer zoneCountReads = null;
    if (zoneCountReadsStr != null)
        zoneCountReads = Integer.parseInt(zoneCountReadsStr);
    String zoneCountWritesStr = store.getChildText(STORE_ZONE_COUNT_WRITES);
    Integer zoneCountWrites = null;
    if (zoneCountWritesStr != null)
        zoneCountWrites = Integer.parseInt(zoneCountWritesStr);
    int requiredReads = Integer.parseInt(store.getChildText(STORE_REQUIRED_READS_ELMT));
    int requiredWrites = Integer.parseInt(store.getChildText(STORE_REQUIRED_WRITES_ELMT));
    String preferredReadsStr = store.getChildText(STORE_PREFERRED_READS_ELMT);
    Integer preferredReads = null;
    if (preferredReadsStr != null)
        preferredReads = Integer.parseInt(preferredReadsStr);
    String preferredWritesStr = store.getChildText(STORE_PREFERRED_WRITES_ELMT);
    Integer preferredWrites = null;
    if (preferredWritesStr != null)
        preferredWrites = Integer.parseInt(preferredWritesStr);
    SerializerDefinition keySerializer = readSerializer(store.getChild(STORE_KEY_SERIALIZER_ELMT));
    if (keySerializer.getAllSchemaInfoVersions().size() > 1)
        throw new MappingException("Only a single schema is allowed for the store key.");
    SerializerDefinition valueSerializer = readSerializer(store.getChild(STORE_VALUE_SERIALIZER_ELMT));
    RoutingTier routingTier = RoutingTier.fromDisplay(store.getChildText(STORE_ROUTING_TIER_ELMT));
    String routingStrategyType = (null != store.getChildText(STORE_ROUTING_STRATEGY)) ? store.getChildText(STORE_ROUTING_STRATEGY) : RoutingStrategyType.CONSISTENT_STRATEGY;
    Element retention = store.getChild(STORE_RETENTION_POLICY_ELMT);
    Integer retentionPolicyDays = null;
    Integer retentionThrottleRate = null;
    Integer retentionFreqDays = null;
    if (retention != null) {
        int retentionDays = Integer.parseInt(retention.getText());
        if (retentionDays > 0) {
            retentionPolicyDays = retentionDays;
            Element throttleRate = store.getChild(STORE_RETENTION_SCAN_THROTTLE_RATE_ELMT);
            if (throttleRate != null)
                retentionThrottleRate = Integer.parseInt(throttleRate.getText());
            Element retentionFreqDaysElement = store.getChild(STORE_RETENTION_FREQ_ELMT);
            if (retentionFreqDaysElement != null)
                retentionFreqDays = Integer.parseInt(retentionFreqDaysElement.getText());
        } else {
            logger.error("Invalid retention policy days set. Should be greater than zero. ignoring value " + retentionDays);
        }
    }
    if (routingStrategyType.compareTo(RoutingStrategyType.ZONE_STRATEGY) == 0 && !SystemStoreConstants.isSystemStore(name)) {
        if (zoneCountReads == null || zoneCountWrites == null || zoneReplicationFactor == null) {
            throw new MappingException("Have not set one of the following correctly for store '" + name + "' - " + STORE_ZONE_COUNT_READS + ", " + STORE_ZONE_COUNT_WRITES + ", " + STORE_ZONE_REPLICATION_FACTOR_ELMT);
        }
    }
    HintedHandoffStrategyType hintedHandoffStrategy = null;
    if (store.getChildText(HINTED_HANDOFF_STRATEGY) != null)
        hintedHandoffStrategy = HintedHandoffStrategyType.fromDisplay(store.getChildText(HINTED_HANDOFF_STRATEGY));
    String hintPrefListSizeStr = store.getChildText(HINT_PREFLIST_SIZE);
    Integer hintPrefListSize = (null != hintPrefListSizeStr) ? Integer.parseInt(hintPrefListSizeStr) : null;
    String memoryFootprintStr = store.getChildText(STORE_MEMORY_FOOTPRINT);
    long memoryFootprintMB = 0;
    if (memoryFootprintStr != null)
        memoryFootprintMB = Long.parseLong(memoryFootprintStr);
    return new StoreDefinitionBuilder().setName(name).setType(storeType).setDescription(description).setOwners(owners).setKeySerializer(keySerializer).setValueSerializer(valueSerializer).setRoutingPolicy(routingTier).setRoutingStrategyType(routingStrategyType).setReplicationFactor(replicationFactor).setPreferredReads(preferredReads).setRequiredReads(requiredReads).setPreferredWrites(preferredWrites).setRequiredWrites(requiredWrites).setRetentionPeriodDays(retentionPolicyDays).setRetentionScanThrottleRate(retentionThrottleRate).setRetentionFrequencyDays(retentionFreqDays).setZoneReplicationFactor(zoneReplicationFactor).setZoneCountReads(zoneCountReads).setZoneCountWrites(zoneCountWrites).setHintedHandoffStrategy(hintedHandoffStrategy).setHintPrefListSize(hintPrefListSize).setMemoryFootprintMB(memoryFootprintMB).build();
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) Element(org.jdom.Element) HintedHandoffStrategyType(voldemort.store.slop.strategy.HintedHandoffStrategyType) RoutingTier(voldemort.client.RoutingTier) ArrayList(java.util.ArrayList) List(java.util.List) SerializerDefinition(voldemort.serialization.SerializerDefinition)

Example 30 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class RebalanceUtils method dropZone.

/**
 * Similar to {@link RebalanceUtils#vacateZone(Cluster, int)}, takes the
 * current store definitions in the cluster and creates store definitions
 * with the specified zone effectively dropped.
 *
 * In order to drop a zone, we adjust the total replication factor and
 * remove zone replication factor for the dropped zone
 *
 * @param currentStoreDefs
 * @param dropZoneId
 * @return the adjusted list of store definitions
 */
public static List<StoreDefinition> dropZone(List<StoreDefinition> currentStoreDefs, int dropZoneId) {
    List<StoreDefinition> adjustedStoreDefList = new ArrayList<StoreDefinition>();
    for (StoreDefinition storeDef : currentStoreDefs) {
        HashMap<Integer, Integer> zoneRepFactorMap = storeDef.getZoneReplicationFactor();
        if (!zoneRepFactorMap.containsKey(dropZoneId)) {
            throw new VoldemortException("Store " + storeDef.getName() + " does not have replication factor for zone " + dropZoneId);
        }
        StoreDefinitionBuilder adjustedStoreDefBuilder = StoreDefinitionUtils.getBuilderForStoreDef(storeDef);
        if (!storeDef.hasPreferredReads()) {
            adjustedStoreDefBuilder.setPreferredReads(null);
        }
        if (!storeDef.hasPreferredWrites()) {
            adjustedStoreDefBuilder.setPreferredWrites(null);
        }
        // Copy all zone replication factor entries except for dropped zone
        HashMap<Integer, Integer> adjustedZoneRepFactorMap = new HashMap<Integer, Integer>();
        for (Integer zoneId : zoneRepFactorMap.keySet()) {
            if (zoneId != dropZoneId) {
                adjustedZoneRepFactorMap.put(zoneId, zoneRepFactorMap.get(zoneId));
            }
        }
        adjustedStoreDefBuilder.setZoneReplicationFactor(adjustedZoneRepFactorMap);
        // adjust the replication factor
        int zoneRepFactor = zoneRepFactorMap.get(dropZoneId);
        adjustedStoreDefBuilder.setReplicationFactor(adjustedStoreDefBuilder.getReplicationFactor() - zoneRepFactor);
        adjustedStoreDefList.add(adjustedStoreDefBuilder.build());
    }
    return adjustedStoreDefList;
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) HashMap(java.util.HashMap) StoreDefinition(voldemort.store.StoreDefinition) ArrayList(java.util.ArrayList) VoldemortException(voldemort.VoldemortException)

Aggregations

StoreDefinitionBuilder (voldemort.store.StoreDefinitionBuilder)57 SerializerDefinition (voldemort.serialization.SerializerDefinition)46 StoreDefinition (voldemort.store.StoreDefinition)42 HashMap (java.util.HashMap)28 ArrayList (java.util.ArrayList)27 Test (org.junit.Test)23 File (java.io.File)17 ByteArray (voldemort.utils.ByteArray)16 Cluster (voldemort.cluster.Cluster)13 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)12 LinkedList (java.util.LinkedList)10 List (java.util.List)9 VoldemortException (voldemort.VoldemortException)9 Node (voldemort.cluster.Node)8 IOException (java.io.IOException)7 Before (org.junit.Before)6 VoldemortTestConstants.getNineNodeCluster (voldemort.VoldemortTestConstants.getNineNodeCluster)6 AdminClient (voldemort.client.protocol.admin.AdminClient)6 Store (voldemort.store.Store)6 FileWriter (java.io.FileWriter)5