Search in sources :

Example 1 with StoragePolicyGroup

use of com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup in project SilverKing by Morgan-Stanley.

the class DependencyWatcher method build.

private void build(Map<String, Long> curBuild) {
    try {
        boolean buildOK;
        long ringConfigVersion;
        Log.warning("New build triggered");
        buildOK = false;
        ringConfigVersion = -1;
        try {
            if (TopoRingConstants.verbose) {
                System.out.println("Building tree");
            }
            RingTreeRecipe recipe;
            Topology topology;
            WeightSpecifications weightSpecs;
            ExclusionSet exclusionSet;
            ExclusionSet instanceExclusionSet;
            StoragePolicyGroup storagePolicyGroup;
            long topologyVersion;
            long weightsVersion;
            long exclusionVersion;
            long instanceExclusionVersion;
            long storagePolicyGroupVersion;
            RingConfiguration ringConfig;
            ZooKeeperExtended zk;
            HostGroupTable hostGroupTable;
            long hostGroupTableVersion;
            ExclusionSet mergedExclusionSet;
            zk = mc.getZooKeeper();
            topologyVersion = curBuild.get(mp.getTopologyPath());
            weightsVersion = curBuild.get(mp.getWeightsPath());
            exclusionVersion = curBuild.get(mp.getExclusionsPath());
            instanceExclusionVersion = curBuild.get(dhtMC.getMetaPaths().getInstanceExclusionsPath());
            storagePolicyGroupVersion = curBuild.get(mp.getStoragePolicyGroupPath());
            ringConfigVersion = curBuild.get(mp.getConfigPath());
            topology = new TopologyZK(cloudMC).readFromZK(topologyVersion, null);
            weightSpecs = new WeightsZK(mc).readFromZK(weightsVersion, null);
            exclusionSet = new ExclusionSet(new ServerSetExtensionZK(mc, mc.getMetaPaths().getExclusionsPath()).readFromZK(exclusionVersion, null));
            if (options.ignoreInstanceExclusions) {
                instanceExclusionSet = ExclusionSet.emptyExclusionSet(0);
            } else {
                try {
                    instanceExclusionSet = new ExclusionSet(new ServerSetExtensionZK(mc, dhtMC.getMetaPaths().getInstanceExclusionsPath()).readFromZK(instanceExclusionVersion, null));
                } catch (Exception e) {
                    Log.warning("No instance ExclusionSet found");
                    instanceExclusionSet = ExclusionSet.emptyExclusionSet(0);
                }
            }
            mergedExclusionSet = ExclusionSet.union(exclusionSet, instanceExclusionSet);
            storagePolicyGroup = new StoragePolicyGroupZK(mc).readFromZK(storagePolicyGroupVersion, null);
            ringConfig = new RingConfigurationZK(mc).readFromZK(ringConfigVersion, null);
            Log.warningf("ringConfiguration %s", ringConfig);
            hostGroupTableVersion = zk.getLatestVersion(cloudMC.getMetaPaths().getHostGroupPath());
            hostGroupTable = new HostGroupTableZK(cloudMC).readFromZK(hostGroupTableVersion, null);
            try {
                recipe = new RingTreeRecipe(topology, ringConfig.getRingParentName(), weightSpecs, mergedExclusionSet, storagePolicyGroup, ringConfig.getStoragePolicyName(), hostGroupTable, ringConfig.getHostGroups(), ringConfigVersion, DHTUtil.currentTimeMillis());
                Log.warning("Recipe.ringParent: " + recipe.ringParent);
            } catch (RuntimeException re) {
                re.printStackTrace(System.out);
                Log.warning("ringConfig: ", ringConfig + " " + re);
                Log.logErrorWarning(re);
                throw re;
            }
            RingTree ringTree;
            RingTree prevRingTree;
            long configInstanceVersion;
            String newInstancePath;
            configInstanceVersion = mc.getLatestConfigInstanceVersion(ringConfigVersion);
            if (configInstanceVersion >= 0 && !ignoreSource) {
                prevRingTree = SingleRingZK.readTree(mc, ringConfigVersion, configInstanceVersion);
            } else {
                prevRingTree = null;
            }
            if (prevRingTree == null || ignoreFeasibility || RingTreeBuilder.convergenceFeasible(prevRingTree, storagePolicyGroup, ringConfig.getStoragePolicyName(), ringConfig.getRingParentName(), exclusionSet)) {
                ringTree = RingTreeBuilder.create(recipe, prevRingTree);
                // ringTree = RingTreeBuilder.create(recipe, null); // for testing without movement reduction
                newInstancePath = mc.createConfigInstancePath(ringConfigVersion);
                SingleRingZK.writeTree(mc, topologyVersion, newInstancePath, ringTree);
                if (TopoRingConstants.verbose) {
                    System.out.println(ringTree);
                    System.out.println(ringConfigVersion);
                    System.out.println(configInstanceVersion);
                    System.out.println(topologyVersion);
                    System.out.println(newInstancePath);
                    System.out.println("Building complete");
                }
                buildOK = true;
            } else {
                Log.warning("Convergence is infeasible. A region in prevTree does not have a viable server.");
            }
        } catch (IOException ioe) {
            Log.logErrorWarning(ioe);
        } catch (KeeperException ke) {
            Log.logErrorWarning(ke);
        }
        if (exitAfterBuild) {
            if (buildOK) {
                try {
                    if (ringConfigVersion < 0) {
                        throw new RuntimeException("ringConfigVersion < 0");
                    }
                    setRing(ringConfigVersion);
                } catch (KeeperException ke) {
                    Log.logErrorWarning(ke);
                    buildOK = false;
                }
            }
            System.exit(buildOK ? 0 : -1);
        }
    } catch (RuntimeException re) {
        re.printStackTrace();
    } finally {
        Log.warning("Leaving build");
    }
}
Also used : RingConfigurationZK(com.ms.silverking.cloud.toporing.meta.RingConfigurationZK) HostGroupTable(com.ms.silverking.cloud.config.HostGroupTable) ZooKeeperExtended(com.ms.silverking.cloud.zookeeper.ZooKeeperExtended) Topology(com.ms.silverking.cloud.topology.Topology) IOException(java.io.IOException) StoragePolicyGroupZK(com.ms.silverking.cloud.toporing.meta.StoragePolicyGroupZK) StoragePolicyGroup(com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup) ServerSetExtensionZK(com.ms.silverking.cloud.meta.ServerSetExtensionZK) KeeperException(org.apache.zookeeper.KeeperException) IOException(java.io.IOException) CmdLineException(org.kohsuke.args4j.CmdLineException) WeightSpecifications(com.ms.silverking.cloud.toporing.meta.WeightSpecifications) TopologyZK(com.ms.silverking.cloud.topology.TopologyZK) RingConfiguration(com.ms.silverking.cloud.toporing.meta.RingConfiguration) NamedRingConfiguration(com.ms.silverking.cloud.toporing.meta.NamedRingConfiguration) WeightsZK(com.ms.silverking.cloud.toporing.meta.WeightsZK) ExclusionSet(com.ms.silverking.cloud.meta.ExclusionSet) HostGroupTableZK(com.ms.silverking.cloud.meta.HostGroupTableZK) KeeperException(org.apache.zookeeper.KeeperException)

Example 2 with StoragePolicyGroup

use of com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup in project SilverKing by Morgan-Stanley.

the class RingTreeBuilder method main.

public static void main(String[] args) {
    try {
        if (args.length != 8) {
            System.out.println("<topology> <weights> <exclusionList> <ringParentID> <storagePolicyGroup> <storagePolicyName> <HostGroupTableFile> <HostGroup,...>");
        } else {
            File topoFile;
            File weightsFile;
            File exclusionFile;
            String ringParentID;
            File storagePolicyGroupFile;
            String storagePolicyName;
            RingTree ringTree;
            Topology topology;
            StoragePolicyGroup storagePolicyGroup;
            HostGroupTable hostGroupTable;
            Set<String> hostGroups;
            topoFile = new File(args[0]);
            weightsFile = new File(args[1]);
            exclusionFile = new File(args[2]);
            ringParentID = args[3];
            storagePolicyGroupFile = new File(args[4]);
            storagePolicyName = args[5];
            topology = TopologyParser.parse(topoFile);
            storagePolicyGroup = new PolicyParser().parsePolicyGroup(storagePolicyGroupFile, VersionedDefinition.NO_VERSION);
            hostGroupTable = HostGroupTable.parse(args[6], VersionedDefinition.NO_VERSION);
            hostGroups = ImmutableSet.copyOf(args[7].split(","));
            ringTree = create(new RingTreeRecipe(topology, (Node) topology.getNodeByID(ringParentID), new WeightSpecifications(VersionedDefinition.NO_VERSION).parse(weightsFile), ExclusionSet.parse(exclusionFile), storagePolicyGroup, storagePolicyName, hostGroupTable, hostGroups, 0L, System.currentTimeMillis()));
            // TopoReplicationSpecification.parse(replicationFile)));
            System.out.println(ringTree);
            ringTree.test("10.188.1.1");
            ringTree.testDistance("10.188.1.1", "10.188.1.1");
            ringTree.testDistance("10.188.1.1", "10.188.1.2");
            ringTree.testDistance("10.188.1.1", "10.188.2.2");
            ringTree.testDistance("10.188.2.1", "10.188.2.2");
            ringTree.testDistance("10.188.2.2", "10.188.2.2");
            ringTree.testDistance("10.188.1.1", "blah");
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : PolicyParser(com.ms.silverking.cloud.storagepolicy.PolicyParser) WeightSpecifications(com.ms.silverking.cloud.toporing.meta.WeightSpecifications) HostGroupTable(com.ms.silverking.cloud.config.HostGroupTable) Topology(com.ms.silverking.cloud.topology.Topology) File(java.io.File) StoragePolicyGroup(com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup)

Example 3 with StoragePolicyGroup

use of com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup in project SilverKing by Morgan-Stanley.

the class StaticRingCreator method createStaticRing.

public static RingCreationResults createStaticRing(String ringName, ZooKeeperConfig zkConfig, Set<String> servers, int replication, UUIDBase myID) {
    try {
        MetaClient mc;
        com.ms.silverking.cloud.meta.MetaClient cloudMC;
        NamedRingConfiguration namedRingConfig;
        RingConfiguration ringConfig;
        RingTreeRecipe recipe;
        Topology topology;
        WeightSpecifications weightSpecs;
        ExclusionSet exclusionSet;
        StoragePolicyGroup storagePolicyGroup;
        HostGroupTable hostGroupTable;
        long ringConfigVersion;
        CloudConfiguration cloudConfig;
        String exclusionSpecsName;
        String hostGroupTableName;
        topology = StaticTopologyCreator.createTopology("topology." + myID, servers);
        exclusionSpecsName = "exclusionSpecs." + myID;
        hostGroupTableName = "hostGroupTable." + myID;
        cloudConfig = new CloudConfiguration(topology.getName(), exclusionSpecsName, hostGroupTableName);
        cloudMC = new com.ms.silverking.cloud.meta.MetaClient(cloudConfig, zkConfig);
        new TopologyZK(cloudMC).writeToZK(topology, null);
        new ExclusionZK(cloudMC).writeToZK(ExclusionSet.emptyExclusionSet(0));
        mc = new MetaClient(NamedRingConfiguration.emptyTemplate.ringName(ringName), zkConfig);
        ringConfig = new RingConfiguration(cloudConfig, "weightSpecsName", StaticTopologyCreator.parentID, SimpleStoragePolicyCreator.storagePolicyGroupName, SimpleStoragePolicyCreator.storagePolicyName, null);
        new RingConfigurationZK(mc).writeToZK(ringConfig, null);
        namedRingConfig = new NamedRingConfiguration(ringName, ringConfig);
        mc = new MetaClient(namedRingConfig, zkConfig);
        EmbeddedSK.setEmbeddedNamedRingConfiguration(ringName, namedRingConfig);
        Log.warning(zkConfig);
        ringConfigVersion = 0;
        weightSpecs = new WeightSpecifications(0);
        exclusionSet = ExclusionSet.emptyExclusionSet(0);
        storagePolicyGroup = SimpleStoragePolicyCreator.createStoragePolicyGroup(replication);
        new StoragePolicyZK(mc).writeToZK(storagePolicyGroup, null);
        hostGroupTable = HostGroupTable.createHostGroupTable(servers, hostGroupName);
        new HostGroupTableZK(cloudMC).writeToZK(hostGroupTable, null);
        recipe = new RingTreeRecipe(topology, StaticTopologyCreator.parentID, weightSpecs, exclusionSet, storagePolicyGroup, SimpleStoragePolicyCreator.storagePolicyName, hostGroupTable, ImmutableSet.of(hostGroupName), ringConfigVersion, DHTUtil.currentTimeMillis());
        Log.warning("Recipe.ringParent: " + recipe.ringParent);
        RingTree ringTree;
        String newInstancePath;
        ringTree = RingTreeBuilder.create(recipe, null);
        newInstancePath = mc.createConfigInstancePath(ringConfigVersion);
        SingleRingZK.writeTree(mc, topology, newInstancePath, ringTree);
        if (TopoRingConstants.verbose) {
            System.out.println(ringTree);
            System.out.println("Building complete");
        }
        return new RingCreationResults(exclusionSpecsName, hostGroupTableName, hostGroupTable, hostGroupName);
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe);
        return null;
    } catch (KeeperException ke) {
        Log.logErrorWarning(ke);
        return null;
    }
}
Also used : MetaClient(com.ms.silverking.cloud.toporing.meta.MetaClient) HostGroupTable(com.ms.silverking.cloud.config.HostGroupTable) RingConfiguration(com.ms.silverking.cloud.toporing.meta.RingConfiguration) NamedRingConfiguration(com.ms.silverking.cloud.toporing.meta.NamedRingConfiguration) RingConfigurationZK(com.ms.silverking.cloud.toporing.meta.RingConfigurationZK) ExclusionZK(com.ms.silverking.cloud.meta.ExclusionZK) NamedRingConfiguration(com.ms.silverking.cloud.toporing.meta.NamedRingConfiguration) Topology(com.ms.silverking.cloud.topology.Topology) StoragePolicyZK(com.ms.silverking.cloud.storagepolicy.StoragePolicyZK) IOException(java.io.IOException) StoragePolicyGroup(com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup) WeightSpecifications(com.ms.silverking.cloud.toporing.meta.WeightSpecifications) TopologyZK(com.ms.silverking.cloud.topology.TopologyZK) CloudConfiguration(com.ms.silverking.cloud.meta.CloudConfiguration) ExclusionSet(com.ms.silverking.cloud.meta.ExclusionSet) HostGroupTableZK(com.ms.silverking.cloud.meta.HostGroupTableZK) KeeperException(org.apache.zookeeper.KeeperException)

Example 4 with StoragePolicyGroup

use of com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup in project SilverKing by Morgan-Stanley.

the class TestTopoRingCreator method main.

public static void main(String[] args) {
    try {
        if (args.length != 8) {
            System.out.println("<topologyFile> <weightSpecsFile> <exclusionList> <nodeID> <storagePolicyGroup1,...> <policyID> <HostGroupTableFile> <HostGroup,...>");
        } else {
            TopologyRingCreator topologyRingCreator;
            Topology topology;
            File topologyFile;
            File weightSpecsFile;
            File exclusionFile;
            String nodeID;
            String policyID;
            // Replication         replication;
            StoragePolicyGroup[] storagePolicyGroups;
            TopologyRing lastRing;
            HostGroupTable hostGroupTable;
            Set<String> hostGroups;
            // Log.setLevelAll();
            topologyFile = new File(args[0]);
            weightSpecsFile = new File(args[1]);
            exclusionFile = new File(args[2]);
            nodeID = args[3];
            storagePolicyGroups = parseStoragePolicyGroups(args[4]);
            policyID = args[5];
            topology = TopologyParser.parse(topologyFile);
            topologyRingCreator = new TopologyRingCreator();
            hostGroupTable = HostGroupTable.parse(args[6], VersionedDefinition.NO_VERSION);
            hostGroups = ImmutableSet.copyOf(args[7].split(","));
            lastRing = null;
            for (StoragePolicyGroup spGroup : storagePolicyGroups) {
                RingTreeRecipe recipe;
                TopologyRing topologyRing;
                recipe = new RingTreeRecipe(topology, topology.getNodeByID(nodeID), new WeightSpecifications(VersionedDefinition.NO_VERSION).parse(weightSpecsFile), ExclusionSet.parse(exclusionFile), spGroup, policyID, hostGroupTable, hostGroups, 0L, DHTUtil.currentTimeMillis());
                System.out.println("********************************");
                System.out.println("RingTreeRecipe: " + recipe);
                topologyRing = topologyRingCreator.create(recipe, nodeID);
                System.out.println("\n\nTopologyRing:\n");
                System.out.println(topologyRing);
                System.out.println("\n\n");
                if (lastRing != null) {
                    topologyRing = topologyRingCreator.reduceDataMovement(lastRing, topologyRing, recipe);
                    System.out.println(topologyRing);
                }
                lastRing = topologyRing;
            }
        }
    } catch (Exception e) {
        Log.logErrorWarning(e);
    }
}
Also used : WeightSpecifications(com.ms.silverking.cloud.toporing.meta.WeightSpecifications) TopologyRingCreator(com.ms.silverking.cloud.toporing.TopologyRingCreator) RingTreeRecipe(com.ms.silverking.cloud.toporing.RingTreeRecipe) TopologyRing(com.ms.silverking.cloud.toporing.TopologyRing) HostGroupTable(com.ms.silverking.cloud.config.HostGroupTable) Topology(com.ms.silverking.cloud.topology.Topology) File(java.io.File) StoragePolicyGroup(com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup) PolicyParseException(com.ms.silverking.cloud.storagepolicy.PolicyParseException)

Example 5 with StoragePolicyGroup

use of com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method main.

// //////////////////////////////////////////////////
/**
 * @param args
 */
public static void main(String[] args) {
    try {
        if (args.length != 8) {
            System.out.println("<topologyFile> <weightSpecsFile> <exclusionList> <nodeID> <storagePolicyGroup> <policyID> <HostGroupTableFile> <HostGroup,...>");
        } else {
            TopologyRingCreator topologyRingCreator;
            Topology topology;
            File topologyFile;
            File weightSpecsFile;
            File exclusionFile;
            File storagePolicyGroupFile;
            String nodeID;
            String policyID;
            // Replication replication;
            StoragePolicyGroup storagePolicyGroup;
            TopologyRing topologyRing;
            Node node;
            RingTreeRecipe recipe;
            HostGroupTable hostGroupTable;
            Set<String> hostGroups;
            // Log.setLevelAll();
            topologyFile = new File(args[0]);
            weightSpecsFile = new File(args[1]);
            exclusionFile = new File(args[2]);
            nodeID = args[3];
            storagePolicyGroupFile = new File(args[4]);
            policyID = args[5];
            storagePolicyGroup = new PolicyParser().parsePolicyGroup(storagePolicyGroupFile, VersionedDefinition.NO_VERSION);
            topology = TopologyParser.parse(topologyFile);
            topologyRingCreator = new TopologyRingCreator();
            hostGroupTable = HostGroupTable.parse(args[6], VersionedDefinition.NO_VERSION);
            hostGroups = ImmutableSet.copyOf(args[7].split(","));
            recipe = new RingTreeRecipe(topology, topology.getNodeByID(nodeID), new WeightSpecifications(VersionedDefinition.NO_VERSION).parse(weightSpecsFile), ExclusionSet.parse(exclusionFile), storagePolicyGroup, policyID, hostGroupTable, hostGroups, // version not
            0L, // this tree
            DHTUtil.currentTimeMillis());
            System.out.println("RingTreeRecipe: " + recipe);
            topologyRing = topologyRingCreator.create(recipe, nodeID);
            System.out.println("\n\nTopologyRing:\n");
            System.out.println(topologyRing);
            System.out.println("\n\n");
            topologyRing = topologyRingCreator.reduceDataMovement(topologyRing, topologyRing, recipe);
            System.out.println(topologyRing);
        }
    } catch (Exception e) {
        Log.logErrorWarning(e);
    }
}
Also used : PolicyParser(com.ms.silverking.cloud.storagepolicy.PolicyParser) Node(com.ms.silverking.cloud.topology.Node) GenericNode(com.ms.silverking.cloud.topology.GenericNode) HostGroupTable(com.ms.silverking.cloud.config.HostGroupTable) Topology(com.ms.silverking.cloud.topology.Topology) StoragePolicyGroup(com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup) WeightSpecifications(com.ms.silverking.cloud.toporing.meta.WeightSpecifications) File(java.io.File)

Aggregations

HostGroupTable (com.ms.silverking.cloud.config.HostGroupTable)5 StoragePolicyGroup (com.ms.silverking.cloud.storagepolicy.StoragePolicyGroup)5 Topology (com.ms.silverking.cloud.topology.Topology)5 WeightSpecifications (com.ms.silverking.cloud.toporing.meta.WeightSpecifications)5 File (java.io.File)3 ExclusionSet (com.ms.silverking.cloud.meta.ExclusionSet)2 HostGroupTableZK (com.ms.silverking.cloud.meta.HostGroupTableZK)2 PolicyParser (com.ms.silverking.cloud.storagepolicy.PolicyParser)2 TopologyZK (com.ms.silverking.cloud.topology.TopologyZK)2 NamedRingConfiguration (com.ms.silverking.cloud.toporing.meta.NamedRingConfiguration)2 RingConfiguration (com.ms.silverking.cloud.toporing.meta.RingConfiguration)2 RingConfigurationZK (com.ms.silverking.cloud.toporing.meta.RingConfigurationZK)2 IOException (java.io.IOException)2 KeeperException (org.apache.zookeeper.KeeperException)2 CloudConfiguration (com.ms.silverking.cloud.meta.CloudConfiguration)1 ExclusionZK (com.ms.silverking.cloud.meta.ExclusionZK)1 ServerSetExtensionZK (com.ms.silverking.cloud.meta.ServerSetExtensionZK)1 PolicyParseException (com.ms.silverking.cloud.storagepolicy.PolicyParseException)1 StoragePolicyZK (com.ms.silverking.cloud.storagepolicy.StoragePolicyZK)1 GenericNode (com.ms.silverking.cloud.topology.GenericNode)1