use of com.ms.silverking.cloud.toporing.meta.RingConfiguration in project SilverKing by Morgan-Stanley.
the class DependencyWatcher method build.
private void build(Map<String, Long> curBuild) {
try {
boolean buildOK;
long ringConfigVersion;
Log.warning("New build triggered");
buildOK = false;
ringConfigVersion = -1;
try {
if (TopoRingConstants.verbose) {
System.out.println("Building tree");
}
RingTreeRecipe recipe;
Topology topology;
WeightSpecifications weightSpecs;
ExclusionSet exclusionSet;
ExclusionSet instanceExclusionSet;
StoragePolicyGroup storagePolicyGroup;
long topologyVersion;
long weightsVersion;
long exclusionVersion;
long instanceExclusionVersion;
long storagePolicyGroupVersion;
RingConfiguration ringConfig;
ZooKeeperExtended zk;
HostGroupTable hostGroupTable;
long hostGroupTableVersion;
ExclusionSet mergedExclusionSet;
zk = mc.getZooKeeper();
topologyVersion = curBuild.get(mp.getTopologyPath());
weightsVersion = curBuild.get(mp.getWeightsPath());
exclusionVersion = curBuild.get(mp.getExclusionsPath());
instanceExclusionVersion = curBuild.get(dhtMC.getMetaPaths().getInstanceExclusionsPath());
storagePolicyGroupVersion = curBuild.get(mp.getStoragePolicyGroupPath());
ringConfigVersion = curBuild.get(mp.getConfigPath());
topology = new TopologyZK(cloudMC).readFromZK(topologyVersion, null);
weightSpecs = new WeightsZK(mc).readFromZK(weightsVersion, null);
exclusionSet = new ExclusionSet(new ServerSetExtensionZK(mc, mc.getMetaPaths().getExclusionsPath()).readFromZK(exclusionVersion, null));
if (options.ignoreInstanceExclusions) {
instanceExclusionSet = ExclusionSet.emptyExclusionSet(0);
} else {
try {
instanceExclusionSet = new ExclusionSet(new ServerSetExtensionZK(mc, dhtMC.getMetaPaths().getInstanceExclusionsPath()).readFromZK(instanceExclusionVersion, null));
} catch (Exception e) {
Log.warning("No instance ExclusionSet found");
instanceExclusionSet = ExclusionSet.emptyExclusionSet(0);
}
}
mergedExclusionSet = ExclusionSet.union(exclusionSet, instanceExclusionSet);
storagePolicyGroup = new StoragePolicyGroupZK(mc).readFromZK(storagePolicyGroupVersion, null);
ringConfig = new RingConfigurationZK(mc).readFromZK(ringConfigVersion, null);
Log.warningf("ringConfiguration %s", ringConfig);
hostGroupTableVersion = zk.getLatestVersion(cloudMC.getMetaPaths().getHostGroupPath());
hostGroupTable = new HostGroupTableZK(cloudMC).readFromZK(hostGroupTableVersion, null);
try {
recipe = new RingTreeRecipe(topology, ringConfig.getRingParentName(), weightSpecs, mergedExclusionSet, storagePolicyGroup, ringConfig.getStoragePolicyName(), hostGroupTable, ringConfig.getHostGroups(), ringConfigVersion, DHTUtil.currentTimeMillis());
Log.warning("Recipe.ringParent: " + recipe.ringParent);
} catch (RuntimeException re) {
re.printStackTrace(System.out);
Log.warning("ringConfig: ", ringConfig + " " + re);
Log.logErrorWarning(re);
throw re;
}
RingTree ringTree;
RingTree prevRingTree;
long configInstanceVersion;
String newInstancePath;
configInstanceVersion = mc.getLatestConfigInstanceVersion(ringConfigVersion);
if (configInstanceVersion >= 0 && !ignoreSource) {
prevRingTree = SingleRingZK.readTree(mc, ringConfigVersion, configInstanceVersion);
} else {
prevRingTree = null;
}
if (prevRingTree == null || ignoreFeasibility || RingTreeBuilder.convergenceFeasible(prevRingTree, storagePolicyGroup, ringConfig.getStoragePolicyName(), ringConfig.getRingParentName(), exclusionSet)) {
ringTree = RingTreeBuilder.create(recipe, prevRingTree);
// ringTree = RingTreeBuilder.create(recipe, null); // for testing without movement reduction
newInstancePath = mc.createConfigInstancePath(ringConfigVersion);
SingleRingZK.writeTree(mc, topologyVersion, newInstancePath, ringTree);
if (TopoRingConstants.verbose) {
System.out.println(ringTree);
System.out.println(ringConfigVersion);
System.out.println(configInstanceVersion);
System.out.println(topologyVersion);
System.out.println(newInstancePath);
System.out.println("Building complete");
}
buildOK = true;
} else {
Log.warning("Convergence is infeasible. A region in prevTree does not have a viable server.");
}
} catch (IOException ioe) {
Log.logErrorWarning(ioe);
} catch (KeeperException ke) {
Log.logErrorWarning(ke);
}
if (exitAfterBuild) {
if (buildOK) {
try {
if (ringConfigVersion < 0) {
throw new RuntimeException("ringConfigVersion < 0");
}
setRing(ringConfigVersion);
} catch (KeeperException ke) {
Log.logErrorWarning(ke);
buildOK = false;
}
}
System.exit(buildOK ? 0 : -1);
}
} catch (RuntimeException re) {
re.printStackTrace();
} finally {
Log.warning("Leaving build");
}
}
use of com.ms.silverking.cloud.toporing.meta.RingConfiguration in project SilverKing by Morgan-Stanley.
the class StaticRingCreator method createStaticRing.
public static RingCreationResults createStaticRing(String ringName, ZooKeeperConfig zkConfig, Set<String> servers, int replication, UUIDBase myID) {
try {
MetaClient mc;
com.ms.silverking.cloud.meta.MetaClient cloudMC;
NamedRingConfiguration namedRingConfig;
RingConfiguration ringConfig;
RingTreeRecipe recipe;
Topology topology;
WeightSpecifications weightSpecs;
ExclusionSet exclusionSet;
StoragePolicyGroup storagePolicyGroup;
HostGroupTable hostGroupTable;
long ringConfigVersion;
CloudConfiguration cloudConfig;
String exclusionSpecsName;
String hostGroupTableName;
topology = StaticTopologyCreator.createTopology("topology." + myID, servers);
exclusionSpecsName = "exclusionSpecs." + myID;
hostGroupTableName = "hostGroupTable." + myID;
cloudConfig = new CloudConfiguration(topology.getName(), exclusionSpecsName, hostGroupTableName);
cloudMC = new com.ms.silverking.cloud.meta.MetaClient(cloudConfig, zkConfig);
new TopologyZK(cloudMC).writeToZK(topology, null);
new ExclusionZK(cloudMC).writeToZK(ExclusionSet.emptyExclusionSet(0));
mc = new MetaClient(NamedRingConfiguration.emptyTemplate.ringName(ringName), zkConfig);
ringConfig = new RingConfiguration(cloudConfig, "weightSpecsName", StaticTopologyCreator.parentID, SimpleStoragePolicyCreator.storagePolicyGroupName, SimpleStoragePolicyCreator.storagePolicyName, null);
new RingConfigurationZK(mc).writeToZK(ringConfig, null);
namedRingConfig = new NamedRingConfiguration(ringName, ringConfig);
mc = new MetaClient(namedRingConfig, zkConfig);
EmbeddedSK.setEmbeddedNamedRingConfiguration(ringName, namedRingConfig);
Log.warning(zkConfig);
ringConfigVersion = 0;
weightSpecs = new WeightSpecifications(0);
exclusionSet = ExclusionSet.emptyExclusionSet(0);
storagePolicyGroup = SimpleStoragePolicyCreator.createStoragePolicyGroup(replication);
new StoragePolicyZK(mc).writeToZK(storagePolicyGroup, null);
hostGroupTable = HostGroupTable.createHostGroupTable(servers, hostGroupName);
new HostGroupTableZK(cloudMC).writeToZK(hostGroupTable, null);
recipe = new RingTreeRecipe(topology, StaticTopologyCreator.parentID, weightSpecs, exclusionSet, storagePolicyGroup, SimpleStoragePolicyCreator.storagePolicyName, hostGroupTable, ImmutableSet.of(hostGroupName), ringConfigVersion, DHTUtil.currentTimeMillis());
Log.warning("Recipe.ringParent: " + recipe.ringParent);
RingTree ringTree;
String newInstancePath;
ringTree = RingTreeBuilder.create(recipe, null);
newInstancePath = mc.createConfigInstancePath(ringConfigVersion);
SingleRingZK.writeTree(mc, topology, newInstancePath, ringTree);
if (TopoRingConstants.verbose) {
System.out.println(ringTree);
System.out.println("Building complete");
}
return new RingCreationResults(exclusionSpecsName, hostGroupTableName, hostGroupTable, hostGroupName);
} catch (IOException ioe) {
Log.logErrorWarning(ioe);
return null;
} catch (KeeperException ke) {
Log.logErrorWarning(ke);
return null;
}
}
use of com.ms.silverking.cloud.toporing.meta.RingConfiguration in project SilverKing by Morgan-Stanley.
the class SKAdmin method getRing.
private static Pair<RingConfiguration, InstantiatedRingTree> getRing(DHTConfiguration dhtConfig, com.ms.silverking.cloud.dht.meta.MetaClient dhtMC) throws IOException, KeeperException {
String ringName;
Pair<Long, Long> ringVersion;
DHTRingCurTargetZK dhtRingCurTargetZK;
Triple<String, Long, Long> curRingNameAndVersion;
RingConfiguration ringConfig;
RingConfigurationZK ringConfigZK;
InstantiatedRingTree ringTree;
com.ms.silverking.cloud.toporing.meta.MetaClient ringMC;
dhtRingCurTargetZK = new DHTRingCurTargetZK(dhtMC, dhtConfig);
curRingNameAndVersion = dhtRingCurTargetZK.getCurRingAndVersionPair();
if (curRingNameAndVersion == null) {
return null;
}
ringName = curRingNameAndVersion.getHead();
ringVersion = curRingNameAndVersion.getTail();
ringMC = com.ms.silverking.cloud.toporing.meta.MetaClient.createMetaClient(ringName, ringVersion.getV1(), dhtMC.getZooKeeper().getZKConfig());
ringConfigZK = new RingConfigurationZK(ringMC);
ringConfig = ringConfigZK.readFromZK(ringVersion.getV1(), null);
// ringTree = SingleRingZK.readTree(ringMC, ringVersion);
// FUTURE - think about whether or not we should keep this, currently unused
ringTree = null;
return new Pair<>(ringConfig, ringTree);
}
use of com.ms.silverking.cloud.toporing.meta.RingConfiguration in project SilverKing by Morgan-Stanley.
the class DHTMetaWatcher method readRing.
// FIXME - think about zkidLimit
public void readRing(String ringName, long zkidLimit) throws KeeperException, IOException {
com.ms.silverking.cloud.toporing.meta.MetaClient ringMC;
// com.ms.silverking.cloud.meta.MetaClient cloudMC;
NamedRingConfiguration namedRingConfig;
RingConfiguration ringConfig;
InstantiatedRingTree ringTree;
long ringConfigVersion;
long configInstanceVersion;
DHTMetaUpdate dhtMetaUpdate;
int readAttemptIndex;
int ringReadAttempts = 20;
int ringReadRetryInvervalSeconds = 2;
ZooKeeperExtended zk;
// cloudMC = new com.ms.silverking.cloud.meta.MetaClient(ringConfig.getCloudConfiguration(),
// zkConfig);
zk = mc.getZooKeeper();
// unresolved
namedRingConfig = new NamedRingConfiguration(ringName, RingConfiguration.emptyTemplate);
ringMC = new com.ms.silverking.cloud.toporing.meta.MetaClient(namedRingConfig, zkConfig);
ringConfigVersion = zk.getVersionPriorTo(ringMC.getMetaPaths().getConfigPath(), zkidLimit);
ringConfig = new RingConfigurationZK(ringMC).readFromZK(ringConfigVersion, null);
// resolved
namedRingConfig = new NamedRingConfiguration(ringName, ringConfig);
ringMC = new com.ms.silverking.cloud.toporing.meta.MetaClient(namedRingConfig, zkConfig);
if (enableLogging) {
Log.warning("ringConfig\t", ringConfig);
}
configInstanceVersion = zk.getVersionPriorTo(ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion), zkidLimit);
if (enableLogging) {
Log.warning("configInstanceVersion: " + configInstanceVersion);
}
if (configInstanceVersion == -1) {
configInstanceVersion = 0;
}
if (DHTConstants.isDaemon || Log.levelMet(Level.INFO)) {
Log.warning("Waiting until valid " + ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion) + " " + configInstanceVersion);
}
SingleRingZK.waitUntilValid(ringMC, ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion), configInstanceVersion);
if (DHTConstants.isDaemon || Log.levelMet(Level.INFO)) {
Log.warning("Valid");
}
ringTree = null;
readAttemptIndex = 0;
while (ringTree == null) {
try {
// , weightsVersion);
ringTree = SingleRingZK.readTree(ringMC, ringConfigVersion, configInstanceVersion);
} catch (Exception e) {
if (++readAttemptIndex >= ringReadAttempts) {
throw new RuntimeException("Ring read failed", e);
} else {
ThreadUtil.sleepSeconds(ringReadRetryInvervalSeconds);
}
}
}
if (enableLogging) {
Log.warning("\t\t###\t" + ringConfigVersion + "\t" + configInstanceVersion);
}
dhtMetaUpdate = new DHTMetaUpdate(dhtConfig, namedRingConfig, ringTree, mc);
notifyListeners(dhtMetaUpdate);
// ringUpdateListener.newRingTree(new NamedRingConfiguration(ringName, ringConfig), ringTree);
// ringUpdateListener.newRingTree(ringTree.getMap(ringConfig.getRingParentName()));
}
use of com.ms.silverking.cloud.toporing.meta.RingConfiguration in project SilverKing by Morgan-Stanley.
the class DHTMetaReader method readRing.
public DHTMetaUpdate readRing(String ringName, long ringConfigVersion, long configInstanceVersion) throws KeeperException, IOException {
com.ms.silverking.cloud.toporing.meta.MetaClient ringMC;
NamedRingConfiguration namedRingConfig;
RingConfiguration ringConfig;
InstantiatedRingTree ringTree;
// long ringConfigVersion;
// long configInstanceVersion;
int readAttemptIndex;
int ringReadAttempts = 20;
int ringReadRetryInvervalSeconds = 2;
ZooKeeperExtended zk;
zk = mc.getZooKeeper();
if (EmbeddedSK.embedded()) {
namedRingConfig = EmbeddedSK.getEmbeddedNamedRingConfiguration(ringName);
} else {
// unresolved
namedRingConfig = new NamedRingConfiguration(ringName, RingConfiguration.emptyTemplate);
ringMC = new com.ms.silverking.cloud.toporing.meta.MetaClient(namedRingConfig, zkConfig);
// ringConfigVersion = zk.getVersionPriorTo(ringMC.getMetaPaths().getConfigPath(), zkidLimit);
try {
ringConfig = new RingConfigurationZK(ringMC).readFromZK(ringConfigVersion, null);
} catch (Exception e) {
Log.warning("Ignoring: ", e);
ringConfig = new RingConfiguration(new CloudConfiguration(null, null, null), null, null, null, null, null);
}
// resolved
namedRingConfig = new NamedRingConfiguration(ringName, ringConfig);
if (enableLogging) {
Log.warning("ringConfig\t", ringConfig);
}
}
ringMC = new com.ms.silverking.cloud.toporing.meta.MetaClient(namedRingConfig, zkConfig);
// configInstanceVersion = zk.getVersionPriorTo(ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion), zkidLimit);
if (enableLogging) {
Log.warning("configInstanceVersion: " + configInstanceVersion);
}
if (configInstanceVersion < 0) {
throw new RuntimeException("Invalid configInstanceVersion: " + configInstanceVersion);
}
// if (configInstanceVersion == -1) {
// configInstanceVersion = 0;
// }
// FUTURE - we shouldn't get here unless it's valid. Think about error messages if invalid, instead of waiting.
Log.warning("Waiting until valid " + ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion) + " " + configInstanceVersion);
SingleRingZK.waitUntilValid(ringMC, ringMC.getMetaPaths().getConfigInstancePath(ringConfigVersion), configInstanceVersion);
Log.warning("Valid");
ringTree = null;
readAttemptIndex = 0;
while (ringTree == null) {
try {
// , weightsVersion);
ringTree = SingleRingZK.readTree(ringMC, ringConfigVersion, configInstanceVersion);
} catch (Exception e) {
if (++readAttemptIndex >= ringReadAttempts) {
throw new RuntimeException("Ring read failed", e);
} else {
ThreadUtil.sleepSeconds(ringReadRetryInvervalSeconds);
}
}
}
if (enableLogging) {
Log.warning("\t\t###\t" + ringConfigVersion + "\t" + configInstanceVersion);
}
return new DHTMetaUpdate(dhtConfig, namedRingConfig, ringTree, mc);
}
Aggregations