use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.
the class TopologyRingCreator method _create2.
/**
* Create a new ring that is as similar as possible to the ring passed in so
* that we minimize the movement of data.
*
* @param sourceRing
* an immutable SingleRing that may or may not meet the
* requirements of the recipe
* @param recipe
* @return an immutable SingleRing (returned as a TopologyRing) that meets
* the requirements of the recipe
*/
private TopologyRing _create2(SingleRing sourceRing, RingTreeRecipe recipe, String ringParentID) {
/*
* Single StoragePolicy applies. StoragePolicy has primary and secondary
* SubPolicy Each SubPolicy has multiple SubPolicyMembers
*
* Each SubPolicyMember has a logical ring associated with it - The set
* of nodes in each of these rings is disjoint with respect to all other
* ring node sets - Weights must be normalized for each member
*
* Loop structure: For {primary, secondary} For all SubPolicyMembers For
* all replicas
*/
Node parent;
NodeClass nodeClass;
if (debug) {
System.out.println("TopologyRingCreator.create2()");
}
Preconditions.checkNotNull(sourceRing);
parent = recipe.topology.getNodeByID(ringParentID);
nodeClass = parent.getChildNodeClass();
ProtoRegionList prList;
prList = ProtoRegionList.createIsomorphic(sourceRing);
for (SubPolicy subPolicy : recipe.storagePolicy.getSubPolicies()) {
allocateSubPolicy2(sourceRing, prList, subPolicy, ringParentID, recipe);
}
System.out.println("*********************");
System.out.println(prList);
return prList.toSingleRing(nodeClass, recipe);
}
use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.
the class TopologyRingCreator method allocateSubPolicyMember.
private ProtoRegionList allocateSubPolicyMember(ProtoRegionList prList, SubPolicyMember member, RingTreeRecipe recipe, Node parent, ReplicationType rType) {
List<Node> nodes;
int replicas;
NodeClass nodeClass;
List<Long> nodeRegionSizes;
List<Integer> shuffle;
boolean allReplicasTag;
if (recipe.getWeight(parent) == 0.0) {
System.out.printf("Ignoring allocation for zero-weighted node %s", parent.getIDString());
return prList;
}
nodeClass = parent.getChildNodeClass();
if (nodeClass == null) {
System.err.println("parent.getChildren().size(): " + parent.getChildren().size());
throw new RuntimeException("No nodes for parent " + parent);
}
// System.out.printf("parent %s\n", parent.getIDString());
nodes = member.getNodesList(parent, recipe);
if (debug) {
System.out.println("nodes.size() " + nodes.size());
}
nodeRegionSizes = RingRegion.allRingspace.dividedRegionSizes(recipe.weightSpecs.getWeights(nodes));
if (debug) {
System.out.println("nodeRegionSizes");
System.out.println(CollectionUtil.toString(nodeRegionSizes, '\n'));
}
shuffle = new ArrayList<>();
for (int i = 0; i < nodes.size(); i++) {
shuffle.add(i);
}
replicas = member.getQuantity();
allReplicasTag = replicas == SubPolicyMember.ALL;
if (allReplicasTag) {
replicas = nodes.size();
}
for (int replicaIndex = 0; replicaIndex < replicas; replicaIndex++) {
ProtoRegionList _prList;
int attemptIndex;
boolean successful;
if (debug) {
System.out.printf("\nreplicaIndex %d", replicaIndex);
}
_prList = prList.duplicate();
attemptIndex = 0;
successful = false;
while (!successful && attemptIndex++ < maxShuffleAttempts) {
Collections.shuffle(shuffle, random);
try {
for (int _nodeIndex = 0; _nodeIndex < nodes.size(); _nodeIndex++) {
Node node;
int regionIndex;
long ringspaceToAllocate;
long ringspaceAllocated;
int nodeIndex;
nodeIndex = shuffle.get(_nodeIndex);
if (debug) {
System.out.printf("\nreplicaIndex %d nodeIndex %d _nodeIndex %d attemptIndex %d\n", replicaIndex, nodeIndex, _nodeIndex, attemptIndex);
}
// Determine how much ringspace to allocate to this node
node = nodes.get(nodeIndex);
ringspaceToAllocate = nodeRegionSizes.get(nodeIndex);
ringspaceAllocated = 0;
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
// regionIndex = _nodeIndex;
regionIndex = 0;
while (ringspaceAllocated < ringspaceToAllocate - magnitudeTolerance) {
ProtoRegion protoRegion;
if (debug) {
System.out.printf("\tringspaceAllocated %d ringspaceToAllocate %d\tremaining %d\n", ringspaceAllocated, ringspaceToAllocate, ringspaceToAllocate - ringspaceAllocated);
System.out.printf("regionIndex %d\n", regionIndex);
}
// Find an available region
regionIndex = prList.nextRegion(regionIndex, node, replicaIndex, rType);
if (debug) {
System.out.printf("next regionIndex %d\n", regionIndex);
}
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
if (regionIndex < 0) {
if (allReplicasTag) {
break;
} else {
Map<String, Long> allocations;
allocations = prList.getAllocations();
for (int i = 0; i < nodes.size(); i++) {
long regionSize;
Node n;
long allocation;
n = nodes.get(i);
regionSize = nodeRegionSizes.get(i);
allocation = allocations.get(n.getIDString());
System.out.printf("%s\t%d\t%d\t%d\n", n.getIDString(), regionSize, allocation, regionSize * (replicaIndex + 1) - allocation);
}
throw new RuntimeException("Couldn't find a free region");
}
}
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
// Now add this node to the region found
protoRegion = prList.get(regionIndex);
if (debug) {
System.out.println(protoRegion.getRegion());
System.out.printf("%d %d\n", protoRegion.getRegion().getSize(), ringspaceToAllocate - magnitudeTolerance);
}
if (protoRegion.getRegion().getSize() <= NumUtil.addWithClamp(ringspaceToAllocate - ringspaceAllocated, magnitudeTolerance)) {
// The candidate region size is <= ringspaceToAllocate,
// so we add the entire region
ringspaceAllocated += protoRegion.getRegion().getSize();
protoRegion.addOwner(rType, node);
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
} else {
// portion.
if (debug) {
System.out.println("\t***splitProtoRegion***");
}
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
prList.splitProtoRegion(regionIndex, ringspaceToAllocate - ringspaceAllocated);
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
// Need to get protoRegion from list again since the
// previous region was split
protoRegion = prList.get(regionIndex);
if (protoRegion.getRegion().getSize() != ringspaceToAllocate - ringspaceAllocated) {
throw new RuntimeException("panic");
}
if (debug) {
System.out.printf("protoRegion %s\n", protoRegion);
}
// Add the entire region
ringspaceAllocated += protoRegion.getRegion().getSize();
protoRegion.addOwner(rType, node);
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
}
if (debug)
sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
if (ringspaceAllocated < ringspaceToAllocate - magnitudeTolerance) {
if (debug) {
System.out.printf("incrementing regionIndex %d\n", regionIndex);
}
regionIndex = (regionIndex + 1) % prList.size();
}
if (debug) {
System.out.printf("after regionIndex %d\n", regionIndex);
}
}
}
successful = true;
} catch (RuntimeException re) {
if (attemptIndex >= maxShuffleAttempts) {
throw re;
} else {
Log.warningf("Ignoring exception. Trying a new shuffle. attemptIndex %s", attemptIndex);
prList = _prList.duplicate();
}
}
}
}
return prList;
}
use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.
the class TopologyRingCreator method create.
/**
* Create a TopologyRing given a root node ID.
*
* @param recipe
* @param nodeID
* @return
*/
public TopologyRing create(RingTreeRecipe recipe, String nodeID) {
Node node;
node = recipe.topology.getNodeByID(nodeID);
if (node != null) {
SingleRing sourceRing;
TopologyRing _ring;
if (debug) {
System.out.println("Create for node: " + node);
}
sourceRing = SingleRing.emptyRing(recipe.topology.getNodeByID(nodeID).getChildNodeClass(), 0, recipe.storagePolicy.getName());
sourceRing.freeze(recipe.weightSpecs);
if (debug) {
System.out.printf("empty source\n%s\n", sourceRing);
}
_ring = _create(sourceRing, recipe, nodeID);
_ring = _ring.simplify();
_ring.freeze(recipe.weightSpecs);
return _ring;
} else {
if (debug) {
System.out.println("Couldn't find nodeID: " + nodeID);
}
return null;
}
}
use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.
the class TopologyRingCreator method create.
public TopologyRing create(RingTreeRecipe recipe, String nodeID, SingleRing sourceRing) {
Node node;
Preconditions.checkNotNull(sourceRing);
node = recipe.topology.getNodeByID(nodeID);
if (node != null) {
TopologyRing _ring;
if (debug) {
System.out.println("Create w/ source for node: " + node);
}
_ring = _create2(sourceRing, recipe, nodeID);
_ring = _ring.simplify();
_ring.freeze(recipe.weightSpecs);
return _ring;
} else {
if (debug) {
System.out.println("Couldn't find nodeID: " + nodeID);
}
return null;
}
}
use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.
the class TopologyRingCreator method dummySetNodes.
private static Set<Node> dummySetNodes(Set<String> nodeIDs) {
Set<Node> dummyNodes;
dummyNodes = new HashSet<>();
for (String nodeID : nodeIDs) {
dummyNodes.add(new GenericNode(NodeClass.server, nodeID));
}
return dummyNodes;
}
Aggregations