Examples with NodeClass com.ms.silverking.cloud.topology.NodeClass used on opensource projects

Example 1 with NodeClass

use of com.ms.silverking.cloud.topology.NodeClass in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method _create.

/**
 * Create a new ring that is as similar as possible to the ring passed in so
 * that we minimize the movement of data.
 *
 * @param sourceRing
 *            an immutable SingleRing that may or may not meet the
 *            requirements of the recipe
 * @param recipe
 * @return an immutable SingleRing (returned as a TopologyRing) that meets
 *         the requirements of the recipe
 */
private TopologyRing _create(SingleRing sourceRing, RingTreeRecipe recipe, String ringParentID) {
    // This version ignores the source ring, _create2 takes the source into
    // account
    /*
         * Single StoragePolicy applies. StoragePolicy has primary and secondary
         * SubPolicy Each SubPolicy has multiple SubPolicyMembers
         * 
         * Each SubPolicyMember has a logical ring associated with it - The set
         * of nodes in each of these rings is disjoint with respect to all other
         * ring node sets - Weights must be normalized for each member
         * 
         * Loop structure: For {primary, secondary} For all SubPolicyMembers For
         * all replicas
         */
    Node parent;
    NodeClass nodeClass;
    parent = recipe.topology.getNodeByID(ringParentID);
    nodeClass = parent.getChildNodeClass();
    ProtoRegionList prList;
    prList = ProtoRegionList.createEmpty();
    for (SubPolicy subPolicy : recipe.storagePolicy.getSubPolicies()) {
        prList = allocateSubPolicy(prList, subPolicy, ringParentID, recipe);
    }
    System.out.println("*********************");
    System.out.println(prList);
    return prList.toSingleRing(nodeClass, recipe);
}

Example 2 with NodeClass

use of com.ms.silverking.cloud.topology.NodeClass in project SilverKing by Morgan-Stanley.

the class SubPolicy method ensureMemberClassesConsistent.

private void ensureMemberClassesConsistent() {
    if (members.size() > 0) {
        NodeClass memberClass;
        memberClass = members.get(0).getNodeClassAndStoragePolicyName().getNodeClass();
        for (SubPolicyMember member : members) {
            System.out.println(member + "\n");
            if (member.getNodeClassAndStoragePolicyName().getNodeClass() != memberClass) {
                System.err.println(member.getNodeClassAndStoragePolicyName().getNodeClass() + " != " + memberClass);
                throw new RuntimeException("Inconsistent member node classes");
            }
        }
    }
}

Example 3 with NodeClass

use of com.ms.silverking.cloud.topology.NodeClass in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method _create2.

/**
 * Create a new ring that is as similar as possible to the ring passed in so
 * that we minimize the movement of data.
 *
 * @param sourceRing
 *            an immutable SingleRing that may or may not meet the
 *            requirements of the recipe
 * @param recipe
 * @return an immutable SingleRing (returned as a TopologyRing) that meets
 *         the requirements of the recipe
 */
private TopologyRing _create2(SingleRing sourceRing, RingTreeRecipe recipe, String ringParentID) {
    /*
         * Single StoragePolicy applies. StoragePolicy has primary and secondary
         * SubPolicy Each SubPolicy has multiple SubPolicyMembers
         * 
         * Each SubPolicyMember has a logical ring associated with it - The set
         * of nodes in each of these rings is disjoint with respect to all other
         * ring node sets - Weights must be normalized for each member
         * 
         * Loop structure: For {primary, secondary} For all SubPolicyMembers For
         * all replicas
         */
    Node parent;
    NodeClass nodeClass;
    if (debug) {
        System.out.println("TopologyRingCreator.create2()");
    }
    Preconditions.checkNotNull(sourceRing);
    parent = recipe.topology.getNodeByID(ringParentID);
    nodeClass = parent.getChildNodeClass();
    ProtoRegionList prList;
    prList = ProtoRegionList.createIsomorphic(sourceRing);
    for (SubPolicy subPolicy : recipe.storagePolicy.getSubPolicies()) {
        allocateSubPolicy2(sourceRing, prList, subPolicy, ringParentID, recipe);
    }
    System.out.println("*********************");
    System.out.println(prList);
    return prList.toSingleRing(nodeClass, recipe);
}

Example 4 with NodeClass

use of com.ms.silverking.cloud.topology.NodeClass in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method allocateSubPolicyMember.

private ProtoRegionList allocateSubPolicyMember(ProtoRegionList prList, SubPolicyMember member, RingTreeRecipe recipe, Node parent, ReplicationType rType) {
    List<Node> nodes;
    int replicas;
    NodeClass nodeClass;
    List<Long> nodeRegionSizes;
    List<Integer> shuffle;
    boolean allReplicasTag;
    if (recipe.getWeight(parent) == 0.0) {
        System.out.printf("Ignoring allocation for zero-weighted node %s", parent.getIDString());
        return prList;
    }
    nodeClass = parent.getChildNodeClass();
    if (nodeClass == null) {
        System.err.println("parent.getChildren().size(): " + parent.getChildren().size());
        throw new RuntimeException("No nodes for parent " + parent);
    }
    // System.out.printf("parent %s\n", parent.getIDString());
    nodes = member.getNodesList(parent, recipe);
    if (debug) {
        System.out.println("nodes.size() " + nodes.size());
    }
    nodeRegionSizes = RingRegion.allRingspace.dividedRegionSizes(recipe.weightSpecs.getWeights(nodes));
    if (debug) {
        System.out.println("nodeRegionSizes");
        System.out.println(CollectionUtil.toString(nodeRegionSizes, '\n'));
    }
    shuffle = new ArrayList<>();
    for (int i = 0; i < nodes.size(); i++) {
        shuffle.add(i);
    }
    replicas = member.getQuantity();
    allReplicasTag = replicas == SubPolicyMember.ALL;
    if (allReplicasTag) {
        replicas = nodes.size();
    }
    for (int replicaIndex = 0; replicaIndex < replicas; replicaIndex++) {
        ProtoRegionList _prList;
        int attemptIndex;
        boolean successful;
        if (debug) {
            System.out.printf("\nreplicaIndex %d", replicaIndex);
        }
        _prList = prList.duplicate();
        attemptIndex = 0;
        successful = false;
        while (!successful && attemptIndex++ < maxShuffleAttempts) {
            Collections.shuffle(shuffle, random);
            try {
                for (int _nodeIndex = 0; _nodeIndex < nodes.size(); _nodeIndex++) {
                    Node node;
                    int regionIndex;
                    long ringspaceToAllocate;
                    long ringspaceAllocated;
                    int nodeIndex;
                    nodeIndex = shuffle.get(_nodeIndex);
                    if (debug) {
                        System.out.printf("\nreplicaIndex %d nodeIndex %d _nodeIndex %d attemptIndex %d\n", replicaIndex, nodeIndex, _nodeIndex, attemptIndex);
                    }
                    // Determine how much ringspace to allocate to this node
                    node = nodes.get(nodeIndex);
                    ringspaceToAllocate = nodeRegionSizes.get(nodeIndex);
                    ringspaceAllocated = 0;
                    sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                    // regionIndex = _nodeIndex;
                    regionIndex = 0;
                    while (ringspaceAllocated < ringspaceToAllocate - magnitudeTolerance) {
                        ProtoRegion protoRegion;
                        if (debug) {
                            System.out.printf("\tringspaceAllocated %d ringspaceToAllocate %d\tremaining %d\n", ringspaceAllocated, ringspaceToAllocate, ringspaceToAllocate - ringspaceAllocated);
                            System.out.printf("regionIndex %d\n", regionIndex);
                        }
                        // Find an available region
                        regionIndex = prList.nextRegion(regionIndex, node, replicaIndex, rType);
                        if (debug) {
                            System.out.printf("next regionIndex %d\n", regionIndex);
                        }
                        if (debug)
                            sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                        if (regionIndex < 0) {
                            if (allReplicasTag) {
                                break;
                            } else {
                                Map<String, Long> allocations;
                                allocations = prList.getAllocations();
                                for (int i = 0; i < nodes.size(); i++) {
                                    long regionSize;
                                    Node n;
                                    long allocation;
                                    n = nodes.get(i);
                                    regionSize = nodeRegionSizes.get(i);
                                    allocation = allocations.get(n.getIDString());
                                    System.out.printf("%s\t%d\t%d\t%d\n", n.getIDString(), regionSize, allocation, regionSize * (replicaIndex + 1) - allocation);
                                }
                                throw new RuntimeException("Couldn't find a free region");
                            }
                        }
                        if (debug)
                            sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                        // Now add this node to the region found
                        protoRegion = prList.get(regionIndex);
                        if (debug) {
                            System.out.println(protoRegion.getRegion());
                            System.out.printf("%d %d\n", protoRegion.getRegion().getSize(), ringspaceToAllocate - magnitudeTolerance);
                        }
                        if (protoRegion.getRegion().getSize() <= NumUtil.addWithClamp(ringspaceToAllocate - ringspaceAllocated, magnitudeTolerance)) {
                            // The candidate region size is <= ringspaceToAllocate,
                            // so we add the entire region
                            ringspaceAllocated += protoRegion.getRegion().getSize();
                            protoRegion.addOwner(rType, node);
                            if (debug)
                                sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                        } else {
                            // portion.
                            if (debug) {
                                System.out.println("\t***splitProtoRegion***");
                            }
                            if (debug)
                                sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                            prList.splitProtoRegion(regionIndex, ringspaceToAllocate - ringspaceAllocated);
                            if (debug)
                                sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                            // Need to get protoRegion from list again since the
                            // previous region was split
                            protoRegion = prList.get(regionIndex);
                            if (protoRegion.getRegion().getSize() != ringspaceToAllocate - ringspaceAllocated) {
                                throw new RuntimeException("panic");
                            }
                            if (debug) {
                                System.out.printf("protoRegion %s\n", protoRegion);
                            }
                            // Add the entire region
                            ringspaceAllocated += protoRegion.getRegion().getSize();
                            protoRegion.addOwner(rType, node);
                            if (debug)
                                sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                        }
                        if (debug)
                            sanityCheckAllocations(prList, nodes, nodeRegionSizes, replicaIndex, nodeIndex, ringspaceToAllocate - ringspaceAllocated, shuffle);
                        if (ringspaceAllocated < ringspaceToAllocate - magnitudeTolerance) {
                            if (debug) {
                                System.out.printf("incrementing regionIndex %d\n", regionIndex);
                            }
                            regionIndex = (regionIndex + 1) % prList.size();
                        }
                        if (debug) {
                            System.out.printf("after regionIndex %d\n", regionIndex);
                        }
                    }
                }
                successful = true;
            } catch (RuntimeException re) {
                if (attemptIndex >= maxShuffleAttempts) {
                    throw re;
                } else {
                    Log.warningf("Ignoring exception. Trying a new shuffle. attemptIndex %s", attemptIndex);
                    prList = _prList.duplicate();
                }
            }
        }
    }
    return prList;
}

Example 5 with NodeClass

use of com.ms.silverking.cloud.topology.NodeClass in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method createNodeAllocationStates.

private List<NodeAllocationState> createNodeAllocationStates(SubPolicyMember member, RingTreeRecipe recipe, Node parent, int replicas) {
    List<NodeAllocationState> nodeAllocationStates;
    List<Node> nodes;
    NodeClass nodeClass;
    List<Long> nodeRegionSizes;
    nodeClass = parent.getChildNodeClass();
    if (nodeClass == null) {
        System.err.println("parent.getChildren().size(): " + parent.getChildren().size());
        throw new RuntimeException("No nodes for parent " + parent);
    }
    nodes = member.getNodesList(parent, recipe);
    if (debug) {
        System.out.println("nodes.size() " + nodes.size());
    }
    nodeRegionSizes = RingRegion.allRingspace.dividedRegionSizes(recipe.weightSpecs.getWeights(nodes));
    if (debug) {
        System.out.println(CollectionUtil.toString(nodeRegionSizes, '\n'));
    }
    nodeAllocationStates = new ArrayList<>(nodes.size());
    for (int i = 0; i < nodes.size(); i++) {
        nodeAllocationStates.add(new NodeAllocationState(nodes.get(i), replicas, nodeRegionSizes.get(i)));
    }
    return nodeAllocationStates;
}