Examples with Node com.ms.silverking.cloud.topology.Node used on opensource projects

Example 16 with Node

use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method allocateSubPolicy.

/*
     * private SingleRing clearOldOwners(SingleRing oldRing, Set<Node>
     * includeNodes) { SingleRing newRing;
     * 
     * newRing = oldRing.cloneEmpty(Mutability.Mutable); for (RingEntry oldEntry
     * : oldRing.getMembers()) {
     * newRing.addEntry(oldEntry.removeOwnersNotInSet(includeNodes)); } return
     * newRing; }
     */
private ProtoRegionList allocateSubPolicy(ProtoRegionList prList, SubPolicy subPolicy, String ringParentID, RingTreeRecipe recipe) {
    Node parent;
    if (debug) {
        System.out.println("subPolicy: " + subPolicy);
        System.out.flush();
    }
    parent = recipe.topology.getNodeByID(ringParentID);
    for (SubPolicyMember member : subPolicy.getMembers()) {
        prList = allocateSubPolicyMember(prList, member, recipe, parent, subPolicy.getReplicationType());
    }
    return prList;
}

Example 17 with Node

use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method main.

// //////////////////////////////////////////////////
/**
 * @param args
 */
public static void main(String[] args) {
    try {
        if (args.length != 8) {
            System.out.println("<topologyFile> <weightSpecsFile> <exclusionList> <nodeID> <storagePolicyGroup> <policyID> <HostGroupTableFile> <HostGroup,...>");
        } else {
            TopologyRingCreator topologyRingCreator;
            Topology topology;
            File topologyFile;
            File weightSpecsFile;
            File exclusionFile;
            File storagePolicyGroupFile;
            String nodeID;
            String policyID;
            // Replication replication;
            StoragePolicyGroup storagePolicyGroup;
            TopologyRing topologyRing;
            Node node;
            RingTreeRecipe recipe;
            HostGroupTable hostGroupTable;
            Set<String> hostGroups;
            // Log.setLevelAll();
            topologyFile = new File(args[0]);
            weightSpecsFile = new File(args[1]);
            exclusionFile = new File(args[2]);
            nodeID = args[3];
            storagePolicyGroupFile = new File(args[4]);
            policyID = args[5];
            storagePolicyGroup = new PolicyParser().parsePolicyGroup(storagePolicyGroupFile, VersionedDefinition.NO_VERSION);
            topology = TopologyParser.parse(topologyFile);
            topologyRingCreator = new TopologyRingCreator();
            hostGroupTable = HostGroupTable.parse(args[6], VersionedDefinition.NO_VERSION);
            hostGroups = ImmutableSet.copyOf(args[7].split(","));
            recipe = new RingTreeRecipe(topology, topology.getNodeByID(nodeID), new WeightSpecifications(VersionedDefinition.NO_VERSION).parse(weightSpecsFile), ExclusionSet.parse(exclusionFile), storagePolicyGroup, policyID, hostGroupTable, hostGroups, // version not
            0L, // this tree
            DHTUtil.currentTimeMillis());
            System.out.println("RingTreeRecipe: " + recipe);
            topologyRing = topologyRingCreator.create(recipe, nodeID);
            System.out.println("\n\nTopologyRing:\n");
            System.out.println(topologyRing);
            System.out.println("\n\n");
            topologyRing = topologyRingCreator.reduceDataMovement(topologyRing, topologyRing, recipe);
            System.out.println(topologyRing);
        }
    } catch (Exception e) {
        Log.logErrorWarning(e);
    }
}

Example 18 with Node

use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method nextRegion.

private int nextRegion(ProtoRegionList prList, RegionAllocationMode regionAllocationMode, SingleRing sourceRing, int regionIndex, Node node, int maxOwners, ReplicationType rType, Set<Node> remainingNodes) {
    int minPotentialOwners;
    int bestCandidateIndex;
    if (debug) {
        System.out.printf("TopologyRingCreator.nextRegion regionIndex %d maxOwners %d\n", regionIndex, maxOwners);
        System.out.printf("regionAllocationMode %s\n", regionAllocationMode);
    }
    minPotentialOwners = Integer.MAX_VALUE;
    bestCandidateIndex = -1;
    RingInteger _candidateRegionIndex = new RingInteger(prList.size() - 1, lastCandidateValue);
    for (int ii = 0; ii < prList.size(); ii++) {
        int candidateRegionIndex = _candidateRegionIndex.getValue();
        // for (int candidateRegionIndex = 0; candidateRegionIndex <
        // prList.size(); candidateRegionIndex++) {
        ProtoRegion candidateRegion;
        boolean valid;
        // Find a region not containing this node
        if (debug) {
            System.out.printf("TopologyRingCreator.nextRegion loop candidateRegionIndex %d\n", candidateRegionIndex);
        }
        // Now check to see if this region meets the ownership requirements
        // of the current RegionAllocationMode
        candidateRegion = prList.get(candidateRegionIndex);
        if (!candidateRegion.contains(node) && candidateRegion.getOwners(rType).size() < maxOwners) {
            if (debug) {
                System.out.printf("candidateRegion %s\n", candidateRegion);
            }
            switch(regionAllocationMode) {
                case Primary:
                    valid = sourceRing.pointOwnedByNode(candidateRegion.getRegion().getStart(), node, OwnerQueryMode.Primary);
                    break;
                case Secondary:
                    valid = sourceRing.pointOwnedByNode(candidateRegion.getRegion().getStart(), node, OwnerQueryMode.Secondary);
                    break;
                case Any:
                    valid = true;
                    break;
                default:
                    throw new RuntimeException("panic");
            }
            if (valid) {
                // return candidateRegionIndex;
                Set<Node> potentialOwnersSet;
                int potentialOwners;
                potentialOwnersSet = new HashSet<>(remainingNodes);
                potentialOwnersSet.removeAll(candidateRegion.getOwnersSet());
                potentialOwners = potentialOwnersSet.size();
                // (candidateRegion.totalOwners() + ii);
                if (potentialOwners < minPotentialOwners) {
                    minPotentialOwners = potentialOwners;
                    bestCandidateIndex = candidateRegionIndex;
                }
            }
        }
        _candidateRegionIndex.increment();
    }
    return bestCandidateIndex;
}

Example 19 with Node

use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.

the class TopologyRingCreator method doAllocation.

private void doAllocation(SingleRing sourceRing, ProtoRegionList prList, ReplicationType rType, int replicas, boolean allReplicasTag, List<NodeAllocationState> nodeAllocationStates) {
    if (debug) {
        System.out.println("\n\ndoAllocation()");
    }
    /*
         * for (ProtoRegion pr : prList.getRegionList()) { if (pr.totalOwners()
         * >= replicas) { fullRegions.add(pr.getRegion()); } }
         */
    for (RegionAllocationMode regionAllocationMode : RegionAllocationMode.values()) {
        for (int replicaIndex = 0; replicaIndex < replicas; replicaIndex++) {
            Set<Node> remainingNodes;
            remainingNodes = new HashSet<>(prList.getOwners());
            List<NodeAllocationState> unallocated;
            unallocated = new ArrayList<>(nodeAllocationStates);
            Collections.sort(unallocated, nasAvailableRingspaceComparator);
            int nodeIndex = 0;
            while (unallocated.size() > 0) {
                // for (int nodeIndex = 0; nodeIndex <
                // nodeAllocationStates.size(); nodeIndex++) {
                NodeAllocationState nas;
                Node node;
                int regionIndex;
                long ringspaceAllocated;
                boolean nodeSearchValid;
                // Collections.sort(unallocated,
                // nasAvailableRingspaceComparator);
                // nas = nodeAllocationStates.get(nodeIndex);
                nas = unallocated.remove(0);
                node = nas.getNode();
                remainingNodes.remove(node);
                ringspaceAllocated = 0;
                if (debug) {
                    System.out.printf("\n\nregionAllocationMode %s replicaIndex %d nodeIndex %d\tnode %s availableRingspace %d\n", regionAllocationMode, replicaIndex, nodeIndex, node, nas.availableRingspace());
                }
                if (allReplicasTag) {
                    regionIndex = nodeIndex;
                } else {
                    regionIndex = 0;
                }
                nodeSearchValid = true;
                while (nodeSearchValid && nas.allocationRemaining(replicaIndex) - magnitudeTolerance > 0) {
                    ProtoRegion protoRegion;
                    if (debug) {
                        System.out.printf("\tringspaceAllocated %d ringspaceToAllocate %d\tremaining %d\n", ringspaceAllocated, nas.ringspaceToAllocate(), nas.allocationRemaining(replicaIndex));
                        System.out.printf("regionIndex %d\n", regionIndex);
                    }
                    // Find an available region
                    if (!allReplicasTag) {
                        regionIndex = nextRegion(prList, regionAllocationMode, sourceRing, regionIndex, node, replicas, rType, remainingNodes);
                    } else {
                        regionIndex = prList.nextRegion(regionIndex, node, replicaIndex, rType);
                    }
                    if (debug) {
                        System.out.printf("next regionIndex %d\n", regionIndex);
                    }
                    if (regionIndex < 0) {
                        if (allReplicasTag) {
                            break;
                        } else {
                            switch(regionAllocationMode) {
                                case Primary:
                                case Secondary:
                                    regionIndex = 0;
                                    nodeSearchValid = false;
                                    continue;
                                case Any:
                                    regionIndex = createNextRegion(prList, node, rType, replicas, nas.allocationRemaining(replicaIndex));
                                    break;
                                default:
                                    throw new RuntimeException("panic");
                            }
                        }
                    }
                    // Now add this node to the region found
                    protoRegion = prList.get(regionIndex);
                    if (debug) {
                        System.out.println(protoRegion.getRegion());
                        System.out.printf("%d %d\n", protoRegion.getRegion().getSize(), nas.ringspaceToAllocate() - magnitudeTolerance);
                        System.out.printf("A nodeAllocated %d node %s ringspaceAllocated %d\n", nas.getAllocated(replicaIndex), node, ringspaceAllocated);
                    }
                    if (protoRegion.getRegion().getSize() <= NumUtil.addWithClamp(nas.allocationRemaining(replicaIndex), magnitudeTolerance)) {
                    // The candidate region size is <=
                    // ringspaceToAllocate, so we add the entire region
                    } else {
                        // appropriate portion.
                        if (debug) {
                            System.out.println("\t***splitProtoRegion***");
                        }
                        prList.splitProtoRegion(regionIndex, nas.allocationRemaining(replicaIndex));
                        // Need to get protoRegion from list again since the
                        // previous region was split
                        protoRegion = prList.get(regionIndex);
                        if (protoRegion.getRegion().getSize() != nas.allocationRemaining(replicaIndex)) {
                            throw new RuntimeException("panic");
                        }
                    }
                    nas.addAllocation(replicaIndex, protoRegion.getRegion());
                    ringspaceAllocated += protoRegion.getRegion().getSize();
                    protoRegion.addOwner(rType, node);
                    if (protoRegion.totalOwners() >= replicas) {
                        if (debug) {
                            System.out.printf("full %s\t[availableRingspace]", protoRegion.getRegion());
                        }
                        fullRegions.add(protoRegion.getRegion());
                    } else {
                        if (debug) {
                            System.out.printf("notfull %s\t[availableRingspace]", protoRegion.getRegion());
                        }
                    }
                    if (ringspaceAllocated < nas.ringspaceToAllocate() - magnitudeTolerance) {
                        if (debug) {
                            System.out.printf("incrementing regionIndex %d\n", regionIndex);
                        }
                        regionIndex = (regionIndex + 1) % prList.size();
                    }
                    if (debug) {
                        System.out.printf("B nodeAllocated %d node %s ringspaceAllocated %d\n", nas.getAllocated(replicaIndex), node, ringspaceAllocated);
                        System.out.printf("after regionIndex %d\n", regionIndex);
                    }
                }
            }
        }
    }
    prList.mergeResidualRegions(residualRegionThreshold);
}

Example 20 with Node

use of com.ms.silverking.cloud.topology.Node in project SilverKing by Morgan-Stanley.

the class PrimarySecondaryListPair method merge.

static PrimarySecondaryListPair merge(PrimarySecondaryListPair p0, PrimarySecondaryListPair p1) {
    List<Node> primaryOwners;
    List<Node> secondaryOwners;
    primaryOwners = new ArrayList<>(p0.getPrimaryOwners());
    // add unique primaries from p1
    for (Node node : p1.getPrimaryOwners()) {
        if (!primaryOwners.contains(node)) {
            primaryOwners.add(node);
        }
    }
    secondaryOwners = new ArrayList<>();
    // add secondaries from p0 that are not primaries (in p1)
    for (Node node : p0.getSecondaryOwners()) {
        if (!primaryOwners.contains(node)) {
            secondaryOwners.add(node);
        }
    }
    // add unique secondaries from p1 that are not primaries (in p0)
    for (Node node : p1.getSecondaryOwners()) {
        if (!primaryOwners.contains(node) && !secondaryOwners.contains(node)) {
            secondaryOwners.add(node);
        }
    }
    return new PrimarySecondaryListPair(primaryOwners, secondaryOwners);
}