Examples with HostOperatorSet com.datatorrent.stram.plan.physical.PTOperator.HostOperatorSet used on opensource projects

Example 1 with HostOperatorSet

use of com.datatorrent.stram.plan.physical.PTOperator.HostOperatorSet in project apex-core by apache.

the class PhysicalPlan method setLocalityGrouping.

private void setLocalityGrouping(PMapping pnodes, PTOperator newOperator, LocalityPrefs localityPrefs, Locality ltype, String host) {
    HostOperatorSet grpObj = newOperator.getGrouping(ltype);
    if (host != null) {
        grpObj.setHost(host);
    }
    Set<PTOperator> s = grpObj.getOperatorSet();
    s.add(newOperator);
    LocalityPref loc = localityPrefs.prefs.get(pnodes);
    if (loc != null) {
        for (PMapping localPM : loc.operators) {
            if (pnodes.parallelPartitions == localPM.parallelPartitions) {
                if (localPM.partitions.size() >= pnodes.partitions.size()) {
                    // apply locality setting per partition
                    s.addAll(localPM.partitions.get(pnodes.partitions.size() - 1).getGrouping(ltype).getOperatorSet());
                }
            } else {
                for (PTOperator otherNode : localPM.partitions) {
                    s.addAll(otherNode.getGrouping(ltype).getOperatorSet());
                }
            }
        }
        for (PTOperator localOper : s) {
            if (grpObj.getHost() == null) {
                grpObj.setHost(localOper.groupings.get(ltype).getHost());
            }
            localOper.groupings.put(ltype, grpObj);
        }
    }
}

Example 2 with HostOperatorSet

use of com.datatorrent.stram.plan.physical.PTOperator.HostOperatorSet in project apex-core by apache.

the class ResourceRequestHandler method getHostForContainer.

/**
   * Get host name where container would be allocated give node local constraints
   * @param container
   * @return
   */
public String getHostForContainer(PTContainer container) {
    for (PTOperator oper : container.getOperators()) {
        HostOperatorSet grpObj = oper.getNodeLocalOperators();
        String host = nodeLocalMapping.get(grpObj.getOperatorSet());
        if (host != null) {
            return host;
        }
        if (grpObj.getHost() != null) {
            host = grpObj.getHost();
            return host;
        }
    }
    return null;
}

Example 3 with HostOperatorSet

use of com.datatorrent.stram.plan.physical.PTOperator.HostOperatorSet in project apex-core by apache.

the class ResourceRequestHandler method getHost.

public String getHost(ContainerStartRequest csr, boolean first) {
    String host = null;
    PTContainer c = csr.container;
    if (first) {
        for (PTOperator oper : c.getOperators()) {
            HostOperatorSet grpObj = oper.getNodeLocalOperators();
            host = nodeLocalMapping.get(grpObj.getOperatorSet());
            if (host != null) {
                antiAffinityMapping.put(c, host);
                return host;
            }
            if (grpObj.getHost() != null) {
                host = grpObj.getHost();
                // using the 1st host value as host for container
                break;
            }
        }
        if (host != null && nodeReportMap.get(host) != null) {
            for (PTOperator oper : c.getOperators()) {
                HostOperatorSet grpObj = oper.getNodeLocalOperators();
                Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
                NodeReport report = nodeReportMap.get(host);
                int aggrMemory = c.getRequiredMemoryMB();
                int vCores = c.getRequiredVCores();
                Set<PTContainer> containers = Sets.newHashSet();
                containers.add(c);
                for (PTOperator nodeLocalOper : nodeLocalSet) {
                    if (!containers.contains(nodeLocalOper.getContainer())) {
                        aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                        vCores += nodeLocalOper.getContainer().getRequiredVCores();
                        containers.add(nodeLocalOper.getContainer());
                    }
                }
                int memAvailable = report.getCapability().getMemory() - report.getUsed().getMemory();
                int vCoresAvailable = report.getCapability().getVirtualCores() - report.getUsed().getVirtualCores();
                if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
                    nodeLocalMapping.put(nodeLocalSet, host);
                    antiAffinityMapping.put(c, host);
                    return host;
                }
            }
        }
    }
    // the host requested didn't have the resources so looking for other hosts
    host = null;
    List<String> antiHosts = new ArrayList<>();
    List<String> antiPreferredHosts = new ArrayList<>();
    if (!c.getStrictAntiPrefs().isEmpty()) {
        // Check if containers are allocated already for the anti-affinity containers
        populateAntiHostList(c, antiHosts);
    }
    if (!c.getPreferredAntiPrefs().isEmpty()) {
        populateAntiHostList(c, antiPreferredHosts);
    }
    LOG.info("Strict anti-affinity = {} for container with operators {}", antiHosts, StringUtils.join(c.getOperators(), ","));
    for (PTOperator oper : c.getOperators()) {
        HostOperatorSet grpObj = oper.getNodeLocalOperators();
        Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
        if (nodeLocalSet.size() > 1 || !c.getStrictAntiPrefs().isEmpty() || !c.getPreferredAntiPrefs().isEmpty()) {
            LOG.info("Finding new host for {}", nodeLocalSet);
            int aggrMemory = c.getRequiredMemoryMB();
            int vCores = c.getRequiredVCores();
            Set<PTContainer> containers = Sets.newHashSet();
            containers.add(c);
            // aggregate memory required for all containers
            for (PTOperator nodeLocalOper : nodeLocalSet) {
                if (!containers.contains(nodeLocalOper.getContainer())) {
                    aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                    vCores += nodeLocalOper.getContainer().getRequiredVCores();
                    containers.add(nodeLocalOper.getContainer());
                }
            }
            host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            if (host == null && !antiPreferredHosts.isEmpty() && !antiHosts.isEmpty()) {
                // Drop the preferred constraint and try allocation
                antiPreferredHosts.clear();
                host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            }
            if (host != null) {
                antiAffinityMapping.put(c, host);
            } else {
                host = INVALID_HOST;
            }
        }
    }
    LOG.info("Found host {}", host);
    return host;
}

Example 4 with HostOperatorSet

use of com.datatorrent.stram.plan.physical.PTOperator.HostOperatorSet in project apex-core by apache.

the class PhysicalPlan method removePTOperator.

void removePTOperator(PTOperator oper) {
    LOG.debug("Removing operator {}", oper);
    // per partition merge operators
    if (!oper.upstreamMerge.isEmpty()) {
        for (PTOperator unifier : oper.upstreamMerge.values()) {
            removePTOperator(unifier);
        }
    }
    // remove inputs from downstream operators
    for (PTOutput out : oper.outputs) {
        for (PTInput sinkIn : out.sinks) {
            if (sinkIn.source.source == oper) {
                ArrayList<PTInput> cowInputs = Lists.newArrayList(sinkIn.target.inputs);
                cowInputs.remove(sinkIn);
                sinkIn.target.inputs = cowInputs;
            }
        }
    }
    // remove from upstream operators
    for (PTInput in : oper.inputs) {
        in.source.sinks.remove(in);
    }
    for (HostOperatorSet s : oper.groupings.values()) {
        s.getOperatorSet().remove(oper);
    }
    // remove checkpoint states
    try {
        synchronized (oper.checkpoints) {
            for (Checkpoint checkpoint : oper.checkpoints) {
                oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).delete(oper.id, checkpoint.windowId);
            }
        }
    } catch (IOException e) {
        LOG.warn("Failed to remove state for " + oper, e);
    }
    List<PTOperator> cowList = Lists.newArrayList(oper.container.operators);
    cowList.remove(oper);
    oper.container.operators = cowList;
    this.deployOpers.remove(oper);
    this.undeployOpers.add(oper);
    this.allOperators.remove(oper.id);
    this.ctx.recordEventAsync(new StramEvent.RemoveOperatorEvent(oper.getName(), oper.getId()));
}