Examples with HadoopMapReducePlanTopology org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology used on opensource projects

Example 1 with HadoopMapReducePlanTopology

use of org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology in project ignite by apache.

the class IgniteHadoopWeightedMapReducePlanner method preparePlan.

/** {@inheritDoc} */
@Override
public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes, @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
    List<HadoopInputSplit> splits = HadoopCommonUtils.sortInputSplits(job.input());
    int reducerCnt = job.reducers();
    if (reducerCnt < 0)
        throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
    HadoopMapReducePlanTopology top = topology(nodes);
    Mappers mappers = assignMappers(splits, top);
    Map<UUID, int[]> reducers = assignReducers(splits, top, mappers, reducerCnt);
    return new HadoopDefaultMapReducePlan(mappers.nodeToSplits, reducers);
}

Example 2 with HadoopMapReducePlanTopology

use of org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology in project ignite by apache.

the class HadoopAbstractMapReducePlanner method topology.

/**
     * Create plan topology.
     *
     * @param nodes Topology nodes.
     * @return Plan topology.
     */
protected static HadoopMapReducePlanTopology topology(Collection<ClusterNode> nodes) {
    Map<String, HadoopMapReducePlanGroup> macsMap = new HashMap<>(nodes.size());
    Map<UUID, HadoopMapReducePlanGroup> idToGrp = new HashMap<>(nodes.size());
    Map<String, HadoopMapReducePlanGroup> hostToGrp = new HashMap<>(nodes.size());
    for (ClusterNode node : nodes) {
        String macs = node.attribute(ATTR_MACS);
        HadoopMapReducePlanGroup grp = macsMap.get(macs);
        if (grp == null) {
            grp = new HadoopMapReducePlanGroup(node, macs);
            macsMap.put(macs, grp);
        } else
            grp.add(node);
        idToGrp.put(node.id(), grp);
        for (String host : node.addresses()) {
            HadoopMapReducePlanGroup hostGrp = hostToGrp.get(host);
            if (hostGrp == null)
                hostToGrp.put(host, grp);
            else
                assert hostGrp == grp;
        }
    }
    return new HadoopMapReducePlanTopology(new ArrayList<>(macsMap.values()), idToGrp, hostToGrp);
}