Examples with HadoopInputSplit org.apache.ignite.hadoop.HadoopInputSplit used on opensource projects

Example 11 with HadoopInputSplit

use of org.apache.ignite.hadoop.HadoopInputSplit in project ignite by apache.

the class HadoopWeightedMapReducePlannerTest method testHdfsSplitsAffinity.

/**
 * Test one HDFS splits.
 *
 * @throws Exception If failed.
 */
public void testHdfsSplitsAffinity() throws Exception {
    IgfsMock igfs = LocationsBuilder.create().add(0, NODE_1).add(50, NODE_2).add(100, NODE_3).buildIgfs();
    final List<HadoopInputSplit> splits = new ArrayList<>();
    splits.add(new HadoopFileBlock(new String[] { HOST_1 }, URI.create("hfds://" + HOST_1 + "/x"), 0, 50));
    splits.add(new HadoopFileBlock(new String[] { HOST_2 }, URI.create("hfds://" + HOST_2 + "/x"), 50, 100));
    splits.add(new HadoopFileBlock(new String[] { HOST_3 }, URI.create("hfds://" + HOST_3 + "/x"), 100, 37));
    // The following splits belong to hosts that are out of Ignite topology at all.
    // This means that these splits should be assigned to any least loaded modes:
    splits.add(new HadoopFileBlock(new String[] { HOST_4 }, URI.create("hfds://" + HOST_4 + "/x"), 138, 2));
    splits.add(new HadoopFileBlock(new String[] { HOST_5 }, URI.create("hfds://" + HOST_5 + "/x"), 140, 3));
    final int expReducers = 7;
    HadoopPlannerMockJob job = new HadoopPlannerMockJob(splits, expReducers);
    IgniteHadoopWeightedMapReducePlanner planner = createPlanner(igfs);
    final HadoopMapReducePlan plan = planner.preparePlan(job, NODES, null);
    checkPlanMappers(plan, splits, NODES, true);
    checkPlanReducers(plan, NODES, expReducers, true);
}

Example 12 with HadoopInputSplit

use of org.apache.ignite.hadoop.HadoopInputSplit in project ignite by apache.

the class HadoopWeightedMapReducePlannerTest method checkPlanMappers.

/**
 * Check mappers for the plan.
 *
 * @param plan Plan.
 * @param splits Splits.
 * @param nodes Nodes.
 * @param expectUniformity WHether uniformity is expected.
 */
private static void checkPlanMappers(HadoopMapReducePlan plan, List<HadoopInputSplit> splits, Collection<ClusterNode> nodes, boolean expectUniformity) {
    // Number of mappers should correspomd to the number of input splits:
    assertEquals(splits.size(), plan.mappers());
    if (expectUniformity) {
        // mappers are assigned to all available nodes:
        assertEquals(nodes.size(), plan.mapperNodeIds().size());
        assertEquals(allIds(nodes), plan.mapperNodeIds());
    }
    // Check all splits are covered by mappers:
    Set<HadoopInputSplit> set = new HashSet<>();
    for (UUID id : plan.mapperNodeIds()) {
        Collection<HadoopInputSplit> sp = plan.mappers(id);
        assert sp != null;
        for (HadoopInputSplit s : sp) assertTrue(set.add(s));
    }
    // must be of the same size & contain same elements:
    assertEquals(set, new HashSet<>(splits));
}

Example 13 with HadoopInputSplit

use of org.apache.ignite.hadoop.HadoopInputSplit in project ignite by apache.

the class HadoopCommonUtils method sortInputSplits.

/**
 * Sort input splits by length.
 *
 * @param splits Splits.
 * @return Sorted splits.
 */
public static List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> splits) {
    int id = 0;
    TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
    for (HadoopInputSplit split : splits) {
        long len = split instanceof HadoopFileBlock ? ((HadoopFileBlock) split).length() : 0;
        sortedSplits.add(new SplitSortWrapper(id++, split, len));
    }
    ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
    for (SplitSortWrapper sortedSplit : sortedSplits) res.add(sortedSplit.split);
    return res;
}

Example 14 with HadoopInputSplit

use of org.apache.ignite.hadoop.HadoopInputSplit in project ignite by apache.

the class HadoopJobTracker method allSplits.

/**
 * Gets all input splits for given hadoop map-reduce plan.
 *
 * @param plan Map-reduce plan.
 * @return Collection of all input splits that should be processed.
 */
@SuppressWarnings("ConstantConditions")
private Map<HadoopInputSplit, Integer> allSplits(HadoopMapReducePlan plan) {
    Map<HadoopInputSplit, Integer> res = new HashMap<>();
    int taskNum = 0;
    for (UUID nodeId : plan.mapperNodeIds()) {
        for (HadoopInputSplit split : plan.mappers(nodeId)) {
            if (res.put(split, taskNum++) != null)
                throw new IllegalStateException("Split duplicate.");
        }
    }
    return res;
}

Example 15 with HadoopInputSplit

use of org.apache.ignite.hadoop.HadoopInputSplit in project ignite by apache.

the class HadoopV2Context method getInputSplit.

/**
 * {@inheritDoc}
 */
@Override
public InputSplit getInputSplit() {
    if (inputSplit == null) {
        HadoopInputSplit split = ctx.taskInfo().inputSplit();
        if (split == null)
            return null;
        if (split instanceof HadoopFileBlock) {
            HadoopFileBlock fileBlock = (HadoopFileBlock) split;
            inputSplit = new FileSplit(new Path(fileBlock.file()), fileBlock.start(), fileBlock.length(), null);
        } else {
            try {
                inputSplit = (InputSplit) ((HadoopV2TaskContext) ctx).getNativeSplit(split);
            } catch (IgniteCheckedException e) {
                throw new IllegalStateException(e);
            }
        }
    }
    return inputSplit;
}