Examples with CombineFileSplit org.apache.hadoop.mapreduce.lib.input.CombineFileSplit used on opensource projects

Example 6 with CombineFileSplit

use of org.apache.hadoop.mapreduce.lib.input.CombineFileSplit in project hadoop by apache.

the class TestGridMixClasses method getLoadSplit.

private LoadSplit getLoadSplit() throws Exception {
    Path[] files = { new Path("one"), new Path("two") };
    long[] start = { 1, 2 };
    long[] lengths = { 100, 200 };
    String[] locations = { "locOne", "loctwo" };
    CombineFileSplit cfSplit = new CombineFileSplit(files, start, lengths, locations);
    ResourceUsageMetrics metrics = new ResourceUsageMetrics();
    metrics.setCumulativeCpuUsage(200);
    ResourceUsageMetrics[] rMetrics = { metrics };
    double[] reduceBytes = { 8.1d, 8.2d };
    double[] reduceRecords = { 9.1d, 9.2d };
    long[] reduceOutputBytes = { 101L, 102L };
    long[] reduceOutputRecords = { 111L, 112L };
    return new LoadSplit(cfSplit, 2, 1, 4L, 5L, 6L, 7L, reduceBytes, reduceRecords, reduceOutputBytes, reduceOutputRecords, metrics, rMetrics);
}

Example 7 with CombineFileSplit

use of org.apache.hadoop.mapreduce.lib.input.CombineFileSplit in project hadoop by apache.

the class TestFileQueue method testRepeat.

@Test
public void testRepeat() throws Exception {
    final Configuration conf = new Configuration();
    Arrays.fill(loc, "");
    Arrays.fill(start, 0L);
    Arrays.fill(len, BLOCK);
    final ByteArrayOutputStream out = fillVerif();
    final FileQueue q = new FileQueue(new CombineFileSplit(paths, start, len, loc), conf);
    final byte[] verif = out.toByteArray();
    final byte[] check = new byte[2 * NFILES * BLOCK];
    q.read(check, 0, NFILES * BLOCK);
    assertArrayEquals(verif, Arrays.copyOf(check, NFILES * BLOCK));
    final byte[] verif2 = new byte[2 * NFILES * BLOCK];
    System.arraycopy(verif, 0, verif2, 0, verif.length);
    System.arraycopy(verif, 0, verif2, verif.length, verif.length);
    q.read(check, 0, 2 * NFILES * BLOCK);
    assertArrayEquals(verif2, check);
}

Example 8 with CombineFileSplit

use of org.apache.hadoop.mapreduce.lib.input.CombineFileSplit in project hadoop by apache.

the class TestFileQueue method testUneven.

@Test
public void testUneven() throws Exception {
    final Configuration conf = new Configuration();
    Arrays.fill(loc, "");
    Arrays.fill(start, 0L);
    Arrays.fill(len, BLOCK);
    final int B2 = BLOCK / 2;
    for (int i = 0; i < NFILES; i += 2) {
        start[i] += B2;
        len[i] -= B2;
    }
    final FileQueue q = new FileQueue(new CombineFileSplit(paths, start, len, loc), conf);
    final ByteArrayOutputStream out = fillVerif();
    final byte[] verif = out.toByteArray();
    final byte[] check = new byte[NFILES / 2 * BLOCK + NFILES / 2 * B2];
    q.read(check, 0, verif.length);
    assertArrayEquals(verif, Arrays.copyOf(check, verif.length));
    q.read(check, 0, verif.length);
    assertArrayEquals(verif, Arrays.copyOf(check, verif.length));
}