Examples with HadoopTaskInput org.apache.ignite.internal.processors.hadoop.HadoopTaskInput used on opensource projects

Example 6 with HadoopTaskInput

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskInput in project ignite by apache.

the class HadoopRunnableTask method runTask.

/**
 * @param perfCntr Performance counter.
 * @throws IgniteCheckedException If failed.
 */
private void runTask(HadoopPerformanceCounter perfCntr) throws IgniteCheckedException {
    if (cancelled)
        throw new HadoopTaskCancelledException("Task cancelled.");
    try (HadoopTaskOutput out = createOutputInternal(ctx);
        HadoopTaskInput in = createInputInternal(ctx)) {
        ctx.input(in);
        ctx.output(out);
        perfCntr.onTaskStart(ctx.taskInfo(), U.currentTimeMillis());
        ctx.run();
    }
}

Example 7 with HadoopTaskInput

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskInput in project ignite by apache.

the class HadoopV1ReduceTask method run.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    HadoopJobEx job = taskCtx.job();
    HadoopV2TaskContext taskCtx0 = (HadoopV2TaskContext) taskCtx;
    if (!reduce && taskCtx.taskInfo().hasMapperIndex())
        HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
    else
        HadoopMapperUtils.clearMapperIndex();
    try {
        JobConf jobConf = taskCtx0.jobConf();
        HadoopTaskInput input = taskCtx.input();
        HadoopV1OutputCollector collector = null;
        try {
            collector = collector(jobConf, taskCtx0, reduce || !job.info().hasReducer(), fileName(), taskCtx0.attemptId());
            Reducer reducer;
            if (reduce)
                reducer = ReflectionUtils.newInstance(jobConf.getReducerClass(), jobConf);
            else
                reducer = ReflectionUtils.newInstance(jobConf.getCombinerClass(), jobConf);
            assert reducer != null;
            try {
                try {
                    while (input.next()) {
                        if (isCancelled())
                            throw new HadoopTaskCancelledException("Reduce task cancelled.");
                        reducer.reduce(input.key(), input.values(), collector, Reporter.NULL);
                    }
                    if (!reduce)
                        taskCtx.onMapperFinished();
                } finally {
                    reducer.close();
                }
            } finally {
                collector.closeWriter();
            }
            collector.commit();
        } catch (Exception e) {
            if (collector != null)
                collector.abort();
            throw new IgniteCheckedException(e);
        }
    } finally {
        if (!reduce)
            HadoopMapperUtils.clearMapperIndex();
    }
}

Example 8 with HadoopTaskInput

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskInput in project ignite by apache.

the class HadoopHashMapSelfTest method check.

private void check(HadoopHashMultimap m, Multimap<Integer, Integer> mm, HadoopTaskContext taskCtx) throws Exception {
    final HadoopTaskInput in = m.input(taskCtx);
    Map<Integer, Collection<Integer>> mmm = mm.asMap();
    int keys = 0;
    while (in.next()) {
        keys++;
        IntWritable k = (IntWritable) in.key();
        assertNotNull(k);
        ArrayList<Integer> vs = new ArrayList<>();
        Iterator<?> it = in.values();
        while (it.hasNext()) vs.add(((IntWritable) it.next()).get());
        Collection<Integer> exp = mmm.get(k.get());
        assertEquals(sorted(exp), sorted(vs));
    }
    X.println("keys: " + keys + " cap: " + m.capacity());
    assertEquals(mmm.size(), keys);
    assertEquals(m.keys(), keys);
    in.close();
}

Example 9 with HadoopTaskInput

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskInput in project ignite by apache.

the class HadoopSkipListSelfTest method testMultiThreaded.

/**
 * @throws Exception if failed.
 */
public void testMultiThreaded() throws Exception {
    GridUnsafeMemory mem = new GridUnsafeMemory(0);
    X.println("___ Started");
    Random rnd = new GridRandom();
    for (int i = 0; i < 20; i++) {
        HadoopJobInfo job = new JobInfo();
        final HadoopTaskContext taskCtx = new TaskContext();
        final HadoopMultimap m = new HadoopSkipList(job, mem);
        final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
        X.println("___ MT");
        multithreaded(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                X.println("___ TH in");
                Random rnd = new GridRandom();
                IntWritable key = new IntWritable();
                IntWritable val = new IntWritable();
                HadoopMultimap.Adder a = m.startAdding(taskCtx);
                for (int i = 0; i < 50000; i++) {
                    int k = rnd.nextInt(32000);
                    int v = rnd.nextInt();
                    key.set(k);
                    val.set(v);
                    a.write(key, val);
                    Collection<Integer> list = mm.get(k);
                    if (list == null) {
                        list = new ConcurrentLinkedQueue<>();
                        Collection<Integer> old = mm.putIfAbsent(k, list);
                        if (old != null)
                            list = old;
                    }
                    list.add(v);
                }
                a.close();
                X.println("___ TH out");
                return null;
            }
        }, 3 + rnd.nextInt(27));
        HadoopTaskInput in = m.input(taskCtx);
        int prevKey = Integer.MIN_VALUE;
        while (in.next()) {
            IntWritable key = (IntWritable) in.key();
            assertTrue(key.get() > prevKey);
            prevKey = key.get();
            Iterator<?> valsIter = in.values();
            Collection<Integer> vals = mm.remove(key.get());
            assertNotNull(vals);
            while (valsIter.hasNext()) {
                IntWritable val = (IntWritable) valsIter.next();
                assertTrue(vals.remove(val.get()));
            }
            assertTrue(vals.isEmpty());
        }
        in.close();
        m.close();
        assertEquals(0, mem.allocatedSize());
    }
}