Examples with HadoopTaskContext org.apache.ignite.internal.processors.hadoop.HadoopTaskContext used on opensource projects

Example 6 with HadoopTaskContext

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskContext in project ignite by apache.

the class HadoopSkipListSelfTest method testMultiThreaded.

/**
 * @throws Exception if failed.
 */
public void testMultiThreaded() throws Exception {
    GridUnsafeMemory mem = new GridUnsafeMemory(0);
    X.println("___ Started");
    Random rnd = new GridRandom();
    for (int i = 0; i < 20; i++) {
        HadoopJobInfo job = new JobInfo();
        final HadoopTaskContext taskCtx = new TaskContext();
        final HadoopMultimap m = new HadoopSkipList(job, mem);
        final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
        X.println("___ MT");
        multithreaded(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                X.println("___ TH in");
                Random rnd = new GridRandom();
                IntWritable key = new IntWritable();
                IntWritable val = new IntWritable();
                HadoopMultimap.Adder a = m.startAdding(taskCtx);
                for (int i = 0; i < 50000; i++) {
                    int k = rnd.nextInt(32000);
                    int v = rnd.nextInt();
                    key.set(k);
                    val.set(v);
                    a.write(key, val);
                    Collection<Integer> list = mm.get(k);
                    if (list == null) {
                        list = new ConcurrentLinkedQueue<>();
                        Collection<Integer> old = mm.putIfAbsent(k, list);
                        if (old != null)
                            list = old;
                    }
                    list.add(v);
                }
                a.close();
                X.println("___ TH out");
                return null;
            }
        }, 3 + rnd.nextInt(27));
        HadoopTaskInput in = m.input(taskCtx);
        int prevKey = Integer.MIN_VALUE;
        while (in.next()) {
            IntWritable key = (IntWritable) in.key();
            assertTrue(key.get() > prevKey);
            prevKey = key.get();
            Iterator<?> valsIter = in.values();
            Collection<Integer> vals = mm.remove(key.get());
            assertNotNull(vals);
            while (valsIter.hasNext()) {
                IntWritable val = (IntWritable) valsIter.next();
                assertTrue(vals.remove(val.get()));
            }
            assertTrue(vals.isEmpty());
        }
        in.close();
        m.close();
        assertEquals(0, mem.allocatedSize());
    }
}

Example 7 with HadoopTaskContext

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskContext in project ignite by apache.

the class HadoopShuffleJob method onShuffleMessage.

/**
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheckedException {
    assert msg.buffer() != null;
    assert msg.offset() > 0;
    HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        final GridUnsafeDataInput dataInput = new GridUnsafeDataInput();
        final UnsafeValue val = new UnsafeValue(msg.buffer());
        msg.visit(new HadoopShuffleMessage.Visitor() {

            /**
             */
            private HadoopMultimap.Key key;

            @Override
            public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException {
                dataInput.bytes(buf, off, off + len);
                key = adder.addKey(dataInput, key);
            }

            @Override
            public void onValue(byte[] buf, int off, int len) {
                val.off = off;
                val.size = len;
                key.add(val);
            }
        });
    }
    if (embedded) {
        // No immediate response.
        if (localShuffleState(src).onShuffleMessage())
            sendFinishResponse(src, msg.jobId());
    } else
        // Response for every message.
        io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
}

Example 8 with HadoopTaskContext

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskContext in project ignite by apache.

the class HadoopShuffleJob method onDirectShuffleMessage.

/**
 * Process shuffle message.
 *
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
    byte[] buf = extractBuffer(msg);
    assert buf != null;
    int rdc = msg.reducer();
    HadoopTaskContext taskCtx = locReducersCtx.get(rdc).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(rdc, U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, rdc);
    HadoopSerialization keySer = taskCtx.keySerialization();
    HadoopSerialization valSer = taskCtx.valueSerialization();
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        HadoopDirectDataInput in = new HadoopDirectDataInput(buf);
        Object key = null;
        Object val = null;
        for (int i = 0; i < msg.count(); i++) {
            key = keySer.read(in, key);
            val = valSer.read(in, val);
            adder.write(key, val);
        }
    }
    if (localShuffleState(src).onShuffleMessage())
        sendFinishResponse(src, msg.jobId());
}

Example 9 with HadoopTaskContext

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskContext in project ignite by apache.

the class HadoopEmbeddedTaskExecutor method run.

/**
 * {@inheritDoc}
 */
@Override
public void run(final HadoopJobEx job, Collection<HadoopTaskInfo> tasks) throws IgniteCheckedException {
    if (log.isDebugEnabled())
        log.debug("Submitting tasks for local execution [locNodeId=" + ctx.localNodeId() + ", tasksCnt=" + tasks.size() + ']');
    Collection<HadoopRunnableTask> executedTasks = jobs.get(job.id());
    if (executedTasks == null) {
        executedTasks = new GridConcurrentHashSet<>();
        Collection<HadoopRunnableTask> extractedCol = jobs.put(job.id(), executedTasks);
        assert extractedCol == null;
    }
    final Collection<HadoopRunnableTask> finalExecutedTasks = executedTasks;
    for (final HadoopTaskInfo info : tasks) {
        assert info != null;
        HadoopRunnableTask task = new HadoopRunnableTask(log, job, ctx.shuffle().memory(), info, ctx.localNodeId()) {

            @Override
            protected void onTaskFinished(HadoopTaskStatus status) {
                if (log.isDebugEnabled())
                    log.debug("Finished task execution [jobId=" + job.id() + ", taskInfo=" + info + ", " + "waitTime=" + waitTime() + ", execTime=" + executionTime() + ']');
                finalExecutedTasks.remove(this);
                jobTracker.onTaskFinished(info, status);
            }

            @Override
            protected HadoopTaskInput createInput(HadoopTaskContext taskCtx) throws IgniteCheckedException {
                return ctx.shuffle().input(taskCtx);
            }

            @Override
            protected HadoopTaskOutput createOutput(HadoopTaskContext taskCtx) throws IgniteCheckedException {
                return ctx.shuffle().output(taskCtx);
            }
        };
        executedTasks.add(task);
        exec.submit(task);
    }
}

Example 10 with HadoopTaskContext

use of org.apache.ignite.internal.processors.hadoop.HadoopTaskContext in project ignite by apache.

the class HadoopV2Job method getTaskContext.

/**
 * {@inheritDoc}
 */
@SuppressWarnings({ "unchecked", "MismatchedQueryAndUpdateOfCollection" })
@Override
public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException {
    T2<HadoopTaskType, Integer> locTaskId = new T2<>(info.type(), info.taskNumber());
    GridFutureAdapter<HadoopTaskContext> fut = ctxs.get(locTaskId);
    if (fut != null)
        return fut.get();
    GridFutureAdapter<HadoopTaskContext> old = ctxs.putIfAbsent(locTaskId, fut = new GridFutureAdapter<>());
    if (old != null)
        return old.get();
    Class<? extends HadoopTaskContext> cls = taskCtxClsPool.poll();
    try {
        if (cls == null) {
            // If there is no pooled class, then load new one.
            // Note that the classloader identified by the task it was initially created for,
            // but later it may be reused for other tasks.
            HadoopClassLoader ldr = sharedClsLdr != null ? sharedClsLdr : createClassLoader(HadoopClassLoader.nameForTask(info, false));
            cls = (Class<? extends HadoopTaskContext>) ldr.loadClass(HadoopV2TaskContext.class.getName());
            fullCtxClsQueue.add(cls);
        }
        Constructor<?> ctr = cls.getConstructor(HadoopTaskInfo.class, HadoopJobEx.class, HadoopJobId.class, UUID.class, DataInput.class);
        if (jobConfData == null)
            synchronized (jobConf) {
                if (jobConfData == null) {
                    ByteArrayOutputStream buf = new ByteArrayOutputStream();
                    jobConf.write(new DataOutputStream(buf));
                    jobConfData = buf.toByteArray();
                }
            }
        HadoopTaskContext res = (HadoopTaskContext) ctr.newInstance(info, this, jobId, locNodeId, new DataInputStream(new ByteArrayInputStream(jobConfData)));
        fut.onDone(res);
        return res;
    } catch (Throwable e) {
        IgniteCheckedException te = transformException(e);
        fut.onDone(te);
        if (e instanceof Error)
            throw (Error) e;
        throw te;
    }
}