Examples with HadoopMultimap org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap used on opensource projects

Example 1 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopSkipListSelfTest method check.

/**
 * Check.
 * @param m The multimap.
 * @param mm The multimap storing expectations.
 * @param vis The multimap to store visitor results.
 * @param taskCtx The task context.
 * @throws Exception On error.
 */
private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis, HadoopTaskContext taskCtx) throws Exception {
    final HadoopTaskInput in = m.input(taskCtx);
    Map<Integer, Collection<Integer>> mmm = mm.asMap();
    int keys = 0;
    int prevKey = Integer.MIN_VALUE;
    while (in.next()) {
        keys++;
        IntWritable k = (IntWritable) in.key();
        assertNotNull(k);
        assertTrue(k.get() > prevKey);
        prevKey = k.get();
        Deque<Integer> vs = new LinkedList<>();
        Iterator<?> it = in.values();
        while (it.hasNext()) vs.addFirst(((IntWritable) it.next()).get());
        Collection<Integer> exp = mmm.get(k.get());
        assertEquals(exp, vs);
    }
    assertEquals(mmm.size(), keys);
    // !        assertEquals(m.keys(), keys);
    // Check visitor.
    final byte[] buf = new byte[4];
    final GridDataInput dataInput = new GridUnsafeDataInput();
    m.visit(false, new HadoopMultimap.Visitor() {

        /**
         */
        IntWritable key = new IntWritable();

        /**
         */
        IntWritable val = new IntWritable();

        @Override
        public void onKey(long keyPtr, int keySize) {
            read(keyPtr, keySize, key);
        }

        @Override
        public void onValue(long valPtr, int valSize) {
            read(valPtr, valSize, val);
            vis.put(key.get(), val.get());
        }

        private void read(long ptr, int size, Writable w) {
            assert size == 4 : size;
            GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
            dataInput.bytes(buf, size);
            try {
                w.readFields(dataInput);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    });
    // X.println("vis: " + vis);
    assertEquals(mm, vis);
    in.close();
}

Example 2 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopSkipListSelfTest method testMapSimple.

/**
 * @throws Exception On error.
 */
public void testMapSimple() throws Exception {
    GridUnsafeMemory mem = new GridUnsafeMemory(0);
    // mem.listen(new GridOffHeapEventListener() {
    // @Override public void onEvent(GridOffHeapEvent evt) {
    // if (evt == GridOffHeapEvent.ALLOCATE)
    // U.dumpStack();
    // }
    // });
    Random rnd = new Random();
    int mapSize = 16 << rnd.nextInt(6);
    HadoopJobInfo job = new JobInfo();
    HadoopTaskContext taskCtx = new TaskContext();
    HadoopMultimap m = new HadoopSkipList(job, mem);
    HadoopMultimap.Adder a = m.startAdding(taskCtx);
    Multimap<Integer, Integer> mm = ArrayListMultimap.create();
    Multimap<Integer, Integer> vis = ArrayListMultimap.create();
    for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
        int key = rnd.nextInt(mapSize);
        int val = rnd.nextInt();
        a.write(new IntWritable(key), new IntWritable(val));
        mm.put(key, val);
        X.println("k: " + key + " v: " + val);
        a.close();
        check(m, mm, vis, taskCtx);
        a = m.startAdding(taskCtx);
    }
    // a.add(new IntWritable(10), new IntWritable(2));
    // mm.put(10, 2);
    // check(m, mm);
    a.close();
    X.println("Alloc: " + mem.allocatedSize());
    m.close();
    assertEquals(0, mem.allocatedSize());
}

Example 3 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopShuffleJob method input.

/**
 * @param taskCtx Task context.
 * @return Input.
 * @throws IgniteCheckedException If failed.
 */
@SuppressWarnings("unchecked")
public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    switch(taskCtx.taskInfo().type()) {
        case REDUCE:
            int reducer = taskCtx.taskInfo().taskNumber();
            HadoopMultimap m = locMaps.get(reducer);
            if (m != null)
                return m.input(taskCtx);
            return new // Empty input.
            HadoopTaskInput() {

                @Override
                public boolean next() {
                    return false;
                }

                @Override
                public Object key() {
                    throw new IllegalStateException();
                }

                @Override
                public Iterator<?> values() {
                    throw new IllegalStateException();
                }

                @Override
                public void close() {
                // No-op.
                }
            };
        default:
            throw new IllegalStateException("Illegal type: " + taskCtx.taskInfo().type());
    }
}

Example 4 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopSkipListSelfTest method testMultiThreaded.

/**
 * @throws Exception if failed.
 */
public void testMultiThreaded() throws Exception {
    GridUnsafeMemory mem = new GridUnsafeMemory(0);
    X.println("___ Started");
    Random rnd = new GridRandom();
    for (int i = 0; i < 20; i++) {
        HadoopJobInfo job = new JobInfo();
        final HadoopTaskContext taskCtx = new TaskContext();
        final HadoopMultimap m = new HadoopSkipList(job, mem);
        final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
        X.println("___ MT");
        multithreaded(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                X.println("___ TH in");
                Random rnd = new GridRandom();
                IntWritable key = new IntWritable();
                IntWritable val = new IntWritable();
                HadoopMultimap.Adder a = m.startAdding(taskCtx);
                for (int i = 0; i < 50000; i++) {
                    int k = rnd.nextInt(32000);
                    int v = rnd.nextInt();
                    key.set(k);
                    val.set(v);
                    a.write(key, val);
                    Collection<Integer> list = mm.get(k);
                    if (list == null) {
                        list = new ConcurrentLinkedQueue<>();
                        Collection<Integer> old = mm.putIfAbsent(k, list);
                        if (old != null)
                            list = old;
                    }
                    list.add(v);
                }
                a.close();
                X.println("___ TH out");
                return null;
            }
        }, 3 + rnd.nextInt(27));
        HadoopTaskInput in = m.input(taskCtx);
        int prevKey = Integer.MIN_VALUE;
        while (in.next()) {
            IntWritable key = (IntWritable) in.key();
            assertTrue(key.get() > prevKey);
            prevKey = key.get();
            Iterator<?> valsIter = in.values();
            Collection<Integer> vals = mm.remove(key.get());
            assertNotNull(vals);
            while (valsIter.hasNext()) {
                IntWritable val = (IntWritable) valsIter.next();
                assertTrue(vals.remove(val.get()));
            }
            assertTrue(vals.isEmpty());
        }
        in.close();
        m.close();
        assertEquals(0, mem.allocatedSize());
    }
}

Example 5 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopShuffleJob method onShuffleMessage.

/**
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheckedException {
    assert msg.buffer() != null;
    assert msg.offset() > 0;
    HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        final GridUnsafeDataInput dataInput = new GridUnsafeDataInput();
        final UnsafeValue val = new UnsafeValue(msg.buffer());
        msg.visit(new HadoopShuffleMessage.Visitor() {

            /**
             */
            private HadoopMultimap.Key key;

            @Override
            public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException {
                dataInput.bytes(buf, off, off + len);
                key = adder.addKey(dataInput, key);
            }

            @Override
            public void onValue(byte[] buf, int off, int len) {
                val.off = off;
                val.size = len;
                key.add(val);
            }
        });
    }
    if (embedded) {
        // No immediate response.
        if (localShuffleState(src).onShuffleMessage())
            sendFinishResponse(src, msg.jobId());
    } else
        // Response for every message.
        io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
}