Examples with IgniteBiTuple org.apache.ignite.lang.IgniteBiTuple used on opensource projects

Example 76 with IgniteBiTuple

use of org.apache.ignite.lang.IgniteBiTuple in project ignite by apache.

the class HadoopExternalTaskExecutor method run.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("ConstantConditions")
@Override
public void run(final HadoopJobEx job, final Collection<HadoopTaskInfo> tasks) throws IgniteCheckedException {
    if (!busyLock.tryReadLock()) {
        if (log.isDebugEnabled())
            log.debug("Failed to start hadoop tasks (grid is stopping, will ignore).");
        return;
    }
    try {
        HadoopProcess proc = runningProcsByJobId.get(job.id());
        HadoopTaskType taskType = F.first(tasks).type();
        if (taskType == HadoopTaskType.SETUP || taskType == HadoopTaskType.ABORT || taskType == HadoopTaskType.COMMIT) {
            if (proc == null || proc.terminated()) {
                runningProcsByJobId.remove(job.id(), proc);
                // Start new process for ABORT task since previous processes were killed.
                proc = startProcess(job, jobTracker.plan(job.id()));
                if (log.isDebugEnabled())
                    log.debug("Starting new process for maintenance task [jobId=" + job.id() + ", proc=" + proc + ", taskType=" + taskType + ']');
            }
        } else
            assert proc != null : "Missing started process for task execution request: " + job.id() + ", tasks=" + tasks;
        final HadoopProcess proc0 = proc;
        proc.initFut.listen(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>>>() {

            @Override
            public void apply(IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>> f) {
                if (!busyLock.tryReadLock())
                    return;
                try {
                    f.get();
                    proc0.addTasks(tasks);
                    if (log.isDebugEnabled())
                        log.debug("Sending task execution request to child process [jobId=" + job.id() + ", proc=" + proc0 + ", tasks=" + tasks + ']');
                    sendExecutionRequest(proc0, job, tasks);
                } catch (IgniteCheckedException e) {
                    notifyTasksFailed(tasks, FAILED, e);
                } finally {
                    busyLock.readUnlock();
                }
            }
        });
    } finally {
        busyLock.readUnlock();
    }
}

Example 77 with IgniteBiTuple

use of org.apache.ignite.lang.IgniteBiTuple in project ignite by apache.

the class GridOffHeapPartitionedMapAbstractSelfTest method testIteratorMultithreaded.

/**
 * @throws Exception If failed.
 */
public void testIteratorMultithreaded() throws Exception {
    initCap = 10;
    map = newMap();
    final AtomicInteger rehashes = new AtomicInteger();
    final AtomicInteger releases = new AtomicInteger();
    map.eventListener(new GridOffHeapEventListener() {

        @Override
        public void onEvent(GridOffHeapEvent evt) {
            switch(evt) {
                case REHASH:
                    rehashes.incrementAndGet();
                    break;
                case RELEASE:
                    releases.incrementAndGet();
                    break;
                // No-op.
                default:
            }
        }
    });
    final int max = 1024;
    final ConcurrentMap<String, String> m = new ConcurrentHashMap<>(max * parts);
    final AtomicInteger part = new AtomicInteger();
    multithreaded(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            int p = part.getAndIncrement();
            for (int i = 0; i < max; i++) {
                String key = string();
                String val = string();
                // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']');
                String old = m.putIfAbsent(key, val);
                if (old != null)
                    val = old;
                assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes()));
                assertTrue(map.contains(p, hash(key), key.getBytes()));
                assertNotNull(map.get(p, hash(key), key.getBytes()));
                assertEquals(new String(map.get(p, hash(key), key.getBytes())), val);
            }
            try (GridCloseableIterator<IgniteBiTuple<byte[], byte[]>> it = map.iterator()) {
                while (it.hasNext()) {
                    IgniteBiTuple<byte[], byte[]> t = it.next();
                    String k = new String(t.get1());
                    String v = new String(t.get2());
                    // info("Entry [k=" + k + ", v=" + v + ']');
                    assertEquals(m.get(k), v);
                }
            }
            return null;
        }
    }, parts);
    int cnt = 0;
    try (GridCloseableIterator<IgniteBiTuple<byte[], byte[]>> it = map.iterator()) {
        while (it.hasNext()) {
            IgniteBiTuple<byte[], byte[]> t = it.next();
            String k = new String(t.get1());
            String v = new String(t.get2());
            // info("Entry [k=" + k + ", v=" + v + ']');
            assertEquals(m.get(k), v);
            cnt++;
        }
    }
    assertEquals(map.size(), cnt);
    assertEquals(max * parts, map.size());
    info("Stats [size=" + map.size() + ", rehashes=" + rehashes + ", releases=" + releases + ']');
    assertTrue(rehashes.get() > 0);
    assertEquals(rehashes.get(), releases.get());
}

Example 78 with IgniteBiTuple

use of org.apache.ignite.lang.IgniteBiTuple in project ignite by apache.

the class GridOffHeapPartitionedMapAbstractSelfTest method testIteratorRemoveMultithreaded.

/**
 * @throws Exception If failed.
 */
public void testIteratorRemoveMultithreaded() throws Exception {
    initCap = 10;
    map = newMap();
    final int max = 1024;
    final Map<String, String> m = new ConcurrentHashMap<>(max * parts);
    for (int i = 0; i < max; i++) {
        String key = string();
        String val = string();
        m.put(key, val);
        map.put(0, hash(key), key.getBytes(), val.getBytes());
    }
    final AtomicBoolean running = new AtomicBoolean(true);
    IgniteInternalFuture<?> iterFut = multithreadedAsync(new Runnable() {

        @Override
        public void run() {
            try {
                while (running.get()) {
                    try (GridCloseableIterator<IgniteBiTuple<byte[], byte[]>> it = map.iterator()) {
                        while (it.hasNext()) {
                            IgniteBiTuple<byte[], byte[]> tup = it.next();
                            String key = new String(tup.get1());
                            String val = new String(tup.get2());
                            String exp = m.get(key);
                            assertEquals(exp, val);
                        }
                    }
                }
            } catch (IgniteCheckedException e) {
                fail("Unexpected exception caught: " + e);
            }
        }
    }, 1);
    for (String key : m.keySet()) map.remove(0, hash(key), key.getBytes());
    running.set(false);
    iterFut.get();
    map.destruct();
}

Example 79 with IgniteBiTuple

use of org.apache.ignite.lang.IgniteBiTuple in project ignite by apache.

the class ColumnDecisionTreeTrainerTest method testByGen.

/**
 */
private <D extends ContinuousRegionInfo> void testByGen(int totalPts, HashMap<Integer, Integer> catsInfo, SplitDataGenerator<DenseLocalOnHeapVector> gen, IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc, IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc, IgniteFunction<DoubleStream, Double> regCalc, Random rnd) {
    List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.points(totalPts, (i, rn) -> i).collect(Collectors.toList());
    int featCnt = gen.featuresCnt();
    Collections.shuffle(lst, rnd);
    SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
    Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
    int i = 0;
    for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
        byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
        byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
        m.setRow(i, bt.get2().getStorage().data());
        i++;
    }
    ColumnDecisionTreeTrainer<D> trainer = new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
    DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
    byRegion.keySet().forEach(k -> {
        LabeledVectorDouble sp = byRegion.get(k).get(0);
        Tracer.showAscii(sp.features());
        X.println("Actual and predicted vectors [act=" + sp.label() + " " + ", pred=" + mdl.apply(sp.features()) + "]");
        assert mdl.apply(sp.features()) == sp.doubleLabel();
    });
}

Example 80 with IgniteBiTuple

use of org.apache.ignite.lang.IgniteBiTuple in project ignite by apache.

the class GridRestProcessor method start.

/**
 * {@inheritDoc}
 */
@Override
public void start() throws IgniteCheckedException {
    if (isRestEnabled()) {
        if (notStartOnClient()) {
            U.quietAndInfo(log, "REST protocols do not start on client node. " + "To start the protocols on client node set '-DIGNITE_REST_START_ON_CLIENT=true' system property.");
            return;
        }
        // Register handlers.
        addHandler(new GridCacheCommandHandler(ctx));
        addHandler(new GridTaskCommandHandler(ctx));
        addHandler(new GridTopologyCommandHandler(ctx));
        addHandler(new GridVersionCommandHandler(ctx));
        addHandler(new DataStructuresCommandHandler(ctx));
        addHandler(new QueryCommandHandler(ctx));
        addHandler(new GridLogCommandHandler(ctx));
        addHandler(new GridChangeStateCommandHandler(ctx));
        addHandler(new UserActionCommandHandler(ctx));
        // Start protocols.
        startTcpProtocol();
        startHttpProtocol();
        for (GridRestProtocol proto : protos) {
            Collection<IgniteBiTuple<String, Object>> props = proto.getProperties();
            if (props != null) {
                for (IgniteBiTuple<String, Object> p : props) {
                    String key = p.getKey();
                    if (key == null)
                        continue;
                    if (ctx.hasNodeAttribute(key))
                        throw new IgniteCheckedException("Node attribute collision for attribute [processor=GridRestProcessor, attr=" + key + ']');
                    ctx.addNodeAttribute(key, p.getValue());
                }
            }
        }
    }
}