Examples with Ignite org.apache.ignite.Ignite used on opensource projects

Example 61 with Ignite

use of org.apache.ignite.Ignite in project ignite by apache.

the class DynamicColumnsAbstractConcurrentSelfTest method testConcurrentOperationsAndNodeStartStopMultithreaded.

/**
 * Test concurrent node start/stop along with add/drop column operations. Nothing should hang.
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("StringConcatenationInLoop")
public void testConcurrentOperationsAndNodeStartStopMultithreaded() throws Exception {
    // Start several stable nodes.
    ignitionStart(serverConfiguration(1));
    ignitionStart(serverConfiguration(2));
    ignitionStart(serverConfiguration(3, true));
    final IgniteEx cli = ignitionStart(clientConfiguration(4));
    createSqlCache(cli);
    run(cli, createSql);
    final AtomicBoolean stopped = new AtomicBoolean();
    // Start node start/stop worker.
    final AtomicInteger nodeIdx = new AtomicInteger(4);
    final AtomicInteger dynColCnt = new AtomicInteger();
    IgniteInternalFuture startStopFut = multithreadedAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            boolean exists = false;
            int lastIdx = 0;
            while (!stopped.get()) {
                if (exists) {
                    stopGrid(lastIdx);
                    exists = false;
                } else {
                    lastIdx = nodeIdx.incrementAndGet();
                    IgniteConfiguration cfg;
                    switch(ThreadLocalRandom.current().nextInt(0, 3)) {
                        case 1:
                            cfg = serverConfiguration(lastIdx, false);
                            break;
                        case 2:
                            cfg = serverConfiguration(lastIdx, true);
                            break;
                        default:
                            cfg = clientConfiguration(lastIdx);
                    }
                    ignitionStart(cfg);
                    exists = true;
                }
                Thread.sleep(ThreadLocalRandom.current().nextLong(500L, 1500L));
            }
            return null;
        }
    }, 1);
    final GridConcurrentHashSet<Integer> fields = new GridConcurrentHashSet<>();
    IgniteInternalFuture idxFut = multithreadedAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            while (!stopped.get()) {
                Ignite node = grid(ThreadLocalRandom.current().nextInt(1, 5));
                IgniteInternalFuture fut;
                int fieldNum = ThreadLocalRandom.current().nextInt(0, dynColCnt.get() + 1);
                boolean removed = fields.remove(fieldNum);
                if (removed)
                    fut = dropCols(node, QueryUtils.DFLT_SCHEMA, "newCol" + fieldNum);
                else {
                    fieldNum = dynColCnt.getAndIncrement();
                    fut = addCols(node, QueryUtils.DFLT_SCHEMA, c("newCol" + fieldNum, Integer.class.getName()));
                }
                try {
                    fut.get();
                    if (!removed)
                        fields.add(fieldNum);
                } catch (SchemaOperationException e) {
                // No-op.
                } catch (Exception e) {
                    fail("Unexpected exception: " + e);
                }
            }
            return null;
        }
    }, 1);
    Thread.sleep(TEST_DUR);
    stopped.set(true);
    // Make sure nothing hanged.
    startStopFut.get();
    idxFut.get();
    // Make sure cache is operational at this point.
    createSqlCache(cli);
    QueryField[] expCols = new QueryField[fields.size()];
    // Too many index columns kills indexing internals, have to limit number of the columns
    // to build the index on.
    int idxColsCnt = Math.min(300, expCols.length);
    Integer[] args = new Integer[idxColsCnt];
    String updQry = "UPDATE " + TBL_NAME + " SET ";
    String idxQry = "CREATE INDEX idx ON " + TBL_NAME + '(';
    Integer[] sorted = fields.toArray(new Integer[fields.size()]);
    Arrays.sort(sorted);
    for (int i = 0; i < expCols.length; i++) {
        int fieldNum = sorted[i];
        expCols[i] = c("newCol" + fieldNum, Integer.class.getName());
        if (i >= idxColsCnt)
            continue;
        if (i > 0) {
            updQry += ", ";
            idxQry += ", ";
        }
        updQry += "\"newCol" + fieldNum + "\" = id + ?";
        idxQry += "\"newCol" + fieldNum + '"';
        args[i] = i;
    }
    idxQry += ')';
    checkTableState(cli, QueryUtils.DFLT_SCHEMA, TBL_NAME, expCols);
    put(cli, 0, 500);
    run(cli.cache(CACHE_NAME), updQry, (Object[]) args);
    run(cli, idxQry);
    run(cli, "DROP INDEX idx");
}

Example 62 with Ignite

use of org.apache.ignite.Ignite in project ignite by apache.

the class SplitDataGenerator method testByGen.

/**
 */
<D extends ContinuousRegionInfo> void testByGen(int totalPts, IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc, IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc, IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
    List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
    Collections.shuffle(lst, rnd);
    SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
    Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
    int i = 0;
    for (IgniteBiTuple<Integer, V> bt : lst) {
        byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
        byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
        m.setRow(i, bt.get2().getStorage().data());
        i++;
    }
    ColumnDecisionTreeTrainer<D> trainer = new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
    DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
    byRegion.keySet().forEach(k -> mdl.apply(byRegion.get(k).get(0).features()));
}

Example 63 with Ignite

use of org.apache.ignite.Ignite in project ignite by apache.

the class IgniteCacheDistributedJoinTest method beforeTestsStarted.

/**
 * {@inheritDoc}
 */
@Override
protected void beforeTestsStarted() throws Exception {
    startGridsMultiThreaded(4);
    awaitPartitionMapExchange();
    conn = DriverManager.getConnection("jdbc:h2:mem:");
    Statement s = conn.createStatement();
    s.execute("create schema a");
    s.execute("create schema b");
    s.execute("create schema c");
    s.execute("create table a.a(a bigint, b bigint, c bigint)");
    s.execute("create table b.b(a bigint, b bigint, c bigint)");
    s.execute("create table c.c(a bigint, b bigint, c bigint)");
    s.execute("create index on a.a(a)");
    s.execute("create index on a.a(b)");
    s.execute("create index on a.a(c)");
    s.execute("create index on b.b(a)");
    s.execute("create index on b.b(b)");
    s.execute("create index on b.b(c)");
    s.execute("create index on c.c(a)");
    s.execute("create index on c.c(b)");
    s.execute("create index on c.c(c)");
    GridRandom rnd = new GridRandom();
    Ignite ignite = ignite(0);
    IgniteCache<Integer, A> a = ignite.cache("a");
    IgniteCache<Integer, B> b = ignite.cache("b");
    IgniteCache<Integer, C> c = ignite.cache("c");
    for (int i = 0; i < 100; i++) {
        a.put(i, insert(s, new A(rnd.nextInt(50), rnd.nextInt(100), rnd.nextInt(150))));
        b.put(i, insert(s, new B(rnd.nextInt(100), rnd.nextInt(50), rnd.nextInt(150))));
        c.put(i, insert(s, new C(rnd.nextInt(150), rnd.nextInt(100), rnd.nextInt(50))));
    }
    checkSameResult(s, a, "select a, count(*) from a group by a order by a");
    checkSameResult(s, a, "select b, count(*) from a group by b order by b");
    checkSameResult(s, a, "select c, count(*) from a group by c order by c");
    checkSameResult(s, b, "select a, count(*) from b group by a order by a");
    checkSameResult(s, b, "select b, count(*) from b group by b order by b");
    checkSameResult(s, b, "select c, count(*) from b group by c order by c");
    checkSameResult(s, c, "select a, count(*) from c group by a order by a");
    checkSameResult(s, c, "select b, count(*) from c group by b order by b");
    checkSameResult(s, c, "select c, count(*) from c group by c order by c");
    s.close();
}

Example 64 with Ignite

use of org.apache.ignite.Ignite in project ignite by apache.

the class IgniteSqlSegmentedIndexSelfTest method checkLocalQueryWithSegmentedIndex.

/**
 * Test local query.
 *
 * @throws Exception If failed.
 */
public void checkLocalQueryWithSegmentedIndex() throws Exception {
    for (int i = 0; i < nodesCount(); i++) {
        final Ignite node = ignite(i);
        IgniteCache<Integer, Person> c1 = node.cache(PERSON_CAHE_NAME);
        IgniteCache<Integer, Organization> c2 = node.cache(ORG_CACHE_NAME);
        Set<Integer> locOrgIds = new HashSet<>();
        for (Cache.Entry<Integer, Organization> e : c2.localEntries()) locOrgIds.add(e.getKey());
        long expPersons = 0;
        for (Cache.Entry<Integer, Person> e : c1.localEntries()) {
            final Integer orgId = e.getValue().orgId;
            if (locOrgIds.contains(orgId))
                expPersons++;
        }
        String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
        List<List<?>> res = c1.query(new SqlFieldsQuery(select0).setLocal(true)).getAll();
        assertEquals(expPersons, res.size());
    }
}

Example 65 with Ignite

use of org.apache.ignite.Ignite in project ignite by apache.

the class IgniteSqlSegmentedIndexSelfTest method checkLocalSizeQueryWithSegmentedIndex.

/**
 * Verifies that local <code>select count(*)</code> query returns a correct result.
 *
 * @throws Exception If failed.
 */
public void checkLocalSizeQueryWithSegmentedIndex() throws Exception {
    for (int i = 0; i < nodesCount(); i++) {
        final Ignite node = ignite(i);
        IgniteCache<Integer, Person> c1 = node.cache(PERSON_CAHE_NAME);
        IgniteCache<Integer, Organization> c2 = node.cache(ORG_CACHE_NAME);
        Set<Integer> locOrgIds = new HashSet<>();
        for (Cache.Entry<Integer, Organization> e : c2.localEntries()) locOrgIds.add(e.getKey());
        int expPersons = 0;
        for (Cache.Entry<Integer, Person> e : c1.localEntries()) {
            final Integer orgId = e.getValue().orgId;
            if (locOrgIds.contains(orgId))
                expPersons++;
        }
        String select0 = "select count(*) from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
        List<List<?>> res = c1.query(new SqlFieldsQuery(select0).setLocal(true)).getAll();
        assertEquals((long) expPersons, res.get(0).get(0));
    }
}