Examples with ContinuousQuery org.apache.ignite.cache.query.ContinuousQuery used on opensource projects

Example 81 with ContinuousQuery

use of org.apache.ignite.cache.query.ContinuousQuery in project ignite by apache.

the class CacheContinuousQueryFailoverAbstractSelfTest method testRebalance.

/**
 * Test that during rebalancing correct old value passed to continuous query.
 *
 * @throws Exception If fail.
 */
@Test
public void testRebalance() throws Exception {
    for (int iter = 0; iter < 5; iter++) {
        log.info("Iteration: " + iter);
        final IgniteEx ignite = startGrid(1);
        final CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>("testCache");
        ccfg.setAtomicityMode(atomicityMode());
        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        ccfg.setCacheMode(cacheMode());
        ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
        ccfg.setBackups(2);
        final IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(ccfg);
        final int KEYS = 10_000;
        for (int i = 0; i < KEYS; i++) cache.put(i, i);
        final ContinuousQuery<Integer, Integer> qry = new ContinuousQuery<>();
        final AtomicBoolean err = new AtomicBoolean();
        final AtomicInteger cntr = new AtomicInteger();
        qry.setLocalListener(new CacheEntryUpdatedListener<Integer, Integer>() {

            @Override
            public void onUpdated(final Iterable<CacheEntryEvent<? extends Integer, ? extends Integer>> cacheEntryEvts) {
                try {
                    for (final CacheEntryEvent<? extends Integer, ? extends Integer> evt : cacheEntryEvts) {
                        final Integer oldVal = evt.getOldValue();
                        final Integer val = evt.getValue();
                        assertNotNull("No old value: " + evt, oldVal);
                        assertEquals("Unexpected old value: " + evt, (Integer) (oldVal + 1), val);
                        cntr.incrementAndGet();
                    }
                } catch (Throwable e) {
                    err.set(true);
                    error("Unexpected error: " + e, e);
                }
            }
        });
        final QueryCursor<Cache.Entry<Integer, Integer>> cur = cache.query(qry);
        final CountDownLatch latch = new CountDownLatch(1);
        final IgniteInternalFuture<Object> updFut = GridTestUtils.runAsync(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                latch.await();
                for (int i = 0; i < KEYS && !err.get(); i++) cache.put(i, i + 1);
                return null;
            }
        });
        final IgniteInternalFuture<Object> rebFut = GridTestUtils.runAsync(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                latch.await();
                for (int i = 2; i <= 5 && !err.get(); i++) startGrid(i);
                return null;
            }
        });
        latch.countDown();
        updFut.get();
        rebFut.get();
        assertFalse("Unexpected error during test", err.get());
        assertTrue(cntr.get() > 0);
        cur.close();
        stopAllGrids();
    }
}

Example 82 with ContinuousQuery

use of org.apache.ignite.cache.query.ContinuousQuery in project ignite by apache.

the class CacheContinuousQueryFailoverAbstractSelfTest method testNoEventLossOnTopologyChange.

/**
 * This is failover test detecting CQ event loss while topology changing.
 *
 * @throws Exception If failed.
 */
@Test
public void testNoEventLossOnTopologyChange() throws Exception {
    final int batchLoadSize = 2000;
    final int restartCycles = 5;
    Ignite qryClient = startGrid(0);
    final CacheEventListener4 lsnr = new CacheEventListener4(atomicityMode() == CacheAtomicityMode.ATOMIC);
    ContinuousQuery<Integer, Integer> qry = new ContinuousQuery<>();
    qry.setLocalListener(lsnr);
    IgniteCache<Integer, Integer> cache = qryClient.cache(DEFAULT_CACHE_NAME);
    QueryCursor<?> cur = cache.query(qry);
    int iteration = 0;
    int putCnt = 0;
    int ignoredDupEvts = 0;
    Thread nodeRestartThread = nodeRestartThread(restartCycles, 2_000, 1_000);
    try {
        nodeRestartThread.start();
        while (!Thread.interrupted() && nodeRestartThread.isAlive()) {
            iteration++;
            for (int i = 0; i < batchLoadSize; i++) cache.put(i, iteration);
            putCnt += batchLoadSize;
            log.info("Batch loaded. Iteration: " + iteration);
            final long expCnt = putCnt + ignoredDupEvts;
            GridTestUtils.waitForCondition(new GridAbsPredicate() {

                @Override
                public boolean apply() {
                    return lsnr.count() == expCnt;
                }
            }, 6_000);
            final long cnt = lsnr.count();
            if (cnt != expCnt) {
                StringBuilder sb = new StringBuilder();
                for (int i = 0; i < batchLoadSize; i++) {
                    Integer key = i;
                    Integer val = cache.get(key);
                    if (!F.eq(val, iteration))
                        sb.append("\n\t").append(">>> WRONG CACHE VALUE (lost data?) [key=").append(key).append(", val=").append(val).append(']');
                }
                for (Map.Entry<Integer, Integer> entry : lsnr.eventMap().entrySet()) {
                    Integer key = entry.getKey();
                    Integer val = entry.getValue();
                    if (!F.eq(val, iteration))
                        sb.append("\n\t").append(">>> WRONG LISTENER VALUE (lost event?) [key=").append(key).append(", val=").append(val).append(']');
                }
                String msg = sb.toString();
                // In atomic mode CQ can receive duplicate update events if update retried after fails.
                if (atomicityMode() == CacheAtomicityMode.ATOMIC && msg.isEmpty() && cnt > expCnt)
                    ignoredDupEvts += cnt - expCnt;
                else
                    fail("Unexpected event updates count: EXPECTED=" + expCnt + ", ACTUAL=" + cnt + ", " + "ITERATION=" + iteration + msg);
            }
            sleep(500);
        }
    } finally {
        nodeRestartThread.interrupt();
        cur.close();
        nodeRestartThread.join(3_000);
    }
}

Example 83 with ContinuousQuery

use of org.apache.ignite.cache.query.ContinuousQuery in project ignite by apache.

the class CacheContinuousQueryFailoverAbstractSelfTest method testRemoteFilter.

/**
 * @throws Exception If failed.
 */
@Test
public void testRemoteFilter() throws Exception {
    this.backups = 2;
    final int SRV_NODES = 4;
    startGridsMultiThreaded(SRV_NODES);
    Ignite qryClient = startClientGrid(SRV_NODES);
    IgniteCache<Object, Object> qryClientCache = qryClient.cache(DEFAULT_CACHE_NAME);
    if (cacheMode() != REPLICATED)
        assertEquals(backups, qryClientCache.getConfiguration(CacheConfiguration.class).getBackups());
    Affinity<Object> aff = qryClient.affinity(DEFAULT_CACHE_NAME);
    ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
    final CacheEventListener3 lsnr = asyncCallback() ? new CacheEventAsyncListener3() : new CacheEventListener3();
    qry.setLocalListener(lsnr);
    qry.setRemoteFilter(lsnr);
    int PARTS = 10;
    QueryCursor<?> cur = qryClientCache.query(qry);
    Map<Object, T2<Object, Object>> updates = new HashMap<>();
    final List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
    for (int i = 0; i < (atomicityMode() == CacheAtomicityMode.ATOMIC ? SRV_NODES - 1 : SRV_NODES - 2); i++) {
        log.info("Stop iteration: " + i);
        TestCommunicationSpi spi = (TestCommunicationSpi) ignite(i).configuration().getCommunicationSpi();
        Ignite ignite = ignite(i);
        IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
        List<Integer> keys = testKeys(cache, PARTS);
        boolean first = true;
        boolean filtered = false;
        for (Integer key : keys) {
            log.info("Put [node=" + ignite.name() + ", key=" + key + ", part=" + aff.partition(key) + ", filtered=" + filtered + ']');
            T2<Object, Object> t = updates.get(key);
            Integer val = filtered ? (key % 2 == 0 ? key + 1 : key) : key * 2;
            if (t == null) {
                updates.put(key, new T2<>((Object) val, null));
                if (!filtered)
                    expEvts.add(new T3<>((Object) key, (Object) val, null));
            } else {
                updates.put(key, new T2<>((Object) val, (Object) key));
                if (!filtered)
                    expEvts.add(new T3<>((Object) key, (Object) val, (Object) key));
            }
            cache.put(key, val);
            if (first) {
                spi.skipMsg = true;
                first = false;
            }
            filtered = !filtered;
        }
        stopGrid(i);
        boolean check = GridTestUtils.waitForCondition(new PAX() {

            @Override
            public boolean applyx() throws IgniteCheckedException {
                return expEvts.size() == lsnr.keys.size();
            }
        }, 5000L);
        if (!check) {
            Set<Integer> keys0 = new HashSet<>(keys);
            keys0.removeAll(lsnr.keys);
            log.info("Missed events for keys: " + keys0);
            fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + keys0.size() + ']');
        }
        checkEvents(expEvts, lsnr, false);
    }
    cur.close();
}

Example 84 with ContinuousQuery

use of org.apache.ignite.cache.query.ContinuousQuery in project ignite by apache.

the class CacheContinuousQueryOrderingEventTest method doOrderingTest.

/**
 * @param ccfg Cache configuration.
 * @param async Async filter.
 * @throws Exception If failed.
 */
protected void doOrderingTest(final CacheConfiguration ccfg, final boolean async) throws Exception {
    ignite(0).createCache(ccfg);
    List<QueryCursor<?>> qries = new ArrayList<>();
    try {
        List<BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>>> rcvdEvts = new ArrayList<>(LISTENER_CNT * NODES);
        final AtomicInteger qryCntr = new AtomicInteger(0);
        final int threadCnt = 20;
        for (int idx = 0; idx < NODES; idx++) {
            for (int i = 0; i < LISTENER_CNT; i++) {
                BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue = new ArrayBlockingQueue<>(ITERATION_CNT * threadCnt);
                ContinuousQuery qry = new ContinuousQuery();
                if (async) {
                    qry.setLocalListener(new TestCacheAsyncEventListener(queue, qryCntr));
                    qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(new CacheTestRemoteFilterAsync(ccfg.getName())));
                } else {
                    qry.setLocalListener(new TestCacheEventListener(queue, qryCntr));
                    qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(new CacheTestRemoteFilter(ccfg.getName())));
                }
                rcvdEvts.add(queue);
                IgniteCache<Object, Object> cache = grid(idx).cache(ccfg.getName());
                QueryCursor qryCursor = cache.query(qry);
                qries.add(qryCursor);
            }
        }
        IgniteInternalFuture<Long> f = GridTestUtils.runMultiThreadedAsync(new Runnable() {

            @Override
            public void run() {
                ThreadLocalRandom rnd = ThreadLocalRandom.current();
                for (int i = 0; i < ITERATION_CNT; i++) {
                    IgniteCache<QueryTestKey, QueryTestValue> cache = grid(rnd.nextInt(NODES)).cache(ccfg.getName());
                    QueryTestKey key = new QueryTestKey(rnd.nextInt(KEYS));
                    boolean startTx = atomicityMode(cache) != ATOMIC && rnd.nextBoolean();
                    Transaction tx = null;
                    boolean committed = false;
                    while (!committed && !Thread.currentThread().isInterrupted()) {
                        try {
                            if (startTx)
                                tx = cache.unwrap(Ignite.class).transactions().txStart(PESSIMISTIC, REPEATABLE_READ);
                            if ((cache.get(key) == null) || rnd.nextBoolean()) {
                                cache.invoke(key, new CacheEntryProcessor<QueryTestKey, QueryTestValue, Object>() {

                                    @Override
                                    public Object process(MutableEntry<QueryTestKey, QueryTestValue> entry, Object... arguments) throws EntryProcessorException {
                                        if (entry.exists())
                                            entry.setValue(new QueryTestValue(entry.getValue().val1 + 1));
                                        else
                                            entry.setValue(new QueryTestValue(0));
                                        return null;
                                    }
                                });
                            } else {
                                QueryTestValue val;
                                QueryTestValue newVal;
                                do {
                                    val = cache.get(key);
                                    newVal = val == null ? new QueryTestValue(0) : new QueryTestValue(val.val1 + 1);
                                } while (!cache.replace(key, val, newVal));
                            }
                            if (tx != null)
                                tx.commit();
                            committed = true;
                        } catch (CacheException e) {
                            assertTrue(e.getCause() instanceof TransactionSerializationException);
                            assertEquals(atomicityMode(cache), TRANSACTIONAL_SNAPSHOT);
                        } finally {
                            if (tx != null)
                                tx.close();
                        }
                    }
                }
            }
        }, threadCnt, "put-thread");
        f.get(15, TimeUnit.SECONDS);
        GridTestUtils.waitForCondition(new PA() {

            @Override
            public boolean apply() {
                return qryCntr.get() >= ITERATION_CNT * threadCnt * LISTENER_CNT * NODES;
            }
        }, 1000L);
        for (BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue : rcvdEvts) checkEvents(queue, ITERATION_CNT * threadCnt);
        assertFalse("Ordering invocations of filter broken.", fail);
    } finally {
        for (QueryCursor<?> qry : qries) qry.close();
        ignite(0).destroyCache(ccfg.getName());
    }
}

Example 85 with ContinuousQuery

use of org.apache.ignite.cache.query.ContinuousQuery in project ignite by apache.

the class CacheContinuousQueryFailoverAbstractSelfTest method testMultiThreadedFailover.

/**
 * @throws Exception If failed.
 */
@Test
public void testMultiThreadedFailover() throws Exception {
    this.backups = 2;
    final int SRV_NODES = 4;
    startGridsMultiThreaded(SRV_NODES);
    final Ignite qryCln = startClientGrid(SRV_NODES);
    final IgniteCache<Object, Object> qryClnCache = qryCln.cache(DEFAULT_CACHE_NAME);
    final CacheEventListener2 lsnr = asyncCallback() ? new CacheEventAsyncListener2() : new CacheEventListener2();
    ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
    qry.setLocalListener(lsnr);
    QueryCursor<?> cur = qryClnCache.query(qry);
    final AtomicBoolean stop = new AtomicBoolean();
    final int THREAD = 4;
    final int PARTS = THREAD;
    final List<List<T3<Object, Object, Object>>> expEvts = new ArrayList<>(THREAD + 5);
    for (int i = 0; i < THREAD; i++) expEvts.add(i, new ArrayList<T3<Object, Object, Object>>());
    final AtomicReference<CyclicBarrier> checkBarrier = new AtomicReference<>();
    final ThreadLocalRandom rnd = ThreadLocalRandom.current();
    IgniteInternalFuture<?> restartFut = GridTestUtils.runAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            try {
                while (!stop.get() && !err) {
                    final int idx = rnd.nextInt(SRV_NODES);
                    log.info("Stop node: " + idx);
                    stopGrid(idx);
                    Thread.sleep(300);
                    GridTestUtils.waitForCondition(new PA() {

                        @Override
                        public boolean apply() {
                            return qryCln.cluster().nodes().size() == SRV_NODES;
                        }
                    }, 5000L);
                    try {
                        log.info("Start node: " + idx);
                        startGrid(idx);
                        Thread.sleep(300);
                        GridTestUtils.waitForCondition(new PA() {

                            @Override
                            public boolean apply() {
                                return qryCln.cluster().nodes().size() == SRV_NODES + 1;
                            }
                        }, 5000L);
                    } catch (Exception e) {
                        log.warning("Failed to stop nodes.", e);
                    }
                    CyclicBarrier bar = new CyclicBarrier(THREAD + 1, /* plus start/stop thread */
                    new Runnable() {

                        @Override
                        public void run() {
                            try {
                                int size0 = 0;
                                for (List<T3<Object, Object, Object>> evt : expEvts) size0 += evt.size();
                                final int size = size0;
                                GridTestUtils.waitForCondition(new PA() {

                                    @Override
                                    public boolean apply() {
                                        return lsnr.size() >= size;
                                    }
                                }, 10_000L);
                                List<T3<Object, Object, Object>> expEvts0 = new ArrayList<>();
                                for (List<T3<Object, Object, Object>> evt : expEvts) expEvts0.addAll(evt);
                                checkEvents(expEvts0, lsnr, false, false);
                                for (List<T3<Object, Object, Object>> evt : expEvts) evt.clear();
                            } catch (Exception e) {
                                log.error("Failed.", e);
                                err = true;
                                stop.set(true);
                            } finally {
                                checkBarrier.set(null);
                            }
                        }
                    });
                    assertTrue(checkBarrier.compareAndSet(null, bar));
                    if (!stop.get() && !err)
                        bar.await(1, MINUTES);
                }
            } catch (Throwable e) {
                log.error("Unexpected error: " + e, e);
                err = true;
                throw e;
            }
            return null;
        }
    });
    final long stopTime = System.currentTimeMillis() + 60_000;
    final AtomicInteger valCntr = new AtomicInteger(0);
    final AtomicInteger threadSeq = new AtomicInteger(0);
    GridTestUtils.runMultiThreaded(new Runnable() {

        @Override
        public void run() {
            try {
                final ThreadLocalRandom rnd = ThreadLocalRandom.current();
                final int threadId = threadSeq.getAndIncrement();
                log.error("Thread id: " + threadId);
                while (System.currentTimeMillis() < stopTime && !stop.get() && !err) {
                    Integer key = rnd.nextInt(PARTS);
                    Integer val = valCntr.incrementAndGet();
                    Integer prevVal = null;
                    boolean updated = false;
                    while (!updated) {
                        try {
                            prevVal = (Integer) qryClnCache.getAndPut(key, val);
                            updated = true;
                        } catch (CacheException e) {
                            assertTrue(e.getCause() instanceof TransactionSerializationException);
                            assertSame(atomicityMode(), CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT);
                        }
                    }
                    expEvts.get(threadId).add(new T3<>((Object) key, (Object) val, (Object) prevVal));
                    CyclicBarrier bar = checkBarrier.get();
                    if (bar != null)
                        bar.await(1, MINUTES);
                }
            } catch (Exception e) {
                log.error("Failed.", e);
                err = true;
                stop.set(true);
            } finally {
                stop.set(true);
            }
        }
    }, THREAD, "update-thread");
    restartFut.get();
    List<T3<Object, Object, Object>> expEvts0 = new ArrayList<>();
    for (List<T3<Object, Object, Object>> evt : expEvts) {
        expEvts0.addAll(evt);
        evt.clear();
    }
    if (!expEvts0.isEmpty())
        checkEvents(expEvts0, lsnr, true);
    cur.close();
    assertFalse("Unexpected error during test, see log for details.", err);
}