Examples with IgniteDataStreamer org.apache.ignite.IgniteDataStreamer used on opensource projects

Example 31 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheQueriesLoadTest1 method preLoading.

/**
 * @throws Exception If fail.
 */
private void preLoading() throws Exception {
    final Thread preloadAccount = new Thread() {

        @Override
        public void run() {
            setName("preloadTraders");
            Ignite ignite = ignite(0);
            try (IgniteDataStreamer dataLdr = ignite.dataStreamer(TRADER_CACHE)) {
                for (int i = 0; i < preloadAmount && !isInterrupted(); i++) {
                    String traderKey = "traderId=" + i;
                    dataLdr.addData(traderKey, createTrader(ignite, traderKey));
                }
            }
        }
    };
    preloadAccount.start();
    Thread preloadTrade = new Thread() {

        @Override
        public void run() {
            setName("preloadDeposits");
            Ignite ignite = ignite(0);
            try (IgniteDataStreamer dataLdr = ignite.dataStreamer(DEPOSIT_CACHE)) {
                for (int i = 0; i < preloadAmount && !isInterrupted(); i++) {
                    int traderId = nextRandom(preloadAmount);
                    String traderKey = "traderId=" + traderId;
                    String key = traderKey + "&depositId=" + i;
                    dataLdr.addData(key, createDeposit(ignite, key, traderKey, i));
                }
            }
        }
    };
    preloadTrade.start();
    preloadTrade.join();
    preloadAccount.join();
}

Example 32 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class DataStreaming method streamReceiverExample.

@Test
void streamReceiverExample() {
    try (Ignite ignite = Ignition.start()) {
        ignite.getOrCreateCache("myCache");
        // tag::streamReceiver[]
        try (IgniteDataStreamer<Integer, String> stmr = ignite.dataStreamer("myCache")) {
            stmr.allowOverwrite(true);
            stmr.receiver((StreamReceiver<Integer, String>) (cache, entries) -> entries.forEach(entry -> {
                // do something with the entry
                cache.put(entry.getKey(), entry.getValue());
            }));
        }
    // end::streamReceiver[]
    }
}

Example 33 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheGroupsTest method cacheDataStreamer.

/**
 * @param cache Cache.
 * @throws Exception If failed.
 */
private void cacheDataStreamer(final IgniteCache cache) throws Exception {
    final int keys = 400;
    final int loaders = 4;
    final Integer[] data = generateData(keys * loaders);
    // Stream through a client node.
    Ignite clientNode = ignite(4);
    List<Callable<?>> cls = new ArrayList<>(loaders);
    for (final int i : sequence(loaders)) {
        final IgniteDataStreamer ldr = clientNode.dataStreamer(cache.getName());
        // TODO FIXME https://issues.apache.org/jira/browse/IGNITE-11793
        ldr.allowOverwrite(true);
        ldr.autoFlushFrequency(0);
        cls.add(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                List<IgniteFuture> futs = new ArrayList<>(keys);
                for (int j = 0, size = keys * loaders; j < size; j++) {
                    if (j % loaders == i)
                        futs.add(ldr.addData(j, data[j]));
                    if (j % (100 * loaders) == 0)
                        ldr.flush();
                }
                ldr.flush();
                for (IgniteFuture fut : futs) fut.get();
                return null;
            }
        });
    }
    GridTestUtils.runMultiThreaded(cls, "loaders");
    Set<Integer> keysSet = sequence(data.length);
    for (Cache.Entry<Integer, Integer> entry : (IgniteCache<Integer, Integer>) cache) {
        assertTrue(keysSet.remove(entry.getKey()));
        assertEquals(data[entry.getKey()], entry.getValue());
    }
    assertTrue(keysSet.isEmpty());
    tearDown(cache);
}

Example 34 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheDynamicStopSelfTest method checkStopStartCacheWithDataLoader.

/**
 * @param allowOverwrite Allow overwrite flag for streamer.
 * @throws Exception If failed.
 */
public void checkStopStartCacheWithDataLoader(final boolean allowOverwrite) throws Exception {
    CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
    ccfg.setCacheMode(CacheMode.PARTITIONED);
    ignite(0).createCache(ccfg);
    final AtomicBoolean stop = new AtomicBoolean();
    IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {

        /**
         * {@inheritDoc}
         */
        @Override
        public Object call() throws Exception {
            while (!stop.get()) {
                try (IgniteDataStreamer<Integer, Integer> str = ignite(0).dataStreamer(DEFAULT_CACHE_NAME)) {
                    str.allowOverwrite(allowOverwrite);
                    int i = 0;
                    while (!stop.get()) {
                        try {
                            str.addData(i % 10_000, i).listen(new CI1<IgniteFuture<?>>() {

                                @Override
                                public void apply(IgniteFuture<?> f) {
                                    try {
                                        f.get();
                                    } catch (CacheException ignore) {
                                    // This may be debugged.
                                    }
                                }
                            });
                        } catch (IllegalStateException ignored) {
                            break;
                        }
                        if (i > 0 && i % 10000 == 0)
                            info("Added: " + i);
                        i++;
                    }
                } catch (IllegalStateException | CacheException ignored) {
                // This may be debugged.
                }
            }
            return null;
        }
    });
    try {
        Thread.sleep(500);
        ignite(0).destroyCache(DEFAULT_CACHE_NAME);
        Thread.sleep(500);
        ignite(0).createCache(ccfg);
        Thread.sleep(1000);
    } finally {
        stop.set(true);
    }
    fut.get();
    int cnt = 0;
    for (Cache.Entry<Object, Object> ignored : ignite(0).cache(DEFAULT_CACHE_NAME)) cnt++;
    info(">>> cnt=" + cnt);
    ignite(0).destroyCache(DEFAULT_CACHE_NAME);
}

Example 35 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class ClusterReadOnlyModeTest method testDataStreamerReadOnlyConcurrent.

/**
 * Common logic for different datastreamers' tests.
 *
 * @param manualFlush If {@code True} {@link IgniteDataStreamer#flush()} will be invoked in the each batch load.
 * @param allowOverride value for {@link IgniteDataStreamer#allowOverwrite(boolean)} method.
 * @throws Exception If something goes wrong.
 */
private void testDataStreamerReadOnlyConcurrent(boolean manualFlush, boolean allowOverride) throws Exception {
    final CountDownLatch firstPackLatch = new CountDownLatch(cacheNames().size());
    final CountDownLatch finishLatch = new CountDownLatch(cacheNames().size());
    final CountDownLatch readOnlyEnabled = new CountDownLatch(1);
    final Map<String, Exception> eMap = new ConcurrentHashMap<>(cacheNames().size());
    Map<String, IgniteInternalFuture<?>> futs = new HashMap<>(cacheNames().size());
    try {
        for (String cacheName : cacheNames()) {
            futs.put(cacheName, GridTestUtils.runAsync(() -> {
                try (IgniteDataStreamer<Integer, Integer> streamer = grid(0).dataStreamer(cacheName)) {
                    streamer.allowOverwrite(allowOverride);
                    doLoad(streamer, 0, 100, manualFlush);
                    firstPackLatch.countDown();
                    assertTrue(cacheName, readOnlyEnabled.await(60, TimeUnit.SECONDS));
                    doLoad(streamer, 100, 1000000, manualFlush);
                    finishLatch.countDown();
                } catch (Exception e) {
                    log.error("Streamer cache exception is thrown for cache " + cacheName, e);
                    assertNull(cacheName, eMap.put(cacheName, e));
                } finally {
                    // Avoid to hanging test in case of unexpected behaviour.
                    firstPackLatch.countDown();
                    finishLatch.countDown();
                }
            }));
        }
        assertTrue(firstPackLatch.await(60, TimeUnit.SECONDS));
        changeClusterReadOnlyMode(true);
        readOnlyEnabled.countDown();
        assertTrue(finishLatch.await(60, TimeUnit.SECONDS));
        assertEquals("exceptions: " + eMap, cacheNames().size(), eMap.size());
        for (String cacheName : cacheNames()) {
            Exception e = eMap.get(cacheName);
            assertNotNull(cacheName, e);
            assertTrue(cacheName + " " + e, X.hasCause(e, IgniteClusterReadOnlyException.class));
        }
    } finally {
        // Avoid to hanging test in case of unexpected behaviour.
        readOnlyEnabled.countDown();
        awaitThreads(futs);
    }
}