Examples with IgniteDataStreamer org.apache.ignite.IgniteDataStreamer used on opensource projects

Example 41 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteClusterActivateDeactivateTestWithPersistence method testDeactivateDuringEvictionAndRebalance.

/**
 * Test that after deactivation during eviction and rebalance and activation again after
 * all data in cache is consistent.
 *
 * @throws Exception If failed.
 */
@Test
public void testDeactivateDuringEvictionAndRebalance() throws Exception {
    Assume.assumeFalse("https://issues.apache.org/jira/browse/IGNITE-7384", MvccFeatureChecker.forcedMvcc());
    IgniteEx srv = startGrids(3);
    srv.cluster().state(ACTIVE);
    CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME).setBackups(1).setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC).setIndexedTypes(Integer.class, Integer.class).setAffinity(new RendezvousAffinityFunction(false, 64)).setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
    IgniteCache cache = srv.createCache(ccfg);
    // High number of keys triggers long partition eviction.
    final int keysCount = 100_000;
    try (IgniteDataStreamer ds = srv.dataStreamer(DEFAULT_CACHE_NAME)) {
        log.info("Writing initial data...");
        ds.allowOverwrite(true);
        for (int k = 1; k <= keysCount; k++) {
            ds.addData(k, k);
            if (k % 50_000 == 0)
                log.info("Written " + k + " entities.");
        }
        log.info("Writing initial data finished.");
    }
    AtomicInteger keyCounter = new AtomicInteger(keysCount);
    AtomicBoolean stop = new AtomicBoolean(false);
    Set<Integer> addedKeys = new GridConcurrentHashSet<>();
    IgniteInternalFuture cacheLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
        while (!stop.get()) {
            int key = keyCounter.incrementAndGet();
            try {
                cache.put(key, key);
                addedKeys.add(key);
                Thread.sleep(10);
            } catch (Exception ignored) {
            }
        }
    }, 2, "cache-load");
    stopGrid(2);
    // Wait for some data.
    Thread.sleep(3000);
    startGrid(2);
    log.info("Stop load...");
    stop.set(true);
    cacheLoadFuture.get();
    // Deactivate and activate again.
    srv.cluster().state(INACTIVE);
    srv.cluster().state(ACTIVE);
    awaitPartitionMapExchange();
    log.info("Checking data...");
    for (Ignite ignite : G.allGrids()) {
        IgniteCache cache1 = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
        for (int k = 1; k <= keysCount; k++) {
            Object val = cache1.get(k);
            Assert.assertNotNull("node=" + ignite.name() + ", key=" + k, val);
            Assert.assertTrue("node=" + ignite.name() + ", key=" + k + ", val=" + val, (int) val == k);
        }
        for (int k : addedKeys) {
            Object val = cache1.get(k);
            Assert.assertNotNull("node=" + ignite.name() + ", key=" + k, val);
            Assert.assertTrue("node=" + ignite.name() + ", key=" + k + ", val=" + val, (int) val == k);
        }
    }
}

Example 42 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class HistoricalRebalanceTwoPartsInDifferentCheckpointsTest method rebalanceTwoPartitions.

/**
 * Checks a historical rebalance for partitions 0 and 2.
 *
 * @param descending When true first updates is for younger partition, false is older one.
 * @throws Exception If failed.
 */
public void rebalanceTwoPartitions(boolean descending) throws Exception {
    IgniteEx ignite0 = startGrids(2);
    ignite0.cluster().state(ClusterState.ACTIVE);
    awaitPartitionMapExchange();
    LinkedHashSet<Integer> partKeys = new LinkedHashSet<>();
    if (descending) {
        partKeys.addAll(partitionKeys(ignite0.cache(DEFAULT_CACHE_NAME), 0, 10, 0));
        partKeys.addAll(partitionKeys(ignite0.cache(DEFAULT_CACHE_NAME), 2, 10, 0));
    } else {
        partKeys.addAll(partitionKeys(ignite0.cache(DEFAULT_CACHE_NAME), 2, 10, 0));
        partKeys.addAll(partitionKeys(ignite0.cache(DEFAULT_CACHE_NAME), 0, 10, 0));
    }
    // It can be achieved by using full_sync write sync mode or by using a data streamer with allowOverwrite == false.
    try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) {
        partKeys.forEach(key -> streamer.addData(key, key));
    }
    info("Data preload completed.");
    stopGrid(1);
    awaitPartitionMapExchange();
    info("Node stopped.");
    IgniteCache cache = ignite0.cache(DEFAULT_CACHE_NAME);
    for (Integer key : partKeys) {
        cache.put(key, key + 1);
        forceCheckpoint();
    }
    IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(1));
    AtomicBoolean hasFullRebalance = new AtomicBoolean();
    AtomicBoolean hasHistoricalRebalance = new AtomicBoolean();
    ((TestRecordingCommunicationSpi) cfg.getCommunicationSpi()).record((node, msg) -> {
        if (msg instanceof GridDhtPartitionDemandMessage) {
            GridDhtPartitionDemandMessage demandMsg = (GridDhtPartitionDemandMessage) msg;
            hasFullRebalance.set(demandMsg.partitions().hasFull());
            hasHistoricalRebalance.set(demandMsg.partitions().hasHistorical());
        }
        return false;
    });
    startGrid(cfg);
    awaitPartitionMapExchange();
    info("Node started and rebalance completed.");
    assertFalse(hasFullRebalance.get());
    assertTrue(hasHistoricalRebalance.get());
}

Example 43 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class CacheIndexStreamerTest method checkStreamer.

/**
 * @param atomicityMode Cache atomicity mode.
 * @throws Exception If failed.
 */
public void checkStreamer(CacheAtomicityMode atomicityMode) throws Exception {
    final Ignite ignite = startGrid(0);
    final IgniteCache<Integer, String> cache = ignite.createCache(cacheConfiguration(atomicityMode));
    final AtomicBoolean stop = new AtomicBoolean();
    final int KEYS = 10_000;
    try {
        IgniteInternalFuture streamerFut = GridTestUtils.runAsync(new Callable() {

            @Override
            public Void call() throws Exception {
                ThreadLocalRandom rnd = ThreadLocalRandom.current();
                while (!stop.get()) {
                    try (IgniteDataStreamer<Integer, String> streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
                        // TODO FIXME https://issues.apache.org/jira/browse/IGNITE-11793
                        streamer.allowOverwrite(atomicityMode == TRANSACTIONAL);
                        for (int i = 0; i < 1; i++) streamer.addData(rnd.nextInt(KEYS), String.valueOf(i));
                    }
                }
                return null;
            }
        }, "streamer-thread");
        IgniteInternalFuture updateFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                ThreadLocalRandom rnd = ThreadLocalRandom.current();
                while (!stop.get()) {
                    for (int i = 0; i < 100; i++) {
                        Integer key = rnd.nextInt(KEYS);
                        cache.put(key, String.valueOf(key));
                        cache.remove(key);
                    }
                }
                return null;
            }
        }, 1, "update-thread");
        U.sleep(30_000);
        stop.set(true);
        streamerFut.get();
        updateFut.get();
    } finally {
        stop.set(true);
        stopAllGrids();
    }
}

Example 44 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class CacheSizeTtlTest method testCacheSizeWorksCorrectlyWithTtl.

/**
 * Tests that cache.size() works correctly for massive amount of puts and ttl.
 */
@Test
public void testCacheSizeWorksCorrectlyWithTtl() throws IgniteInterruptedCheckedException {
    startIgniteServer();
    Ignite client = startIgniteClient();
    try (IgniteDataStreamer dataStreamer = client.dataStreamer(CACHE_NAME)) {
        IntStream.range(0, 100_000).forEach(i -> dataStreamer.addData(1, LocalDateTime.now()));
    }
    assertTrue(GridTestUtils.waitForCondition(() -> client.cache(CACHE_NAME).size(CachePeekMode.PRIMARY) == 0, getTestTimeout()));
}

Example 45 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class DynamicIndexAbstractSelfTest method put.

/**
 * Put key range.
 *
 * @param node Node.
 * @param from From key.
 * @param to To key.
 */
protected static void put(Ignite node, int from, int to) {
    try (IgniteDataStreamer streamer = node.dataStreamer(CACHE_NAME)) {
        streamer.allowOverwrite(true);
        streamer.keepBinary(true);
        for (int i = from; i < to; i++) {
            BinaryObject key = key(node, i);
            BinaryObject val = value(node, i);
            streamer.addData(key, val);
        }
        streamer.flush();
    }
}