Examples with IgniteDataStreamer org.apache.ignite.IgniteDataStreamer used on opensource projects

Example 16 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteClientReconnectApiExceptionTest method cacheOperationsTest.

/**
 * @throws Exception If failed.
 */
@SuppressWarnings("unchecked")
public void cacheOperationsTest() throws Exception {
    clientMode = true;
    final Ignite client = startGrid(serverCount());
    final IgniteCache<Object, Object> dfltCache = client.cache(DEFAULT_CACHE_NAME);
    assertNotNull(dfltCache);
    doTestIgniteOperationOnDisconnect(client, Arrays.asList(// Check put and get operation.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                dfltCache.getAndPut(9999, 9999);
            } catch (CacheException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return dfltCache.getAndPut(9999, 9999);
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            assertNull(o);
            assertEquals(9999, dfltCache.get(9999));
            return true;
        }
    }), // Check put operation.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                dfltCache.put(10000, 10000);
            } catch (CacheException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            dfltCache.put(10000, 10000);
            return true;
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            assertTrue((Boolean) o);
            assertEquals(10000, dfltCache.get(10000));
            return true;
        }
    }), // Check get operation.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                dfltCache.get(10001);
            } catch (CacheException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return dfltCache.get(10001);
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            assertNull(o);
            return true;
        }
    }), // Check put and invoke operation.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                dfltCache.put(CACHE_PUT_INVOKE_KEY, 10000);
                dfltCache.invoke(CACHE_PUT_INVOKE_KEY, new CacheEntryProcessor<Object, Object, Object>() {

                    @Override
                    public Object process(MutableEntry<Object, Object> entry, Object... arguments) throws EntryProcessorException {
                        assertTrue(entry.exists());
                        return (int) entry.getValue() * 2;
                    }
                });
            } catch (CacheException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            dfltCache.put(CACHE_PUT_INVOKE_KEY, 10000);
            return dfltCache.invoke(CACHE_PUT_INVOKE_KEY, new CacheEntryProcessor<Object, Object, Object>() {

                @Override
                public Object process(MutableEntry<Object, Object> entry, Object... arguments) throws EntryProcessorException {
                    assertTrue(entry.exists());
                    return (int) entry.getValue() * 2;
                }
            });
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            assertNotNull(o);
            assertEquals(20000, (int) o);
            return true;
        }
    }), // Check put async operation.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                dfltCache.putAsync(10002, 10002).get();
            } catch (CacheException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return dfltCache.putAsync(10002, 10002).get();
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            assertNull(o);
            assertEquals(10002, dfltCache.get(10002));
            return true;
        }
    }), // Check transaction.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                client.transactions();
            } catch (IgniteClientDisconnectedException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return client.transactions();
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            IgniteTransactions txs = (IgniteTransactions) o;
            assertNotNull(txs);
            return true;
        }
    }), // Check get cache.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                client.cache(DEFAULT_CACHE_NAME);
            } catch (IgniteClientDisconnectedException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return client.cache(DEFAULT_CACHE_NAME);
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            IgniteCache<Object, Object> cache0 = (IgniteCache<Object, Object>) o;
            assertNotNull(cache0);
            cache0.put(1, 1);
            assertEquals(1, cache0.get(1));
            return true;
        }
    }), // Check streamer.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                client.dataStreamer(DEFAULT_CACHE_NAME);
            } catch (IgniteClientDisconnectedException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return client.dataStreamer(DEFAULT_CACHE_NAME);
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            IgniteDataStreamer<Object, Object> streamer = (IgniteDataStreamer<Object, Object>) o;
            streamer.addData(2, 2);
            streamer.close();
            assertEquals(2, client.cache(DEFAULT_CACHE_NAME).get(2));
            return true;
        }
    }), // Check create cache.
    new T2<Callable, C1<Object, Boolean>>(new Callable() {

        @Override
        public Object call() throws Exception {
            boolean failed = false;
            try {
                client.createCache("test_cache");
            } catch (IgniteClientDisconnectedException e) {
                failed = true;
                checkAndWait(e);
            }
            assertTrue(failed);
            return client.createCache("test_cache");
        }
    }, new C1<Object, Boolean>() {

        @Override
        public Boolean apply(Object o) {
            IgniteCache<Object, Object> cache = (IgniteCache<Object, Object>) o;
            assertNotNull(cache);
            cache.put(1, 1);
            assertEquals(1, cache.get(1));
            return true;
        }
    })));
    clientMode = false;
}

Example 17 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheDynamicStopSelfTest method checkStopStartCacheWithDataLoader.

/**
 * @param allowOverwrite Allow overwrite flag for streamer.
 * @throws Exception If failed.
 */
public void checkStopStartCacheWithDataLoader(final boolean allowOverwrite) throws Exception {
    CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
    ccfg.setCacheMode(CacheMode.PARTITIONED);
    ignite(0).createCache(ccfg);
    final AtomicBoolean stop = new AtomicBoolean();
    IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {

        /**
         * {@inheritDoc}
         */
        @Override
        public Object call() throws Exception {
            while (!stop.get()) {
                try (IgniteDataStreamer<Integer, Integer> str = ignite(0).dataStreamer(DEFAULT_CACHE_NAME)) {
                    str.allowOverwrite(allowOverwrite);
                    int i = 0;
                    while (!stop.get()) {
                        try {
                            str.addData(i % 10_000, i).listen(new CI1<IgniteFuture<?>>() {

                                @Override
                                public void apply(IgniteFuture<?> f) {
                                    try {
                                        f.get();
                                    } catch (CacheException ignore) {
                                    // This may be debugged.
                                    }
                                }
                            });
                        } catch (IllegalStateException ignored) {
                            break;
                        }
                        if (i > 0 && i % 10000 == 0)
                            info("Added: " + i);
                        i++;
                    }
                } catch (IllegalStateException | CacheException ignored) {
                // This may be debugged.
                }
            }
            return null;
        }
    });
    try {
        Thread.sleep(500);
        ignite(0).destroyCache(DEFAULT_CACHE_NAME);
        Thread.sleep(500);
        ignite(0).createCache(ccfg);
        Thread.sleep(1000);
    } finally {
        stop.set(true);
    }
    fut.get();
    int cnt = 0;
    for (Cache.Entry<Object, Object> ignored : ignite(0).cache(DEFAULT_CACHE_NAME)) cnt++;
    info(">>> cnt=" + cnt);
    ignite(0).destroyCache(DEFAULT_CACHE_NAME);
}

Example 18 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheGroupsTest method cacheDataStreamer.

/**
 * @param cache Cache.
 * @throws Exception If failed.
 */
private void cacheDataStreamer(final IgniteCache cache) throws Exception {
    final int keys = 400;
    final int loaders = 4;
    final Integer[] data = generateData(keys * loaders);
    // Stream through a client node.
    Ignite clientNode = ignite(4);
    List<Callable<?>> cls = new ArrayList<>(loaders);
    for (final int i : sequence(loaders)) {
        final IgniteDataStreamer ldr = clientNode.dataStreamer(cache.getName());
        ldr.autoFlushFrequency(0);
        cls.add(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                List<IgniteFuture> futs = new ArrayList<>(keys);
                for (int j = 0, size = keys * loaders; j < size; j++) {
                    if (j % loaders == i)
                        futs.add(ldr.addData(j, data[j]));
                    if (j % (100 * loaders) == 0)
                        ldr.flush();
                }
                ldr.flush();
                for (IgniteFuture fut : futs) fut.get();
                return null;
            }
        });
    }
    GridTestUtils.runMultiThreaded(cls, "loaders");
    Set<Integer> keysSet = sequence(data.length);
    for (Cache.Entry<Integer, Integer> entry : (IgniteCache<Integer, Integer>) cache) {
        assertTrue(keysSet.remove(entry.getKey()));
        assertEquals(data[entry.getKey()], entry.getValue());
    }
    assertTrue(keysSet.isEmpty());
    tearDown(cache);
}

Example 19 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class JdbcThinStreamingSelfTest method testSimultaneousStreaming.

/**
 * @throws SQLException if failed.
 */
public void testSimultaneousStreaming() throws Exception {
    try (Connection anotherConn = createOrdinaryConnection()) {
        execute(anotherConn, "CREATE TABLE PUBLIC.T(x int primary key, y int) WITH " + "\"cache_name=T,wrap_value=false\"");
    }
    // Timeout to let connection close be handled on server side.
    U.sleep(500);
    try (Connection conn = createStreamedConnection(false, 10000)) {
        assertStreamingState(true);
        PreparedStatement firstStmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)");
        PreparedStatement secondStmt = conn.prepareStatement("insert into PUBLIC.T(x, y) values (?, ?)");
        try {
            for (int i = 1; i <= 10; i++) {
                firstStmt.setInt(1, i);
                firstStmt.setString(2, nameForId(i));
                firstStmt.executeUpdate();
            }
            for (int i = 51; i <= 67; i++) {
                secondStmt.setInt(1, i);
                secondStmt.setInt(2, i);
                secondStmt.executeUpdate();
            }
            for (int i = 11; i <= 50; i++) {
                firstStmt.setInt(1, i);
                firstStmt.setString(2, nameForId(i));
                firstStmt.executeUpdate();
            }
            for (int i = 68; i <= 100; i++) {
                secondStmt.setInt(1, i);
                secondStmt.setInt(2, i);
                secondStmt.executeUpdate();
            }
            assertCacheEmpty();
            SqlClientContext cliCtx = sqlClientContext();
            HashMap<String, IgniteDataStreamer<?, ?>> streamers = U.field(cliCtx, "streamers");
            assertEquals(2, streamers.size());
            assertEqualsCollections(new HashSet<>(Arrays.asList("person", "T")), streamers.keySet());
        } finally {
            U.closeQuiet(firstStmt);
            U.closeQuiet(secondStmt);
        }
    }
    // Let's wait a little so that all data arrives to destination - we can't intercept streamers' flush
    // on connection close in any way.
    U.sleep(1000);
    // Now let's check it's all there.
    for (int i = 1; i <= 50; i++) assertEquals(nameForId(i), nameForIdInCache(i));
    for (int i = 51; i <= 100; i++) assertEquals(i, grid(0).cache("T").get(i));
}

Example 20 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class GridCacheRebalancingWithAsyncClearingTest method testCorrectRebalancingCurrentlyRentingPartitions.

/**
 * Test that partitions belong to affinity in state RENTING or EVICTED are correctly rebalanced.
 *
 * @throws Exception If failed.
 */
public void testCorrectRebalancingCurrentlyRentingPartitions() throws Exception {
    IgniteEx ignite = (IgniteEx) startGrids(3);
    ignite.cluster().active(true);
    // High number of keys triggers long partition eviction.
    final int keysCount = 500_000;
    try (IgniteDataStreamer ds = ignite.dataStreamer(CACHE_NAME)) {
        log.info("Writing initial data...");
        ds.allowOverwrite(true);
        for (int k = 1; k <= keysCount; k++) {
            ds.addData(k, k);
            if (k % 50_000 == 0)
                log.info("Written " + k + " entities.");
        }
        log.info("Writing initial data finished.");
    }
    startGrid(3);
    // Trigger partition eviction from other nodes.
    resetBaselineTopology();
    stopGrid(3);
    // Trigger evicting partitions rebalancing.
    resetBaselineTopology();
    // Emulate stopping grid during partition eviction.
    stopGrid(1);
    // Started node should have partition in RENTING or EVICTED state.
    startGrid(1);
    awaitPartitionMapExchange();
    // Check no data loss.
    for (int k = 1; k <= keysCount; k++) {
        Integer value = (Integer) ignite.cache(CACHE_NAME).get(k);
        Assert.assertNotNull("Value for " + k + " is null", value);
        Assert.assertEquals("Check failed for " + k + " = " + value, k, (int) value);
    }
}