Examples with IgniteDataStreamer org.apache.ignite.IgniteDataStreamer used on opensource projects

Example 11 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteClientReconnectFailoverTest method testReconnectStreamerApi.

/**
 * @throws Exception If failed.
 */
public void testReconnectStreamerApi() throws Exception {
    final Ignite client = grid(serverCount());
    reconnectFailover(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            stream(ATOMIC_CACHE);
            stream(TX_CACHE);
            return null;
        }

        private void stream(String cacheName) {
            ThreadLocalRandom rnd = ThreadLocalRandom.current();
            try (IgniteDataStreamer<Integer, Integer> streamer = client.dataStreamer(cacheName)) {
                streamer.allowOverwrite(true);
                streamer.perNodeBufferSize(10);
                for (int i = 0; i < 100; i++) streamer.addData(rnd.nextInt(100_000), 0);
            }
        }
    });
}

Example 12 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgniteCacheClearDuringRebalanceTest method populate.

/**
 * @param node Ignite node;
 * @throws Exception If failed.
 */
private void populate(final Ignite node) throws Exception {
    final AtomicInteger id = new AtomicInteger();
    final int tCnt = Runtime.getRuntime().availableProcessors();
    final byte[] data = new byte[1024];
    ThreadLocalRandom.current().nextBytes(data);
    GridTestUtils.runMultiThreaded(new Runnable() {

        @Override
        public void run() {
            try (IgniteDataStreamer<Object, Object> str = node.dataStreamer(CACHE_NAME)) {
                int idx = id.getAndIncrement();
                str.autoFlushFrequency(0);
                for (int i = idx; i < 500_000; i += tCnt) {
                    str.addData(i, data);
                    if (i % (100 * tCnt) == idx)
                        str.flush();
                }
                str.flush();
            }
        }
    }, tCnt, "ldr");
    assertEquals(500_000, node.cache(CACHE_NAME).size());
}

Example 13 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class IgnitePdsCacheRebalancingAbstractTest method testPartitionCounterConsistencyOnUnstableTopology.

/**
 * @throws Exception If failed
 */
public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception {
    System.setProperty(IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "1");
    final Ignite ig = startGrids(4);
    ig.cluster().active(true);
    int k = 0;
    try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) {
        ds.allowOverwrite(true);
        for (int k0 = k; k < k0 + 50_000; k++) ds.addData(k, k);
    }
    for (int t = 0; t < 5; t++) {
        int t0 = t;
        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
            try {
                stopGrid(3);
                // Clear checkpoint history to avoid rebalance from WAL.
                forceCheckpoint();
                forceCheckpoint();
                // Wait for data load.
                U.sleep(500);
                IgniteEx ig0 = startGrid(3);
                // Wait for node join.
                U.sleep(2000);
                if (t0 % 2 == 1) {
                    stopGrid(2);
                    awaitPartitionMapExchange();
                    // Clear checkpoint history to avoid rebalance from WAL.
                    forceCheckpoint();
                    forceCheckpoint();
                    startGrid(2);
                    awaitPartitionMapExchange();
                }
                ig0.cache(cacheName).rebalance().get();
            } catch (Exception e) {
                error("Unable to start/stop grid", e);
                throw new RuntimeException(e);
            }
        });
        try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) {
            ds.allowOverwrite(true);
            while (!fut.isDone()) {
                int k0 = k;
                for (; k < k0 + 3; k++) ds.addData(k, k);
                U.sleep(1);
            }
        } catch (Exception e) {
            log.error("Unable to write data", e);
        }
        fut.get();
        log.info("Checking data...");
        Map<Integer, Long> cntrs = new HashMap<>();
        for (int g = 0; g < 4; g++) {
            IgniteEx ig0 = grid(g);
            for (GridDhtLocalPartition part : ig0.cachex(cacheName).context().topology().currentLocalPartitions()) {
                if (cntrs.containsKey(part.id()))
                    assertEquals(String.valueOf(part.id()), (long) cntrs.get(part.id()), part.updateCounter());
                else
                    cntrs.put(part.id(), part.updateCounter());
            }
            for (int k0 = 0; k0 < k; k0++) assertEquals(String.valueOf(k0) + " " + g, k0, ig0.cache(cacheName).get(k0));
        }
        assertEquals(ig.affinity(cacheName).partitions(), cntrs.size());
    }
}

Example 14 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class StreamVisitorExample method main.

public static void main(String[] args) throws Exception {
    // Mark this cluster member as client.
    Ignition.setClientMode(true);
    try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
        if (!ExamplesUtils.hasServerNodes(ignite))
            return;
        // Market data cache with default configuration.
        CacheConfiguration<String, Double> mktDataCfg = new CacheConfiguration<>("marketTicks");
        // Financial instrument cache configuration.
        CacheConfiguration<String, Instrument> instCfg = new CacheConfiguration<>("instCache");
        // Index key and value for querying financial instruments.
        // Note that Instrument class has @QuerySqlField annotation for secondary field indexing.
        instCfg.setIndexedTypes(String.class, Instrument.class);
        // Auto-close caches at the end of the example.
        try (IgniteCache<String, Double> mktCache = ignite.getOrCreateCache(mktDataCfg);
            IgniteCache<String, Instrument> instCache = ignite.getOrCreateCache(instCfg)) {
            try (IgniteDataStreamer<String, Double> mktStmr = ignite.dataStreamer(mktCache.getName())) {
                // Note that we receive market data, but do not populate 'mktCache' (it remains empty).
                // Instead we update the instruments in the 'instCache'.
                // Since both, 'instCache' and 'mktCache' use the same key, updates are collocated.
                mktStmr.receiver(StreamVisitor.from((cache, e) -> {
                    String symbol = e.getKey();
                    Double tick = e.getValue();
                    Instrument inst = instCache.get(symbol);
                    if (inst == null)
                        inst = new Instrument(symbol);
                    // Don't populate market cache, as we don't use it for querying.
                    // Update cached instrument based on the latest market tick.
                    inst.update(tick);
                    instCache.put(symbol, inst);
                }));
                // Stream 10 million market data ticks into the system.
                for (int i = 1; i <= 10_000_000; i++) {
                    int idx = RAND.nextInt(INSTRUMENTS.length);
                    // Use gaussian distribution to ensure that
                    // numbers closer to 0 have higher probability.
                    double price = round2(INITIAL_PRICES[idx] + RAND.nextGaussian());
                    mktStmr.addData(INSTRUMENTS[idx], price);
                    if (i % 500_000 == 0)
                        System.out.println("Number of tuples streamed into Ignite: " + i);
                }
            }
            // Select top 3 best performing instruments.
            SqlFieldsQuery top3qry = new SqlFieldsQuery("select symbol, (latest - open) from Instrument order by (latest - open) desc limit 3");
            // Execute queries.
            List<List<?>> top3 = instCache.query(top3qry).getAll();
            System.out.println("Top performing financial instruments: ");
            // Print top 10 words.
            ExamplesUtils.printQueryResults(top3);
        } finally {
            // Distributed cache could be removed from cluster only by #destroyCache() call.
            ignite.destroyCache(mktDataCfg.getName());
            ignite.destroyCache(instCfg.getName());
        }
    }
}

Example 15 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class ColumnDecisionTreeTrainerBenchmark method loadVectorsIntoSparseDistributedMatrixCache.

/**
 * Load vectors into sparse distributed matrix.
 *
 * @param cacheName Name of cache where matrix is stored.
 * @param uuid UUID of matrix.
 * @param iter Iterator over vectors.
 * @param vectorSize size of vectors.
 */
private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid, Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
    try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer = Ignition.localIgnite().dataStreamer(cacheName)) {
        int sampleIdx = 0;
        streamer.allowOverwrite(true);
        streamer.receiver(StreamTransformer.from((e, arg) -> {
            Map<Integer, Double> val = e.getValue();
            if (val == null)
                val = new Int2DoubleOpenHashMap();
            val.putAll((Map<Integer, Double>) arg[0]);
            e.setValue(val);
            return null;
        }));
        // Feature index -> (sample index -> value)
        Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
        IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        int batchSize = 1000;
        while (iter.hasNext()) {
            org.apache.ignite.ml.math.Vector next = iter.next();
            for (int i = 0; i < vectorSize; i++) batch.get(i).put(sampleIdx, next.getX(i));
            X.println("Sample index: " + sampleIdx);
            if (sampleIdx % batchSize == 0) {
                batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
                IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
            }
            sampleIdx++;
        }
        if (sampleIdx % batchSize != 0) {
            batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
            IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        }
    }
}