Examples with IgniteDataStreamer org.apache.ignite.IgniteDataStreamer used on opensource projects

Example 1 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class PlatformProcessorImpl method dataStreamer.

/** {@inheritDoc} */
@Override
public PlatformTargetProxy dataStreamer(@Nullable String cacheName, boolean keepBinary) throws IgniteCheckedException {
    IgniteDataStreamer ldr = ctx.dataStream().dataStreamer(cacheName);
    ldr.keepBinary(true);
    return proxy(new PlatformDataStreamer(platformCtx, cacheName, (DataStreamerImpl) ldr, keepBinary));
}

Example 2 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class StreamTransformerExample method main.

public static void main(String[] args) throws Exception {
    // Mark this cluster member as client.
    Ignition.setClientMode(true);
    try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
        if (!ExamplesUtils.hasServerNodes(ignite))
            return;
        CacheConfiguration<Integer, Long> cfg = new CacheConfiguration<>(CACHE_NAME);
        // Index key and value.
        cfg.setIndexedTypes(Integer.class, Long.class);
        // Auto-close cache at the end of the example.
        try (IgniteCache<Integer, Long> stmCache = ignite.getOrCreateCache(cfg)) {
            try (IgniteDataStreamer<Integer, Long> stmr = ignite.dataStreamer(stmCache.getName())) {
                // Allow data updates.
                stmr.allowOverwrite(true);
                // Configure data transformation to count random numbers added to the stream.
                stmr.receiver(StreamTransformer.from((e, arg) -> {
                    // Get current count.
                    Long val = e.getValue();
                    // Increment count by 1.
                    e.setValue(val == null ? 1L : val + 1);
                    return null;
                }));
                // Stream 10 million of random numbers into the streamer cache.
                for (int i = 1; i <= 10_000_000; i++) {
                    stmr.addData(RAND.nextInt(RANGE), 1L);
                    if (i % 500_000 == 0)
                        System.out.println("Number of tuples streamed into Ignite: " + i);
                }
            }
            // Query top 10 most popular numbers every.
            SqlFieldsQuery top10Qry = new SqlFieldsQuery("select _key, _val from Long order by _val desc limit 10");
            // Execute queries.
            List<List<?>> top10 = stmCache.query(top10Qry).getAll();
            System.out.println("Top 10 most popular numbers:");
            // Print top 10 words.
            ExamplesUtils.printQueryResults(top10);
        } finally {
            // Distributed cache could be removed from cluster only by #destroyCache() call.
            ignite.destroyCache(CACHE_NAME);
        }
    }
}

Example 3 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class StreamVisitorExample method main.

public static void main(String[] args) throws Exception {
    // Mark this cluster member as client.
    Ignition.setClientMode(true);
    try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
        if (!ExamplesUtils.hasServerNodes(ignite))
            return;
        // Market data cache with default configuration.
        CacheConfiguration<String, Double> mktDataCfg = new CacheConfiguration<>("marketTicks");
        // Financial instrument cache configuration.
        CacheConfiguration<String, Instrument> instCfg = new CacheConfiguration<>("instCache");
        // Index key and value for querying financial instruments.
        // Note that Instrument class has @QuerySqlField annotation for secondary field indexing.
        instCfg.setIndexedTypes(String.class, Instrument.class);
        // Auto-close caches at the end of the example.
        try (IgniteCache<String, Double> mktCache = ignite.getOrCreateCache(mktDataCfg);
            IgniteCache<String, Instrument> instCache = ignite.getOrCreateCache(instCfg)) {
            try (IgniteDataStreamer<String, Double> mktStmr = ignite.dataStreamer(mktCache.getName())) {
                // Note that we receive market data, but do not populate 'mktCache' (it remains empty).
                // Instead we update the instruments in the 'instCache'.
                // Since both, 'instCache' and 'mktCache' use the same key, updates are collocated.
                mktStmr.receiver(StreamVisitor.from((cache, e) -> {
                    String symbol = e.getKey();
                    Double tick = e.getValue();
                    Instrument inst = instCache.get(symbol);
                    if (inst == null)
                        inst = new Instrument(symbol);
                    // Don't populate market cache, as we don't use it for querying.
                    // Update cached instrument based on the latest market tick.
                    inst.update(tick);
                    instCache.put(symbol, inst);
                }));
                // Stream 10 million market data ticks into the system.
                for (int i = 1; i <= 10_000_000; i++) {
                    int idx = RAND.nextInt(INSTRUMENTS.length);
                    // Use gaussian distribution to ensure that
                    // numbers closer to 0 have higher probability.
                    double price = round2(INITIAL_PRICES[idx] + RAND.nextGaussian());
                    mktStmr.addData(INSTRUMENTS[idx], price);
                    if (i % 500_000 == 0)
                        System.out.println("Number of tuples streamed into Ignite: " + i);
                }
            }
            // Select top 3 best performing instruments.
            SqlFieldsQuery top3qry = new SqlFieldsQuery("select symbol, (latest - open) from Instrument order by (latest - open) desc limit 3");
            // Execute queries.
            List<List<?>> top3 = instCache.query(top3qry).getAll();
            System.out.println("Top performing financial instruments: ");
            // Print top 10 words.
            ExamplesUtils.printQueryResults(top3);
        } finally {
            // Distributed cache could be removed from cluster only by #destroyCache() call.
            ignite.destroyCache(mktDataCfg.getName());
            ignite.destroyCache(instCfg.getName());
        }
    }
}

Example 4 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class ColumnDecisionTreeTrainerBenchmark method loadVectorsIntoSparseDistributedMatrixCache.

/**
 * Load vectors into sparse distributed matrix.
 *
 * @param cacheName Name of cache where matrix is stored.
 * @param uuid UUID of matrix.
 * @param iter Iterator over vectors.
 * @param vectorSize size of vectors.
 */
private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid, Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
    try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer = Ignition.localIgnite().dataStreamer(cacheName)) {
        int sampleIdx = 0;
        streamer.allowOverwrite(true);
        streamer.receiver(StreamTransformer.from((e, arg) -> {
            Map<Integer, Double> val = e.getValue();
            if (val == null)
                val = new Int2DoubleOpenHashMap();
            val.putAll((Map<Integer, Double>) arg[0]);
            e.setValue(val);
            return null;
        }));
        // Feature index -> (sample index -> value)
        Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
        IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        int batchSize = 1000;
        while (iter.hasNext()) {
            org.apache.ignite.ml.math.Vector next = iter.next();
            for (int i = 0; i < vectorSize; i++) batch.get(i).put(sampleIdx, next.getX(i));
            X.println("Sample index: " + sampleIdx);
            if (sampleIdx % batchSize == 0) {
                batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
                IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
            }
            sampleIdx++;
        }
        if (sampleIdx % batchSize != 0) {
            batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
            IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        }
    }
}

Example 5 with IgniteDataStreamer

use of org.apache.ignite.IgniteDataStreamer in project ignite by apache.

the class PendingExchangeTest method createClusterWithPendingExchnageDuringRebalance.

/**
 * @param clo Closure triggering exchange.
 * @throws Exception If failed.
 */
private void createClusterWithPendingExchnageDuringRebalance(PendingExchangeTrigger clo) throws Exception {
    IgniteEx ignite0 = startGrids(3);
    try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) {
        for (int i = 0; i < 1000; i++) streamer.addData(i, i);
    }
    awaitPartitionMapExchange();
    GridCachePartitionExchangeManager exchangeManager1 = ignite(1).context().cache().context().exchange();
    CountDownLatch exchangeLatch = new CountDownLatch(1);
    AffinityTopologyVersion readyTop = exchangeManager1.readyAffinityVersion();
    exchangeManager1.registerExchangeAwareComponent(new PartitionsExchangeAware() {

        @Override
        public void onInitAfterTopologyLock(GridDhtPartitionsExchangeFuture fut) {
            U.awaitQuiet(exchangeLatch);
        }
    });
    IgniteInternalFuture startNodeFut = GridTestUtils.runAsync(() -> stopGrid(2));
    assertTrue(GridTestUtils.waitForCondition(() -> exchangeManager1.lastTopologyFuture().initialVersion().after(readyTop), 10_000));
    IgniteInternalFuture exchangeTrigger = clo.trigger(ignite0, exchangeManager1);
    assertTrue(GridTestUtils.waitForCondition(exchangeManager1::hasPendingServerExchange, 10_000));
    exchangeLatch.countDown();
    startNodeFut.get(10_000);
    exchangeTrigger.get(10_000);
    awaitPartitionMapExchange();
}