Examples with DataStreamerImpl org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl used on opensource projects

Example 1 with DataStreamerImpl

use of org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl in project ignite by apache.

the class PlatformProcessorImpl method dataStreamer.

/** {@inheritDoc} */
@Override
public PlatformTargetProxy dataStreamer(@Nullable String cacheName, boolean keepBinary) throws IgniteCheckedException {
    IgniteDataStreamer ldr = ctx.dataStream().dataStreamer(cacheName);
    ldr.keepBinary(true);
    return proxy(new PlatformDataStreamer(platformCtx, cacheName, (DataStreamerImpl) ldr, keepBinary));
}

Example 2 with DataStreamerImpl

use of org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl in project ignite by apache.

the class CacheLoadingConcurrentGridStartSelfTest method loadCacheWithDataStreamerSequential.

/**
 * @throws Exception if failed
 */
private void loadCacheWithDataStreamerSequential() throws Exception {
    startGrid(1);
    Ignite g0 = startGrid(0);
    IgniteInternalFuture<Object> restartFut = runAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            while (restarts) {
                stopGrid(1);
                startGrid(1);
                U.sleep(100);
            }
            return null;
        }
    });
    CountDownLatch startNodesLatch = new CountDownLatch(1);
    IgniteInternalFuture<Object> fut = runAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            startNodesLatch.await();
            for (int i = 2; i < GRIDS_CNT; i++) startGrid(i);
            return null;
        }
    });
    final HashSet<IgniteFuture> set = new HashSet<>();
    boolean stop = false;
    int insertedKeys = 0;
    startNodesLatch.countDown();
    try (IgniteDataStreamer<Integer, String> dataStreamer = g0.dataStreamer(DEFAULT_CACHE_NAME)) {
        dataStreamer.allowOverwrite(allowOverwrite);
        ((DataStreamerImpl) dataStreamer).maxRemapCount(Integer.MAX_VALUE);
        long startingEndTs = -1L;
        while (!stop) {
            set.add(dataStreamer.addData(insertedKeys, "Data"));
            insertedKeys = insertedKeys + 1;
            if (insertedKeys % 100000 == 0)
                log.info("Streaming " + insertedKeys + "'th entry.");
            // When all nodes started we continue restart nodes during 1 second and stop it after this timeout.
            if (fut.isDone() && startingEndTs == -1)
                startingEndTs = System.currentTimeMillis();
            if (// Nodes starting was ended and we check restarts duration after it.
            startingEndTs != -1)
                restarts = (System.currentTimeMillis() - startingEndTs) < 1000;
            // Stop test when all keys were inserted or restarts timeout was exceeded.
            stop = insertedKeys >= KEYS_CNT || (fut.isDone() && !restarts);
        }
    }
    log.info("Data loaded.");
    restarts = false;
    fut.get();
    restartFut.get();
    for (IgniteFuture res : set) assertNull(res.get());
    IgniteCache<Integer, String> cache = grid(0).cache(DEFAULT_CACHE_NAME);
    long size = cache.size(CachePeekMode.PRIMARY);
    if (size != insertedKeys) {
        Set<Integer> failedKeys = new LinkedHashSet<>();
        for (int i = 0; i < insertedKeys; i++) if (!cache.containsKey(i)) {
            log.info("Actual cache size: " + size);
            for (Ignite ignite : G.allGrids()) {
                IgniteEx igniteEx = (IgniteEx) ignite;
                log.info("Missed key info:" + igniteEx.localNode().id() + " primary=" + ignite.affinity(DEFAULT_CACHE_NAME).isPrimary(igniteEx.localNode(), i) + " backup=" + ignite.affinity(DEFAULT_CACHE_NAME).isBackup(igniteEx.localNode(), i) + " local peek=" + ignite.cache(DEFAULT_CACHE_NAME).localPeek(i, CachePeekMode.ONHEAP));
            }
            for (int j = i; j < i + 10000; j++) {
                if (!cache.containsKey(j))
                    failedKeys.add(j);
            }
            break;
        }
        assert failedKeys.isEmpty() : "Some failed keys: " + failedKeys.toString();
    }
    assertCacheSize(insertedKeys);
}

Example 3 with DataStreamerImpl

use of org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl in project ignite by apache.

the class IgniteDataStreamerTest method testStreamerIgniteUuid.

/**
 * @throws Exception If failed.
 */
@Test
public void testStreamerIgniteUuid() throws Exception {
    Ignite client = grid("client");
    IgniteCache<IgniteUuid, Integer> cache = client.createCache(cacheConfiguration(IgniteUuid.class, Integer.class));
    try (IgniteDataStreamer<IgniteUuid, Integer> streamer = client.dataStreamer(CACHE_NAME)) {
        assertTrue("Expecting " + DataStreamerImpl.class.getName(), streamer instanceof DataStreamerImpl);
        ((DataStreamerImpl<IgniteUuid, Integer>) streamer).maxRemapCount(0);
        List<IgniteFuture> futs = new ArrayList<>();
        for (int i = 0; i < DATA_SIZE; i++) {
            IgniteFuture<?> fut = streamer.addData(IgniteUuid.randomUuid(), i);
            futs.add(fut);
        }
        streamer.flush();
        for (IgniteFuture fut : futs) {
            // This should not throw any exception.
            Object res = fut.get(WAIT_TIMEOUT);
            if (log.isDebugEnabled()) {
                // Printing future result to log to prevent jvm optimization
                log.debug(String.valueOf(res));
            }
        }
        assertTrue(cache.size(ALL) == DATA_SIZE);
    }
}

Example 4 with DataStreamerImpl

use of org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl in project ignite by apache.

the class GridCacheAdapter method localLoadCache.

/**
 * {@inheritDoc}
 */
@Override
public void localLoadCache(final IgniteBiPredicate<K, V> p, Object[] args) throws IgniteCheckedException {
    // TODO IGNITE-7954
    MvccUtils.verifyMvccOperationSupport(ctx, "Load");
    final boolean replicate = ctx.isDrEnabled();
    final AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
    CacheOperationContext opCtx = ctx.operationContextPerCall();
    ExpiryPolicy plc0 = opCtx != null ? opCtx.expiry() : null;
    final ExpiryPolicy plc = plc0 != null ? plc0 : ctx.expiry();
    final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
    if (p != null)
        ctx.kernalContext().resource().injectGeneric(p);
    try {
        if (ctx.store().isLocal()) {
            DataStreamerImpl ldr = ctx.kernalContext().dataStream().dataStreamer(ctx.name());
            try {
                ldr.skipStore(true);
                ldr.receiver(new IgniteDrDataStreamerCacheUpdater());
                ldr.keepBinary(keepBinary);
                LocalStoreLoadClosure c = new LocalStoreLoadClosure(p, ldr, plc);
                ctx.store().loadCache(c, args);
                c.onDone();
            } finally {
                ldr.closeEx(false);
            }
        } else {
            // Version for all loaded entries.
            final GridCacheVersion ver0 = ctx.versions().nextForLoad();
            ctx.store().loadCache(new CIX3<KeyCacheObject, Object, GridCacheVersion>() {

                @Override
                public void applyx(KeyCacheObject key, Object val, @Nullable GridCacheVersion ver) throws IgniteException {
                    assert ver == null;
                    long ttl = CU.ttlForLoad(plc);
                    if (ttl == CU.TTL_ZERO)
                        return;
                    loadEntry(key, val, ver0, (IgniteBiPredicate<Object, Object>) p, topVer, replicate, ttl);
                }
            }, args);
        }
    } finally {
        if (p instanceof PlatformCacheEntryFilter)
            ((PlatformCacheEntryFilter) p).onClose();
    }
}

Example 5 with DataStreamerImpl

use of org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl in project ignite by apache.

the class GridCacheAdapter method localLoad.

/**
 * @param keys Keys to load.
 * @param plc Optional expiry policy.
 * @throws IgniteCheckedException If failed.
 */
public void localLoad(Collection<? extends K> keys, @Nullable ExpiryPolicy plc, final boolean keepBinary) throws IgniteCheckedException {
    // TODO IGNITE-7954
    MvccUtils.verifyMvccOperationSupport(ctx, "Load");
    final boolean replicate = ctx.isDrEnabled();
    final AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
    final ExpiryPolicy plc0 = plc != null ? plc : ctx.expiry();
    Collection<KeyCacheObject> keys0 = ctx.cacheKeysView(keys);
    if (ctx.store().isLocal()) {
        DataStreamerImpl ldr = ctx.kernalContext().dataStream().dataStreamer(ctx.name());
        try {
            ldr.skipStore(true);
            ldr.keepBinary(keepBinary);
            ldr.receiver(new IgniteDrDataStreamerCacheUpdater());
            LocalStoreLoadClosure c = new LocalStoreLoadClosure(null, ldr, plc0);
            ctx.store().localStoreLoadAll(null, keys0, c);
            c.onDone();
        } finally {
            ldr.closeEx(false);
        }
    } else {
        // Version for all loaded entries.
        final GridCacheVersion ver0 = ctx.versions().nextForLoad();
        ctx.store().loadAll(null, keys0, new CI2<KeyCacheObject, Object>() {

            @Override
            public void apply(KeyCacheObject key, Object val) {
                long ttl = CU.ttlForLoad(plc0);
                if (ttl == CU.TTL_ZERO)
                    return;
                loadEntry(key, val, ver0, null, topVer, replicate, ttl);
            }
        });
    }
}