Search in sources :

Example 1 with A

use of org.apache.ignite.internal.util.typedef.internal.A in project ignite by apache.

the class CacheUtils method sparseFold.

/**
 * Sparse version of fold. This method also applicable to sparse zeroes.
 *
 * @param cacheName Cache name.
 * @param folder Folder.
 * @param keyFilter Key filter.
 * @param accumulator Accumulator.
 * @param zeroValSupp Zero value supplier.
 * @param defVal Default value.
 * @param defKey Default key.
 * @param defValCnt Def value count.
 * @param isNilpotent Is nilpotent.
 */
private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder, IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, IgniteSupplier<A> zeroValSupp, V defVal, K defKey, long defValCnt, boolean isNilpotent) {
    A defRes = zeroValSupp.get();
    if (!isNilpotent)
        for (int i = 0; i < defValCnt; i++) defRes = folder.apply(new CacheEntryImpl<>(defKey, defVal), defRes);
    Collection<A> totalRes = bcast(cacheName, () -> {
        Ignite ignite = Ignition.localIgnite();
        IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
        int partsCnt = ignite.affinity(cacheName).partitions();
        // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
        Affinity affinity = ignite.affinity(cacheName);
        ClusterNode locNode = ignite.cluster().localNode();
        A a = zeroValSupp.get();
        // Iterate over all partitions. Some of them will be stored on that local node.
        for (int part = 0; part < partsCnt; part++) {
            int p = part;
            // Query returns an empty cursor if this partition is not stored on this node.
            for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) a = folder.apply(entry, a);
        }
        return a;
    });
    return totalRes.stream().reduce(defRes, accumulator);
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) IgniteCallable(org.apache.ignite.lang.IgniteCallable) MatrixBlockEntry(org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry) RowColMatrixKey(org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey) DataStructureCacheKey(org.apache.ignite.ml.math.distributed.keys.DataStructureCacheKey) ClusterNode(org.apache.ignite.cluster.ClusterNode) MatrixBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey) VectorBlockEntry(org.apache.ignite.ml.math.impls.vector.VectorBlockEntry) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) Map(java.util.Map) Cache(javax.cache.Cache) IgniteBinaryOperator(org.apache.ignite.ml.math.functions.IgniteBinaryOperator) UnsupportedOperationException(org.apache.ignite.ml.math.exceptions.UnsupportedOperationException) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) VectorBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey) A(org.apache.ignite.internal.util.typedef.internal.A) IgniteSupplier(org.apache.ignite.ml.math.functions.IgniteSupplier) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IgniteTriFunction(org.apache.ignite.ml.math.functions.IgniteTriFunction) Set(java.util.Set) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) Stream(java.util.stream.Stream) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) IgniteDoubleFunction(org.apache.ignite.ml.math.functions.IgniteDoubleFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) A(org.apache.ignite.internal.util.typedef.internal.A) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) Affinity(org.apache.ignite.cache.affinity.Affinity) Ignite(org.apache.ignite.Ignite) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Example 2 with A

use of org.apache.ignite.internal.util.typedef.internal.A in project ignite by apache.

the class CacheUtils method fold.

/**
 * <b>Currently fold supports only commutative operations.<b/>
 *
 * @param cacheName Cache name.
 * @param folder Fold function operating over cache entries.
 * @param <K> Cache key object type.
 * @param <V> Cache value object type.
 * @param <A> Fold result type.
 * @return Fold operation result.
 */
public static <K, V, A> Collection<A> fold(String cacheName, IgniteBiFunction<CacheEntry<K, V>, A, A> folder, IgnitePredicate<K> keyFilter) {
    return bcast(cacheName, () -> {
        Ignite ignite = Ignition.localIgnite();
        IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
        int partsCnt = ignite.affinity(cacheName).partitions();
        // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
        Affinity affinity = ignite.affinity(cacheName);
        ClusterNode locNode = ignite.cluster().localNode();
        A a = null;
        // Iterate over all partitions. Some of them will be stored on that local node.
        for (int part = 0; part < partsCnt; part++) {
            int p = part;
            // Query returns an empty cursor if this partition is not stored on this node.
            for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) a = folder.apply(new CacheEntry<>(entry, cache), a);
        }
        return a;
    });
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) IgniteCallable(org.apache.ignite.lang.IgniteCallable) MatrixBlockEntry(org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry) RowColMatrixKey(org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey) DataStructureCacheKey(org.apache.ignite.ml.math.distributed.keys.DataStructureCacheKey) ClusterNode(org.apache.ignite.cluster.ClusterNode) MatrixBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey) VectorBlockEntry(org.apache.ignite.ml.math.impls.vector.VectorBlockEntry) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) Map(java.util.Map) Cache(javax.cache.Cache) IgniteBinaryOperator(org.apache.ignite.ml.math.functions.IgniteBinaryOperator) UnsupportedOperationException(org.apache.ignite.ml.math.exceptions.UnsupportedOperationException) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) VectorBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey) A(org.apache.ignite.internal.util.typedef.internal.A) IgniteSupplier(org.apache.ignite.ml.math.functions.IgniteSupplier) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IgniteTriFunction(org.apache.ignite.ml.math.functions.IgniteTriFunction) Set(java.util.Set) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) Stream(java.util.stream.Stream) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) IgniteDoubleFunction(org.apache.ignite.ml.math.functions.IgniteDoubleFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) A(org.apache.ignite.internal.util.typedef.internal.A) Affinity(org.apache.ignite.cache.affinity.Affinity) Ignite(org.apache.ignite.Ignite) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Example 3 with A

use of org.apache.ignite.internal.util.typedef.internal.A in project ignite by apache.

the class CacheUtils method update.

/**
 * @param cacheName Cache name.
 * @param fun An operation that accepts a cache entry and processes it.
 * @param ignite Ignite.
 * @param keysGen Keys generator.
 * @param <K> Cache key object type.
 * @param <V> Cache value object type.
 */
public static <K, V> void update(String cacheName, Ignite ignite, IgniteBiFunction<Ignite, Cache.Entry<K, V>, Stream<Cache.Entry<K, V>>> fun, IgniteSupplier<Set<K>> keysGen) {
    bcast(cacheName, ignite, () -> {
        Ignite ig = Ignition.localIgnite();
        IgniteCache<K, V> cache = ig.getOrCreateCache(cacheName);
        Affinity<K> affinity = ig.affinity(cacheName);
        ClusterNode locNode = ig.cluster().localNode();
        Collection<K> ks = affinity.mapKeysToNodes(keysGen.get()).get(locNode);
        if (ks == null)
            return;
        Map<K, V> m = new ConcurrentHashMap<>();
        ks.parallelStream().forEach(k -> {
            V v = cache.localPeek(k);
            if (v != null)
                (fun.apply(ignite, new CacheEntryImpl<>(k, v))).forEach(ent -> m.put(ent.getKey(), ent.getValue()));
        });
        cache.putAll(m);
    });
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) IgniteCallable(org.apache.ignite.lang.IgniteCallable) MatrixBlockEntry(org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry) RowColMatrixKey(org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey) DataStructureCacheKey(org.apache.ignite.ml.math.distributed.keys.DataStructureCacheKey) ClusterNode(org.apache.ignite.cluster.ClusterNode) MatrixBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey) VectorBlockEntry(org.apache.ignite.ml.math.impls.vector.VectorBlockEntry) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) Map(java.util.Map) Cache(javax.cache.Cache) IgniteBinaryOperator(org.apache.ignite.ml.math.functions.IgniteBinaryOperator) UnsupportedOperationException(org.apache.ignite.ml.math.exceptions.UnsupportedOperationException) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) VectorBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey) A(org.apache.ignite.internal.util.typedef.internal.A) IgniteSupplier(org.apache.ignite.ml.math.functions.IgniteSupplier) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IgniteTriFunction(org.apache.ignite.ml.math.functions.IgniteTriFunction) Set(java.util.Set) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) Stream(java.util.stream.Stream) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) IgniteDoubleFunction(org.apache.ignite.ml.math.functions.IgniteDoubleFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) Ignite(org.apache.ignite.Ignite) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 4 with A

use of org.apache.ignite.internal.util.typedef.internal.A in project ignite by apache.

the class CacheUtils method foreach.

/**
 * @param cacheName Cache name.
 * @param fun An operation that accepts a cache entry and processes it.
 * @param keyFilter Cache keys filter.
 * @param <K> Cache key object type.
 * @param <V> Cache value object type.
 */
protected static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun, IgnitePredicate<K> keyFilter) {
    bcast(cacheName, () -> {
        Ignite ignite = Ignition.localIgnite();
        IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
        int partsCnt = ignite.affinity(cacheName).partitions();
        // Use affinity in filter for scan query. Otherwise we accept consumer in each node which is wrong.
        Affinity affinity = ignite.affinity(cacheName);
        ClusterNode locNode = ignite.cluster().localNode();
        // Iterate over all partitions. Some of them will be stored on that local node.
        for (int part = 0; part < partsCnt; part++) {
            int p = part;
            // Query returns an empty cursor if this partition is not stored on this node.
            for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) fun.accept(new CacheEntry<>(entry, cache));
        }
    });
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) IgniteCallable(org.apache.ignite.lang.IgniteCallable) MatrixBlockEntry(org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry) RowColMatrixKey(org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey) DataStructureCacheKey(org.apache.ignite.ml.math.distributed.keys.DataStructureCacheKey) ClusterNode(org.apache.ignite.cluster.ClusterNode) MatrixBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey) VectorBlockEntry(org.apache.ignite.ml.math.impls.vector.VectorBlockEntry) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) Map(java.util.Map) Cache(javax.cache.Cache) IgniteBinaryOperator(org.apache.ignite.ml.math.functions.IgniteBinaryOperator) UnsupportedOperationException(org.apache.ignite.ml.math.exceptions.UnsupportedOperationException) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) VectorBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey) A(org.apache.ignite.internal.util.typedef.internal.A) IgniteSupplier(org.apache.ignite.ml.math.functions.IgniteSupplier) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IgniteTriFunction(org.apache.ignite.ml.math.functions.IgniteTriFunction) Set(java.util.Set) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) Stream(java.util.stream.Stream) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) IgniteDoubleFunction(org.apache.ignite.ml.math.functions.IgniteDoubleFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) Affinity(org.apache.ignite.cache.affinity.Affinity) Ignite(org.apache.ignite.Ignite) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Example 5 with A

use of org.apache.ignite.internal.util.typedef.internal.A in project ignite by apache.

the class CacheUtils method reduce.

/**
 * Distributed version of fold operation. This method also applicable to sparse zeroes.
 *
 * @param cacheName Cache name.
 * @param ignite ignite
 * @param acc Accumulator
 * @param supp supplier
 * @param entriesGen entries generator
 * @param comb combiner
 * @param zeroValSupp Zero value supplier.
 * @return aggregated result
 */
public static <K, V, A, W> A reduce(String cacheName, Ignite ignite, IgniteTriFunction<W, Cache.Entry<K, V>, A, A> acc, IgniteSupplier<W> supp, IgniteSupplier<Iterable<Cache.Entry<K, V>>> entriesGen, IgniteBinaryOperator<A> comb, IgniteSupplier<A> zeroValSupp) {
    A defRes = zeroValSupp.get();
    Collection<A> totalRes = bcast(cacheName, ignite, () -> {
        // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
        A a = zeroValSupp.get();
        W w = supp.get();
        for (Cache.Entry<K, V> kvEntry : entriesGen.get()) a = acc.apply(w, kvEntry, a);
        return a;
    });
    return totalRes.stream().reduce(defRes, comb);
}
Also used : A(org.apache.ignite.internal.util.typedef.internal.A) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Aggregations

Cache (javax.cache.Cache)5 IgniteCache (org.apache.ignite.IgniteCache)5 A (org.apache.ignite.internal.util.typedef.internal.A)5 Collection (java.util.Collection)4 Collections (java.util.Collections)4 Map (java.util.Map)4 Objects (java.util.Objects)4 Set (java.util.Set)4 UUID (java.util.UUID)4 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)4 BinaryOperator (java.util.function.BinaryOperator)4 Stream (java.util.stream.Stream)4 Ignite (org.apache.ignite.Ignite)4 Ignition (org.apache.ignite.Ignition)4 Affinity (org.apache.ignite.cache.affinity.Affinity)4 ScanQuery (org.apache.ignite.cache.query.ScanQuery)4 ClusterGroup (org.apache.ignite.cluster.ClusterGroup)4 ClusterNode (org.apache.ignite.cluster.ClusterNode)4 CacheEntryImpl (org.apache.ignite.internal.processors.cache.CacheEntryImpl)4 IgniteBiTuple (org.apache.ignite.lang.IgniteBiTuple)4