Search in sources :

Example 1 with IgniteConsumer

use of org.apache.ignite.ml.math.functions.IgniteConsumer in project ignite by apache.

the class CacheUtils method foreach.

/**
     * @param cacheName Cache name.
     * @param fun An operation that accepts a cache entry and processes it.
     * @param keyFilter Cache keys filter.
     * @param <K> Cache key object type.
     * @param <V> Cache value object type.
     */
public static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun, IgnitePredicate<K> keyFilter) {
    bcast(cacheName, () -> {
        Ignite ignite = Ignition.localIgnite();
        IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
        int partsCnt = ignite.affinity(cacheName).partitions();
        // Use affinity in filter for scan query. Otherwise we accept consumer in each node which is wrong.
        Affinity affinity = ignite.affinity(cacheName);
        ClusterNode locNode = ignite.cluster().localNode();
        // Iterate over all partitions. Some of them will be stored on that local node.
        for (int part = 0; part < partsCnt; part++) {
            int p = part;
            // Query returns an empty cursor if this partition is not stored on this node.
            for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) fun.accept(new CacheEntry<>(entry, cache));
        }
    });
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) SparseDistributedMatrix(org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix) SparseDistributedMatrixStorage(org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage) IgniteCallable(org.apache.ignite.lang.IgniteCallable) ClusterNode(org.apache.ignite.cluster.ClusterNode) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) ValueMapper(org.apache.ignite.ml.math.ValueMapper) Map(java.util.Map) Cache(javax.cache.Cache) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) Collection(java.util.Collection) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) IgniteUuid(org.apache.ignite.lang.IgniteUuid) Affinity(org.apache.ignite.cache.affinity.Affinity) Ignite(org.apache.ignite.Ignite) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Example 2 with IgniteConsumer

use of org.apache.ignite.ml.math.functions.IgniteConsumer in project ignite by apache.

the class CacheUtils method foreach.

/**
 * @param cacheName Cache name.
 * @param fun An operation that accepts a cache entry and processes it.
 * @param keyFilter Cache keys filter.
 * @param <K> Cache key object type.
 * @param <V> Cache value object type.
 */
protected static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun, IgnitePredicate<K> keyFilter) {
    bcast(cacheName, () -> {
        Ignite ignite = Ignition.localIgnite();
        IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
        int partsCnt = ignite.affinity(cacheName).partitions();
        // Use affinity in filter for scan query. Otherwise we accept consumer in each node which is wrong.
        Affinity affinity = ignite.affinity(cacheName);
        ClusterNode locNode = ignite.cluster().localNode();
        // Iterate over all partitions. Some of them will be stored on that local node.
        for (int part = 0; part < partsCnt; part++) {
            int p = part;
            // Query returns an empty cursor if this partition is not stored on this node.
            for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) fun.accept(new CacheEntry<>(entry, cache));
        }
    });
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteConsumer(org.apache.ignite.ml.math.functions.IgniteConsumer) IgniteFunction(org.apache.ignite.ml.math.functions.IgniteFunction) Affinity(org.apache.ignite.cache.affinity.Affinity) IgniteCallable(org.apache.ignite.lang.IgniteCallable) MatrixBlockEntry(org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry) RowColMatrixKey(org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey) DataStructureCacheKey(org.apache.ignite.ml.math.distributed.keys.DataStructureCacheKey) ClusterNode(org.apache.ignite.cluster.ClusterNode) MatrixBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey) VectorBlockEntry(org.apache.ignite.ml.math.impls.vector.VectorBlockEntry) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) Map(java.util.Map) Cache(javax.cache.Cache) IgniteBinaryOperator(org.apache.ignite.ml.math.functions.IgniteBinaryOperator) UnsupportedOperationException(org.apache.ignite.ml.math.exceptions.UnsupportedOperationException) ClusterGroup(org.apache.ignite.cluster.ClusterGroup) KeyMapper(org.apache.ignite.ml.math.KeyMapper) CacheEntryImpl(org.apache.ignite.internal.processors.cache.CacheEntryImpl) VectorBlockKey(org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey) A(org.apache.ignite.internal.util.typedef.internal.A) IgniteSupplier(org.apache.ignite.ml.math.functions.IgniteSupplier) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IgniteTriFunction(org.apache.ignite.ml.math.functions.IgniteTriFunction) Set(java.util.Set) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) BinaryOperator(java.util.function.BinaryOperator) IgniteCache(org.apache.ignite.IgniteCache) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) Stream(java.util.stream.Stream) Ignition(org.apache.ignite.Ignition) IgniteBiFunction(org.apache.ignite.ml.math.functions.IgniteBiFunction) IgniteDoubleFunction(org.apache.ignite.ml.math.functions.IgniteDoubleFunction) Collections(java.util.Collections) ScanQuery(org.apache.ignite.cache.query.ScanQuery) Affinity(org.apache.ignite.cache.affinity.Affinity) Ignite(org.apache.ignite.Ignite) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache)

Aggregations

Collection (java.util.Collection)2 Collections (java.util.Collections)2 Map (java.util.Map)2 BinaryOperator (java.util.function.BinaryOperator)2 Cache (javax.cache.Cache)2 Ignite (org.apache.ignite.Ignite)2 IgniteCache (org.apache.ignite.IgniteCache)2 Ignition (org.apache.ignite.Ignition)2 Affinity (org.apache.ignite.cache.affinity.Affinity)2 ScanQuery (org.apache.ignite.cache.query.ScanQuery)2 ClusterGroup (org.apache.ignite.cluster.ClusterGroup)2 ClusterNode (org.apache.ignite.cluster.ClusterNode)2 CacheEntryImpl (org.apache.ignite.internal.processors.cache.CacheEntryImpl)2 IgniteBiTuple (org.apache.ignite.lang.IgniteBiTuple)2 IgniteCallable (org.apache.ignite.lang.IgniteCallable)2 IgnitePredicate (org.apache.ignite.lang.IgnitePredicate)2 IgniteRunnable (org.apache.ignite.lang.IgniteRunnable)2 KeyMapper (org.apache.ignite.ml.math.KeyMapper)2 IgniteBiFunction (org.apache.ignite.ml.math.functions.IgniteBiFunction)2 IgniteConsumer (org.apache.ignite.ml.math.functions.IgniteConsumer)2