Search in sources :

Example 1 with Configurations

use of org.infinispan.configuration.cache.Configurations in project infinispan by infinispan.

the class ScatteredStateConsumerImpl method onTaskCompletion.

@Override
protected void onTaskCompletion(InboundTransferTask inboundTransfer) {
    // a bit of overkill since we start these tasks for single segment
    IntSet completedSegments = IntSets.immutableEmptySet();
    if (log.isTraceEnabled())
        log.tracef("Inbound transfer finished %s: %s", inboundTransfer, inboundTransfer.isCompletedSuccessfully() ? "successfully" : "unsuccessfuly");
    synchronized (transferMapsLock) {
        // transferMapsLock is held when all the tasks are added so we see that all of them are done
        for (PrimitiveIterator.OfInt iter = inboundTransfer.getSegments().iterator(); iter.hasNext(); ) {
            int segment = iter.nextInt();
            List<InboundTransferTask> transfers = transfersBySegment.get(segment);
            if (transfers == null) {
                // It is possible that two task complete concurrently, one of them checks is all tasks
                // for given segments have been completed successfully and (finding out that it's true)
                // removes the transfer for given segment. The second task arrives and finds out that
                // its record int transfersBySegment is gone, but that's OK, as the segment has been handled.
                log.tracef("Transfers for segment %d have not been found.", segment);
            } else {
                // We are removing here rather than in removeTransfer, because we need to know if we're the last
                // finishing task.
                transfers.remove(inboundTransfer);
                if (transfers.isEmpty()) {
                    transfersBySegment.remove(segment);
                    if (log.isTraceEnabled()) {
                        log.tracef("All transfer tasks for segment %d have completed.", segment);
                    }
                    svm.notifyKeyTransferFinished(segment, inboundTransfer.isCompletedSuccessfully(), inboundTransfer.isCancelled());
                    switch(completedSegments.size()) {
                        case 0:
                            completedSegments = IntSets.immutableSet(segment);
                            break;
                        case 1:
                            completedSegments = IntSets.mutableCopyFrom(completedSegments);
                        // Intentional falls through
                        default:
                            completedSegments.set(segment);
                    }
                }
            }
        }
    }
    if (completedSegments.isEmpty()) {
        log.tracef("Not requesting any values yet because no segments have been completed.");
    } else if (inboundTransfer.isCompletedSuccessfully()) {
        log.tracef("Requesting values from segments %s, for in-memory keys", completedSegments);
        dataContainer.forEach(completedSegments, ice -> {
            // TODO: could the version be null in here?
            if (ice.getMetadata() instanceof RemoteMetadata) {
                Address backup = ((RemoteMetadata) ice.getMetadata()).getAddress();
                retrieveEntry(ice.getKey(), backup);
                for (Address member : cacheTopology.getActualMembers()) {
                    if (!member.equals(backup)) {
                        invalidate(ice.getKey(), ice.getMetadata().version(), member);
                    }
                }
            } else {
                backupEntry(ice);
                for (Address member : nonBackupAddresses) {
                    invalidate(ice.getKey(), ice.getMetadata().version(), member);
                }
            }
        });
        // With passivation, some key could be activated here and we could miss it,
        // but then it should be broadcast-loaded in PrefetchInvalidationInterceptor
        Publisher<MarshallableEntry<Object, Object>> persistencePublisher = persistenceManager.publishEntries(completedSegments, k -> dataContainer.peek(k) == null, true, true, Configurations::isStateTransferStore);
        try {
            blockingSubscribe(Flowable.fromPublisher(persistencePublisher).doOnNext(me -> {
                try {
                    Metadata metadata = me.getMetadata();
                    if (metadata instanceof RemoteMetadata) {
                        Address backup = ((RemoteMetadata) metadata).getAddress();
                        retrieveEntry(me.getKey(), backup);
                        for (Address member : cacheTopology.getActualMembers()) {
                            if (!member.equals(backup)) {
                                invalidate(me.getKey(), metadata.version(), member);
                            }
                        }
                    } else {
                        backupEntry(entryFactory.create(me.getKey(), me.getValue(), me.getMetadata()));
                        for (Address member : nonBackupAddresses) {
                            invalidate(me.getKey(), metadata.version(), member);
                        }
                    }
                } catch (CacheException e) {
                    log.failedLoadingValueFromCacheStore(me.getKey(), e);
                }
            }));
        } catch (CacheException e) {
            PERSISTENCE.failedLoadingKeysFromCacheStore(e);
        }
    }
    boolean lastTransfer = false;
    synchronized (transferMapsLock) {
        inboundSegments.removeAll(completedSegments);
        log.tracef("Unfinished inbound segments: " + inboundSegments);
        if (inboundSegments.isEmpty()) {
            lastTransfer = true;
        }
    }
    if (lastTransfer) {
        for (Map.Entry<Address, BlockingQueue<Object>> pair : retrievedEntries.entrySet()) {
            BlockingQueue<Object> queue = pair.getValue();
            List<Object> keys = new ArrayList<>(queue.size());
            queue.drainTo(keys);
            if (!keys.isEmpty()) {
                getValuesAndApply(pair.getKey(), keys);
            }
        }
        List<InternalCacheEntry<?, ?>> entries = new ArrayList<>(backupQueue.size());
        backupQueue.drainTo(entries);
        if (!entries.isEmpty()) {
            backupEntries(entries);
        }
        for (Map.Entry<Address, BlockingQueue<KeyAndVersion>> pair : invalidations.entrySet()) {
            BlockingQueue<KeyAndVersion> queue = pair.getValue();
            List<KeyAndVersion> list = new ArrayList<>(queue.size());
            queue.drainTo(list);
            if (!list.isEmpty()) {
                invalidate(list, pair.getKey());
            }
        }
    }
    // we must not remove the transfer before the requests for values are sent
    // as we could notify the end of rebalance too soon
    removeTransfer(inboundTransfer);
    if (lastTransfer) {
        if (log.isTraceEnabled())
            log.tracef("Inbound transfer removed, chunk counter is %s", chunkCounter.get());
        if (chunkCounter.get() == 0) {
            // No values to transfer after all the keys were received, we can end state transfer immediately
            notifyEndOfStateTransferIfNeeded();
        }
    }
}
Also used : IntSets(org.infinispan.commons.util.IntSets) LogFactory(org.infinispan.util.logging.LogFactory) EntryVersion(org.infinispan.container.versioning.EntryVersion) RemoteMetadata(org.infinispan.container.entries.RemoteMetadata) ClusteredGetAllCommand(org.infinispan.commands.remote.ClusteredGetAllCommand) CONTAINER(org.infinispan.util.logging.Log.CONTAINER) CompletableFutures(org.infinispan.util.concurrent.CompletableFutures) Map(java.util.Map) PERSISTENCE(org.infinispan.util.logging.Log.PERSISTENCE) ConsistentHash(org.infinispan.distribution.ch.ConsistentHash) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) BlockingQueue(java.util.concurrent.BlockingQueue) CacheTopology(org.infinispan.topology.CacheTopology) ComponentStatus(org.infinispan.lifecycle.ComponentStatus) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) IntSet(org.infinispan.commons.util.IntSet) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) SuccessfulResponse(org.infinispan.remoting.responses.SuccessfulResponse) PutKeyValueCommand(org.infinispan.commands.write.PutKeyValueCommand) InboundTransferTask(org.infinispan.statetransfer.InboundTransferTask) GuardedBy(net.jcip.annotations.GuardedBy) InternalCacheValue(org.infinispan.container.entries.InternalCacheValue) InvalidateVersionsCommand(org.infinispan.commands.write.InvalidateVersionsCommand) InternalCacheEntry(org.infinispan.container.entries.InternalCacheEntry) HashMap(java.util.HashMap) Metadata(org.infinispan.metadata.Metadata) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) MapResponseCollector(org.infinispan.remoting.transport.impl.MapResponseCollector) FlagBitSets(org.infinispan.context.impl.FlagBitSets) InternalMetadataImpl(org.infinispan.metadata.impl.InternalMetadataImpl) Configurations(org.infinispan.configuration.cache.Configurations) SingleResponseCollector(org.infinispan.remoting.transport.impl.SingleResponseCollector) Log(org.infinispan.util.logging.Log) MarshallableEntry(org.infinispan.persistence.spi.MarshallableEntry) Address(org.infinispan.remoting.transport.Address) Response(org.infinispan.remoting.responses.Response) InternalEntryFactory(org.infinispan.container.impl.InternalEntryFactory) CacheException(org.infinispan.commons.CacheException) Flowable(io.reactivex.rxjava3.core.Flowable) Iterator(java.util.Iterator) Publisher(org.reactivestreams.Publisher) PutMapCommand(org.infinispan.commands.write.PutMapCommand) SimpleClusteredVersion(org.infinispan.container.versioning.SimpleClusteredVersion) Inject(org.infinispan.factories.annotations.Inject) StateConsumerImpl(org.infinispan.statetransfer.StateConsumerImpl) PrimitiveIterator(java.util.PrimitiveIterator) AtomicLong(java.util.concurrent.atomic.AtomicLong) CacheRpcCommand(org.infinispan.commands.remote.CacheRpcCommand) Collections(java.util.Collections) ScatteredVersionManager(org.infinispan.scattered.ScatteredVersionManager) SuspectException(org.infinispan.remoting.transport.jgroups.SuspectException) Address(org.infinispan.remoting.transport.Address) CacheException(org.infinispan.commons.CacheException) IntSet(org.infinispan.commons.util.IntSet) RemoteMetadata(org.infinispan.container.entries.RemoteMetadata) Metadata(org.infinispan.metadata.Metadata) ArrayList(java.util.ArrayList) InboundTransferTask(org.infinispan.statetransfer.InboundTransferTask) BlockingQueue(java.util.concurrent.BlockingQueue) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) RemoteMetadata(org.infinispan.container.entries.RemoteMetadata) PrimitiveIterator(java.util.PrimitiveIterator) Publisher(org.reactivestreams.Publisher) InternalCacheEntry(org.infinispan.container.entries.InternalCacheEntry) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap)

Example 2 with Configurations

use of org.infinispan.configuration.cache.Configurations in project infinispan by infinispan.

the class ScatteredStateProviderImpl method publishStoreKeys.

private Flowable<InternalCacheEntry<Object, Object>> publishStoreKeys(IntSet segments) {
    Address localAddress = rpcManager.getAddress();
    Publisher<MarshallableEntry<Object, Object>> loaderPublisher = persistenceManager.publishEntries(segments, k -> dataContainer.peek(k) == null, true, true, Configurations::isStateTransferStore);
    return Flowable.fromPublisher(loaderPublisher).filter(me -> me.getMetadata() != null && me.getMetadata().version() != null).map(me -> entryFactory.create(me.getKey(), null, new RemoteMetadata(localAddress, me.getMetadata().version())));
}
Also used : InternalCacheEntry(org.infinispan.container.entries.InternalCacheEntry) IntSets(org.infinispan.commons.util.IntSets) LogFactory(org.infinispan.util.logging.LogFactory) RemoteMetadata(org.infinispan.container.entries.RemoteMetadata) CompletableFuture(java.util.concurrent.CompletableFuture) StateProviderImpl(org.infinispan.statetransfer.StateProviderImpl) MapResponseCollector(org.infinispan.remoting.transport.impl.MapResponseCollector) HashSet(java.util.HashSet) StateChunk(org.infinispan.statetransfer.StateChunk) Configurations(org.infinispan.configuration.cache.Configurations) CompletableFutures(org.infinispan.util.concurrent.CompletableFutures) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Log(org.infinispan.util.logging.Log) MarshallableEntry(org.infinispan.persistence.spi.MarshallableEntry) Address(org.infinispan.remoting.transport.Address) Flowable(io.reactivex.rxjava3.core.Flowable) Iterator(java.util.Iterator) Collection(java.util.Collection) Publisher(org.reactivestreams.Publisher) ScatteredStateProvider(org.infinispan.scattered.ScatteredStateProvider) Set(java.util.Set) CacheTopology(org.infinispan.topology.CacheTopology) SimpleClusteredVersion(org.infinispan.container.versioning.SimpleClusteredVersion) Collectors(java.util.stream.Collectors) Inject(org.infinispan.factories.annotations.Inject) IntSet(org.infinispan.commons.util.IntSet) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) OutboundTransferTask(org.infinispan.statetransfer.OutboundTransferTask) ScatteredVersionManager(org.infinispan.scattered.ScatteredVersionManager) RemoteMetadata(org.infinispan.container.entries.RemoteMetadata) Address(org.infinispan.remoting.transport.Address) MarshallableEntry(org.infinispan.persistence.spi.MarshallableEntry) Configurations(org.infinispan.configuration.cache.Configurations)

Aggregations

Flowable (io.reactivex.rxjava3.core.Flowable)2 Collection (java.util.Collection)2 Iterator (java.util.Iterator)2 List (java.util.List)2 CompletionStage (java.util.concurrent.CompletionStage)2 IntSet (org.infinispan.commons.util.IntSet)2 IntSets (org.infinispan.commons.util.IntSets)2 Configurations (org.infinispan.configuration.cache.Configurations)2 InternalCacheEntry (org.infinispan.container.entries.InternalCacheEntry)2 RemoteMetadata (org.infinispan.container.entries.RemoteMetadata)2 SimpleClusteredVersion (org.infinispan.container.versioning.SimpleClusteredVersion)2 Inject (org.infinispan.factories.annotations.Inject)2 MarshallableEntry (org.infinispan.persistence.spi.MarshallableEntry)2 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 Map (java.util.Map)1 PrimitiveIterator (java.util.PrimitiveIterator)1 Set (java.util.Set)1