Search in sources :

Example 1 with InternalCompletableFuture

use of com.hazelcast.spi.impl.InternalCompletableFuture in project hazelcast by hazelcast.

the class InvalidationMetaDataFetcher method fetchMetadata.

public final void fetchMetadata(ConcurrentMap<String, RepairingHandler> handlers) {
    if (handlers.isEmpty()) {
        return;
    }
    List<String> dataStructureNames = getDataStructureNames(handlers);
    Map<Member, InternalCompletableFuture> futureByMember = fetchMembersMetadataFor(dataStructureNames);
    for (Map.Entry<Member, InternalCompletableFuture> entry : futureByMember.entrySet()) {
        Member member = entry.getKey();
        InternalCompletableFuture future = entry.getValue();
        processMemberMetadata(member, future, handlers);
    }
}
Also used : InternalCompletableFuture(com.hazelcast.spi.impl.InternalCompletableFuture) Member(com.hazelcast.cluster.Member) MapUtil.createHashMap(com.hazelcast.internal.util.MapUtil.createHashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap) Map(java.util.Map)

Example 2 with InternalCompletableFuture

use of com.hazelcast.spi.impl.InternalCompletableFuture in project hazelcast by hazelcast.

the class InvalidationMetaDataFetcher method fetchMembersMetadataFor.

private Map<Member, InternalCompletableFuture> fetchMembersMetadataFor(List<String> names) {
    Collection<Member> members = getDataMembers();
    if (members.isEmpty()) {
        return Collections.emptyMap();
    }
    Map<Member, InternalCompletableFuture> futureByMember = createHashMap(members.size());
    for (Member member : members) {
        Address address = member.getAddress();
        try {
            futureByMember.put(member, fetchMetadataOf(member, names));
        } catch (Exception e) {
            handleExceptionWhileProcessingMetadata(member, e);
        }
    }
    return futureByMember;
}
Also used : Address(com.hazelcast.cluster.Address) InternalCompletableFuture(com.hazelcast.spi.impl.InternalCompletableFuture) Member(com.hazelcast.cluster.Member) ExecutionException(java.util.concurrent.ExecutionException)

Example 3 with InternalCompletableFuture

use of com.hazelcast.spi.impl.InternalCompletableFuture in project hazelcast by hazelcast.

the class MasterContext method invokeOnParticipant.

private void invokeOnParticipant(MemberInfo memberInfo, Supplier<Operation> operationSupplier, @Nullable Consumer<Collection<Map.Entry<MemberInfo, Object>>> completionCallback, @Nullable BiConsumer<Address, Object> individualCallback, boolean retryOnTimeoutException, ConcurrentMap<MemberInfo, Object> collectedResponses, AtomicInteger remainingCount) {
    Operation operation = operationSupplier.get();
    InternalCompletableFuture<Object> future = nodeEngine.getOperationService().createInvocationBuilder(JetServiceBackend.SERVICE_NAME, operation, memberInfo.getAddress()).invoke();
    future.whenCompleteAsync(withTryCatch(logger, (r, throwable) -> {
        Object response = r != null ? r : throwable != null ? peel(throwable) : NULL_OBJECT;
        if (retryOnTimeoutException && throwable instanceof OperationTimeoutException) {
            logger.warning("Retrying " + operation.getClass().getName() + " that failed with " + OperationTimeoutException.class.getSimpleName() + " in " + jobIdString());
            invokeOnParticipant(memberInfo, operationSupplier, completionCallback, individualCallback, retryOnTimeoutException, collectedResponses, remainingCount);
            return;
        }
        if (individualCallback != null) {
            individualCallback.accept(memberInfo.getAddress(), throwable != null ? peel(throwable) : r);
        }
        Object oldResponse = collectedResponses.put(memberInfo, response);
        assert oldResponse == null : "Duplicate response for " + memberInfo.getAddress() + ". Old=" + oldResponse + ", new=" + response;
        if (remainingCount.decrementAndGet() == 0 && completionCallback != null) {
            completionCallback.accept(collectedResponses.entrySet().stream().map(e -> e.getValue() == NULL_OBJECT ? entry(e.getKey(), null) : e).collect(Collectors.toList()));
        }
    }));
}
Also used : Address(com.hazelcast.cluster.Address) NOT_RUNNING(com.hazelcast.jet.core.JobStatus.NOT_RUNNING) CompletableFuture(java.util.concurrent.CompletableFuture) StartExecutionOperation(com.hazelcast.jet.impl.operation.StartExecutionOperation) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ConcurrentMap(java.util.concurrent.ConcurrentMap) InternalCompletableFuture(com.hazelcast.spi.impl.InternalCompletableFuture) Util.jobNameAndExecutionId(com.hazelcast.jet.impl.util.Util.jobNameAndExecutionId) ILogger(com.hazelcast.logging.ILogger) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Operation(com.hazelcast.spi.impl.operationservice.Operation) Util.entry(com.hazelcast.jet.Util.entry) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) BiConsumer(java.util.function.BiConsumer) SUSPENDED(com.hazelcast.jet.core.JobStatus.SUSPENDED) JobStatus(com.hazelcast.jet.core.JobStatus) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Collectors.toConcurrentMap(java.util.stream.Collectors.toConcurrentMap) Collection(java.util.Collection) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Collectors(java.util.stream.Collectors) Consumer(java.util.function.Consumer) Util.idToString(com.hazelcast.jet.Util.idToString) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) OperationTimeoutException(com.hazelcast.core.OperationTimeoutException) Entry(java.util.Map.Entry) OperationTimeoutException(com.hazelcast.core.OperationTimeoutException) StartExecutionOperation(com.hazelcast.jet.impl.operation.StartExecutionOperation) Operation(com.hazelcast.spi.impl.operationservice.Operation)

Example 4 with InternalCompletableFuture

use of com.hazelcast.spi.impl.InternalCompletableFuture in project hazelcast by hazelcast.

the class InvocationUtil method invokeOnStableClusterSerial.

/**
 * Invoke operation on all cluster members.
 *
 * The invocation is serial: It iterates over all members starting from the oldest member to the youngest one.
 * If there is a cluster membership change while invoking then it will restart invocations on all members. This
 * implies the operation should be idempotent.
 *
 * If there is an exception - other than {@link com.hazelcast.core.MemberLeftException} or
 * {@link com.hazelcast.spi.exception.TargetNotMemberException} while invoking then the iteration
 * is interrupted and the exception is propagated to the caller.
 */
public static <V> InternalCompletableFuture<V> invokeOnStableClusterSerial(NodeEngine nodeEngine, Supplier<? extends Operation> operationSupplier, int maxRetries) {
    ClusterService clusterService = nodeEngine.getClusterService();
    if (!clusterService.isJoined()) {
        return newCompletedFuture(null);
    }
    RestartingMemberIterator memberIterator = new RestartingMemberIterator(clusterService, maxRetries);
    // we are going to iterate over all members and invoke an operation on each of them
    InvokeOnMemberFunction invokeOnMemberFunction = new InvokeOnMemberFunction(operationSupplier, nodeEngine, memberIterator);
    Iterator<InternalCompletableFuture<Object>> invocationIterator = map(memberIterator, invokeOnMemberFunction);
    // the future itself completes only when the last invocation completes (or if there is an error)
    return new ChainingFuture(invocationIterator, memberIterator);
}
Also used : ClusterService(com.hazelcast.internal.cluster.ClusterService) ChainingFuture(com.hazelcast.internal.util.futures.ChainingFuture) InternalCompletableFuture(com.hazelcast.spi.impl.InternalCompletableFuture) RestartingMemberIterator(com.hazelcast.internal.util.iterator.RestartingMemberIterator)

Example 5 with InternalCompletableFuture

use of com.hazelcast.spi.impl.InternalCompletableFuture in project hazelcast by hazelcast.

the class MapProxySupport method putAllInternal.

/**
 * This method will group all entries per partition and send one operation
 * per member. If there are e.g. five keys for a single member, even if
 * they are from different partitions, there will only be a single remote
 * invocation instead of five.
 * <p>
 * There is also an optional support for batching to send smaller packages.
 * Takes care about {@code null} checks for keys and values.
 *
 * @param future iff not-null, execute asynchronously by completing this future.
 *               Batching is not supported in async mode
 */
@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" })
protected void putAllInternal(Map<? extends K, ? extends V> map, @Nullable InternalCompletableFuture<Void> future, boolean triggerMapLoader) {
    try {
        int mapSize = map.size();
        if (mapSize == 0) {
            if (future != null) {
                future.complete(null);
            }
            return;
        }
        boolean useBatching = future == null && isPutAllUseBatching(mapSize);
        int partitionCount = partitionService.getPartitionCount();
        int initialSize = getPutAllInitialSize(useBatching, mapSize, partitionCount);
        Map<Address, List<Integer>> memberPartitionsMap = partitionService.getMemberPartitionsMap();
        // init counters for batching
        MutableLong[] counterPerMember = null;
        Address[] addresses = null;
        if (useBatching) {
            counterPerMember = new MutableLong[partitionCount];
            addresses = new Address[partitionCount];
            for (Entry<Address, List<Integer>> addressListEntry : memberPartitionsMap.entrySet()) {
                MutableLong counter = new MutableLong();
                Address address = addressListEntry.getKey();
                for (int partitionId : addressListEntry.getValue()) {
                    counterPerMember[partitionId] = counter;
                    addresses[partitionId] = address;
                }
            }
        }
        // fill entriesPerPartition
        MapEntries[] entriesPerPartition = new MapEntries[partitionCount];
        for (Entry entry : map.entrySet()) {
            checkNotNull(entry.getKey(), NULL_KEY_IS_NOT_ALLOWED);
            checkNotNull(entry.getValue(), NULL_VALUE_IS_NOT_ALLOWED);
            Data keyData = toDataWithStrategy(entry.getKey());
            int partitionId = partitionService.getPartitionId(keyData);
            MapEntries entries = entriesPerPartition[partitionId];
            if (entries == null) {
                entries = new MapEntries(initialSize);
                entriesPerPartition[partitionId] = entries;
            }
            entries.add(keyData, toData(entry.getValue()));
            if (useBatching) {
                long currentSize = ++counterPerMember[partitionId].value;
                if (currentSize % putAllBatchSize == 0) {
                    List<Integer> partitions = memberPartitionsMap.get(addresses[partitionId]);
                    invokePutAllOperation(addresses[partitionId], partitions, entriesPerPartition, triggerMapLoader).get();
                }
            }
        }
        // invoke operations for entriesPerPartition
        AtomicInteger counter = new AtomicInteger(memberPartitionsMap.size());
        InternalCompletableFuture<Void> resultFuture = future != null ? future : new InternalCompletableFuture<>();
        BiConsumer<Void, Throwable> callback = (response, t) -> {
            if (t != null) {
                resultFuture.completeExceptionally(t);
            }
            if (counter.decrementAndGet() == 0) {
                try {
                    // don't ignore errors here, see https://github.com/hazelcast/hazelcast-jet/issues/3046
                    finalizePutAll(map);
                } catch (Throwable e) {
                    resultFuture.completeExceptionally(e);
                    return;
                }
                if (!resultFuture.isDone()) {
                    resultFuture.complete(null);
                }
            }
        };
        for (Entry<Address, List<Integer>> entry : memberPartitionsMap.entrySet()) {
            invokePutAllOperation(entry.getKey(), entry.getValue(), entriesPerPartition, triggerMapLoader).whenCompleteAsync(callback);
        }
        // if executing in sync mode, block for the responses
        if (future == null) {
            resultFuture.get();
        }
    } catch (Throwable e) {
        throw rethrow(e);
    }
}
Also used : MutableLong(com.hazelcast.internal.util.MutableLong) Arrays(java.util.Arrays) PartitionPredicate(com.hazelcast.query.PartitionPredicate) ExceptionUtil.rethrow(com.hazelcast.internal.util.ExceptionUtil.rethrow) AbstractDistributedObject(com.hazelcast.spi.impl.AbstractDistributedObject) Collections.singletonList(java.util.Collections.singletonList) MapConfig(com.hazelcast.config.MapConfig) Future(java.util.concurrent.Future) MapEntries(com.hazelcast.map.impl.MapEntries) TruePredicate(com.hazelcast.query.impl.predicates.TruePredicate) SerializationService(com.hazelcast.internal.serialization.SerializationService) PartitionContainer(com.hazelcast.map.impl.PartitionContainer) Map(java.util.Map) LockSupportServiceImpl(com.hazelcast.internal.locksupport.LockSupportServiceImpl) PartitioningStrategy(com.hazelcast.partition.PartitioningStrategy) IPartitionService(com.hazelcast.internal.partition.IPartitionService) ExceptionUtil(com.hazelcast.internal.util.ExceptionUtil) Set(java.util.Set) EntryView(com.hazelcast.core.EntryView) Result(com.hazelcast.map.impl.query.Result) QueryCacheContext(com.hazelcast.map.impl.querycache.QueryCacheContext) Predicates.alwaysFalse(com.hazelcast.query.Predicates.alwaysFalse) AddIndexOperation(com.hazelcast.map.impl.operation.AddIndexOperation) Target(com.hazelcast.map.impl.query.Target) IterableUtil.nullToEmpty(com.hazelcast.internal.util.IterableUtil.nullToEmpty) IPartition(com.hazelcast.internal.partition.IPartition) EventListener(java.util.EventListener) AwaitMapFlushOperation(com.hazelcast.map.impl.operation.AwaitMapFlushOperation) MapOperationProvider(com.hazelcast.map.impl.operation.MapOperationProvider) MapStoreConfig(com.hazelcast.config.MapStoreConfig) LockProxySupport(com.hazelcast.internal.locksupport.LockProxySupport) IterationType(com.hazelcast.internal.util.IterationType) PartitionIdSet(com.hazelcast.internal.util.collection.PartitionIdSet) QueryEngine(com.hazelcast.map.impl.query.QueryEngine) HazelcastInstanceAware(com.hazelcast.core.HazelcastInstanceAware) Query(com.hazelcast.map.impl.query.Query) EntryEventFilter(com.hazelcast.map.impl.EntryEventFilter) Math.log10(java.lang.Math.log10) InMemoryFormat(com.hazelcast.config.InMemoryFormat) Supplier(java.util.function.Supplier) CLEAR_ALL(com.hazelcast.core.EntryEventType.CLEAR_ALL) RemoveInterceptorOperationSupplier(com.hazelcast.map.impl.operation.RemoveInterceptorOperationSupplier) ArrayList(java.util.ArrayList) QueryEventFilter(com.hazelcast.map.impl.query.QueryEventFilter) EntryListenerConfig(com.hazelcast.config.EntryListenerConfig) BiConsumer(java.util.function.BiConsumer) BinaryOperationFactory(com.hazelcast.spi.impl.operationservice.BinaryOperationFactory) Nullable(javax.annotation.Nullable) LocalMapStats(com.hazelcast.map.LocalMapStats) InvocationUtil.invokeOnStableClusterSerial(com.hazelcast.internal.util.InvocationUtil.invokeOnStableClusterSerial) ThreadUtil.getThreadId(com.hazelcast.internal.util.ThreadUtil.getThreadId) InternalCompletableFuture.newCompletedFuture(com.hazelcast.spi.impl.InternalCompletableFuture.newCompletedFuture) IsPartitionLoadedOperationFactory(com.hazelcast.map.impl.operation.IsPartitionLoadedOperationFactory) ClassLoaderUtil(com.hazelcast.internal.nio.ClassLoaderUtil) TargetMode(com.hazelcast.map.impl.query.Target.TargetMode) QueryCacheEndToEndProvider(com.hazelcast.map.impl.querycache.subscriber.QueryCacheEndToEndProvider) CollectionUtil.asIntegerList(com.hazelcast.internal.util.CollectionUtil.asIntegerList) EventFilter(com.hazelcast.spi.impl.eventservice.EventFilter) Preconditions.checkNotNull(com.hazelcast.internal.util.Preconditions.checkNotNull) Target.createPartitionTarget(com.hazelcast.map.impl.query.Target.createPartitionTarget) LocalMapStatsImpl(com.hazelcast.internal.monitor.impl.LocalMapStatsImpl) MapEventPublisher(com.hazelcast.map.impl.event.MapEventPublisher) MapPartitionLostListenerConfig(com.hazelcast.config.MapPartitionLostListenerConfig) EntryProcessor(com.hazelcast.map.EntryProcessor) ReadOnly(com.hazelcast.core.ReadOnly) AddInterceptorOperationSupplier(com.hazelcast.map.impl.operation.AddInterceptorOperationSupplier) OperationFactory(com.hazelcast.spi.impl.operationservice.OperationFactory) OperationService(com.hazelcast.spi.impl.operationservice.OperationService) Address(com.hazelcast.cluster.Address) InvocationFuture(com.hazelcast.spi.impl.operationservice.impl.InvocationFuture) InternalCompletableFuture(com.hazelcast.spi.impl.InternalCompletableFuture) IsEmptyOperationFactory(com.hazelcast.map.impl.operation.IsEmptyOperationFactory) MapListener(com.hazelcast.map.listener.MapListener) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) InitializingObject(com.hazelcast.spi.impl.InitializingObject) IndexUtils(com.hazelcast.query.impl.IndexUtils) Predicate(com.hazelcast.query.Predicate) CALLER_RUNS(com.hazelcast.internal.util.ConcurrencyUtil.CALLER_RUNS) Collection(java.util.Collection) IsKeyLoadFinishedOperation(com.hazelcast.map.impl.operation.IsKeyLoadFinishedOperation) UUID(java.util.UUID) Math.ceil(java.lang.Math.ceil) Math.min(java.lang.Math.min) SubscriberContext(com.hazelcast.map.impl.querycache.subscriber.SubscriberContext) IndexConfig(com.hazelcast.config.IndexConfig) List(java.util.List) Preconditions.checkFalse(com.hazelcast.internal.util.Preconditions.checkFalse) SetUtil.createHashSet(com.hazelcast.internal.util.SetUtil.createHashSet) TimeUtil.timeInMsOrOneIfResultIsZero(com.hazelcast.internal.util.TimeUtil.timeInMsOrOneIfResultIsZero) Projection(com.hazelcast.projection.Projection) IterableUtil(com.hazelcast.internal.util.IterableUtil) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) MapUtil.createHashMap(com.hazelcast.internal.util.MapUtil.createHashMap) Aggregator(com.hazelcast.aggregation.Aggregator) ListenerConfig(com.hazelcast.config.ListenerConfig) IndexType(com.hazelcast.config.IndexType) Operation(com.hazelcast.spi.impl.operationservice.Operation) HazelcastProperty(com.hazelcast.spi.properties.HazelcastProperty) Collections.singletonMap(java.util.Collections.singletonMap) MapUtil.toIntSize(com.hazelcast.internal.util.MapUtil.toIntSize) Nonnull(javax.annotation.Nonnull) Timer(com.hazelcast.internal.util.Timer) NodeEngine(com.hazelcast.spi.impl.NodeEngine) Iterator(java.util.Iterator) EntryEventType(com.hazelcast.core.EntryEventType) MapOperation(com.hazelcast.map.impl.operation.MapOperation) MapInterceptor(com.hazelcast.map.MapInterceptor) HazelcastProperties(com.hazelcast.spi.properties.HazelcastProperties) Data(com.hazelcast.internal.serialization.Data) MapService(com.hazelcast.map.impl.MapService) MapServiceContext(com.hazelcast.map.impl.MapServiceContext) RecordStore(com.hazelcast.map.impl.recordstore.RecordStore) TimeUnit(java.util.concurrent.TimeUnit) SERVICE_NAME(com.hazelcast.map.impl.MapService.SERVICE_NAME) MapPartitionLostListener(com.hazelcast.map.listener.MapPartitionLostListener) ENTRY_REMOVING_PROCESSOR(com.hazelcast.map.impl.EntryRemovingProcessor.ENTRY_REMOVING_PROCESSOR) Collections(java.util.Collections) IMap(com.hazelcast.map.IMap) Address(com.hazelcast.cluster.Address) Data(com.hazelcast.internal.serialization.Data) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MutableLong(com.hazelcast.internal.util.MutableLong) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MapEntries(com.hazelcast.map.impl.MapEntries) Collections.singletonList(java.util.Collections.singletonList) ArrayList(java.util.ArrayList) CollectionUtil.asIntegerList(com.hazelcast.internal.util.CollectionUtil.asIntegerList) List(java.util.List)

Aggregations

InternalCompletableFuture (com.hazelcast.spi.impl.InternalCompletableFuture)90 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)47 QuickTest (com.hazelcast.test.annotation.QuickTest)47 Test (org.junit.Test)47 OperationService (com.hazelcast.spi.impl.operationservice.OperationService)19 HazelcastInstance (com.hazelcast.core.HazelcastInstance)17 Accessors.getOperationService (com.hazelcast.test.Accessors.getOperationService)15 Data (com.hazelcast.internal.serialization.Data)10 ArrayList (java.util.ArrayList)10 Map (java.util.Map)10 Operation (com.hazelcast.spi.impl.operationservice.Operation)9 UUID (java.util.UUID)9 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)8 Member (com.hazelcast.cluster.Member)7 ApplyRaftRunnable (com.hazelcast.cp.internal.raft.impl.dataservice.ApplyRaftRunnable)7 Future (java.util.concurrent.Future)7 Address (com.hazelcast.cluster.Address)6 List (java.util.List)6 BiConsumer (java.util.function.BiConsumer)6 Nonnull (javax.annotation.Nonnull)6