use of org.apache.geode.internal.cache.execute.PartitionedRegionFunctionResultSender in project geode by apache.
the class PartitionedRegion method executeOnMultipleNodes.
/**
* Executes function on multiple nodes
*/
private ResultCollector executeOnMultipleNodes(final Function function, final PartitionedRegionFunctionExecutor execution, ResultCollector rc, boolean isPRSingleHop, boolean isBucketSetAsFilter) {
final Set routingKeys = execution.getFilter();
final boolean primaryMembersNeeded = function.optimizeForWrite();
HashMap<Integer, HashSet> bucketToKeysMap = FunctionExecutionNodePruner.groupByBucket(this, routingKeys, primaryMembersNeeded, false, isBucketSetAsFilter);
HashMap<InternalDistributedMember, HashSet> memberToKeysMap = new HashMap<InternalDistributedMember, HashSet>();
HashMap<InternalDistributedMember, HashSet<Integer>> memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketToKeysMap.keySet(), primaryMembersNeeded);
if (isPRSingleHop && (memberToBuckets.size() > 1)) {
// memberToBuckets.remove(getMyId()); // don't remove
for (InternalDistributedMember targetNode : memberToBuckets.keySet()) {
if (!targetNode.equals(getMyId())) {
for (Integer bucketId : memberToBuckets.get(targetNode)) {
Set<ServerBucketProfile> profiles = this.getRegionAdvisor().getClientBucketProfiles(bucketId);
if (profiles != null) {
for (ServerBucketProfile profile : profiles) {
if (profile.getDistributedMember().equals(targetNode)) {
if (logger.isDebugEnabled()) {
logger.debug("FunctionServiceSingleHop: Found multiple nodes.{}", getMyId());
}
throw new InternalFunctionInvocationTargetException(LocalizedStrings.PartitionedRegion_MULTIPLE_TARGET_NODE_FOUND_FOR.toLocalizedString());
}
}
}
}
}
}
}
while (!execution.getFailedNodes().isEmpty()) {
Set memberKeySet = memberToBuckets.keySet();
RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
Iterator iterator = memberKeySet.iterator();
boolean hasRemovedNode = false;
while (iterator.hasNext()) {
if (execution.getFailedNodes().contains(((DistributedMember) iterator.next()).getId())) {
hasRemovedNode = true;
}
}
if (hasRemovedNode) {
if (retryTime.overMaximum()) {
PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
// NOTREACHED
}
retryTime.waitToRetryNode();
memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketToKeysMap.keySet(), primaryMembersNeeded);
} else {
execution.clearFailedNodes();
}
}
for (Map.Entry entry : memberToBuckets.entrySet()) {
InternalDistributedMember member = (InternalDistributedMember) entry.getKey();
HashSet<Integer> buckets = (HashSet) entry.getValue();
for (Integer bucket : buckets) {
HashSet keys = memberToKeysMap.get(member);
if (keys == null) {
keys = new HashSet();
}
keys.addAll(bucketToKeysMap.get(bucket));
memberToKeysMap.put(member, keys);
}
}
// memberToKeysMap.keySet().retainAll(memberToBuckets.keySet());
if (memberToKeysMap.isEmpty()) {
throw new FunctionException(LocalizedStrings.PartitionedRegion_NO_TARGET_NODE_FOUND_FOR_KEY_0.toLocalizedString(routingKeys));
}
Set<InternalDistributedMember> dest = memberToKeysMap.keySet();
execution.validateExecution(function, dest);
// added for the data aware procedure.
execution.setExecutionNodes(dest);
// end
final HashSet localKeys = memberToKeysMap.remove(getMyId());
HashSet<Integer> localBucketSet = null;
boolean remoteOnly = false;
if (localKeys == null) {
remoteOnly = true;
} else {
localBucketSet = FunctionExecutionNodePruner.getBucketSet(PartitionedRegion.this, localKeys, false, isBucketSetAsFilter);
remoteOnly = false;
}
final LocalResultCollector<?, ?> localResultCollector = execution.getLocalResultCollector(function, rc);
final DM dm = getDistributionManager();
final PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, this, 0L, localResultCollector, execution.getServerResultSender(), memberToKeysMap.isEmpty(), remoteOnly, execution.isForwardExceptions(), function, localBucketSet);
if (localKeys != null) {
final RegionFunctionContextImpl prContext = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(getMyId().getId()), localKeys, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, localBucketSet), localBucketSet, resultSender, execution.isReExecute());
if (logger.isDebugEnabled()) {
logger.debug("FunctionService: Executing on local node with keys.{}", localKeys);
}
execution.executeFunctionOnLocalPRNode(function, prContext, resultSender, dm, isTX());
}
if (!memberToKeysMap.isEmpty()) {
HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
for (Map.Entry me : memberToKeysMap.entrySet()) {
InternalDistributedMember recip = (InternalDistributedMember) me.getKey();
HashSet memKeys = (HashSet) me.getValue();
FunctionRemoteContext context = new FunctionRemoteContext(function, execution.getArgumentsForMember(recip.getId()), memKeys, FunctionExecutionNodePruner.getBucketSet(this, memKeys, false, isBucketSetAsFilter), execution.isReExecute(), execution.isFnSerializationReqd());
recipMap.put(recip, context);
}
if (logger.isDebugEnabled()) {
logger.debug("FunctionService: Executing on remote nodes with member to keys map.{}", memberToKeysMap);
}
PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), localResultCollector, function, resultSender);
return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
}
return localResultCollector;
}
use of org.apache.geode.internal.cache.execute.PartitionedRegionFunctionResultSender in project geode by apache.
the class PartitionedRegion method executeOnBucketSet.
public ResultCollector executeOnBucketSet(final Function function, PartitionedRegionFunctionExecutor execution, ResultCollector rc, Set<Integer> bucketSet) {
Set<Integer> actualBucketSet = this.getRegionAdvisor().getBucketSet();
try {
bucketSet.retainAll(actualBucketSet);
} catch (NoSuchElementException ignore) {
// done
}
HashMap<InternalDistributedMember, HashSet<Integer>> memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
if (memberToBuckets.isEmpty()) {
if (logger.isDebugEnabled()) {
logger.debug("Executing on bucketset : {} executeOnBucketSet Member to buckets map is : {} bucketSet is empty", bucketSet, memberToBuckets);
}
throw new EmptyRegionFunctionException(LocalizedStrings.PartitionedRegion_FUNCTION_NOT_EXECUTED_AS_REGION_IS_EMPTY.toLocalizedString());
} else {
if (logger.isDebugEnabled()) {
logger.debug("Executing on bucketset : {} executeOnBucketSet Member to buckets map is : {}", bucketSet, memberToBuckets);
}
}
if (memberToBuckets.size() > 1) {
for (InternalDistributedMember targetNode : memberToBuckets.keySet()) {
if (!targetNode.equals(getMyId())) {
for (Integer bucketId : memberToBuckets.get(targetNode)) {
Set<ServerBucketProfile> profiles = this.getRegionAdvisor().getClientBucketProfiles(bucketId);
if (profiles != null) {
for (ServerBucketProfile profile : profiles) {
if (profile.getDistributedMember().equals(targetNode)) {
if (logger.isDebugEnabled()) {
logger.debug("FunctionServiceSingleHop: Found multiple nodes for executing on bucket set.{}", getMyId());
}
throw new InternalFunctionInvocationTargetException(LocalizedStrings.PartitionedRegion_MULTIPLE_TARGET_NODE_FOUND_FOR.toLocalizedString());
}
}
}
}
}
}
}
execution = (PartitionedRegionFunctionExecutor) execution.withFilter(new HashSet());
while (!execution.getFailedNodes().isEmpty()) {
Set memberKeySet = memberToBuckets.keySet();
RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
Iterator iterator = memberKeySet.iterator();
boolean hasRemovedNode = false;
while (iterator.hasNext()) {
if (execution.getFailedNodes().contains(((DistributedMember) iterator.next()).getId())) {
hasRemovedNode = true;
}
}
if (hasRemovedNode) {
if (retryTime.overMaximum()) {
PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
// NOTREACHED
}
retryTime.waitToRetryNode();
memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
} else {
execution.clearFailedNodes();
}
}
Set<InternalDistributedMember> dest = memberToBuckets.keySet();
if (function.optimizeForWrite() && cache.getInternalResourceManager().getHeapMonitor().containsHeapCriticalMembers(dest) && !MemoryThresholds.isLowMemoryExceptionDisabled()) {
Set<InternalDistributedMember> hcm = cache.getResourceAdvisor().adviseCritialMembers();
Set<DistributedMember> sm = SetUtils.intersection(hcm, dest);
throw new LowMemoryException(LocalizedStrings.ResourceManager_LOW_MEMORY_FOR_0_FUNCEXEC_MEMBERS_1.toLocalizedString(function.getId(), sm), sm);
}
boolean isSelf = false;
execution.setExecutionNodes(dest);
final Set localBucketSet = memberToBuckets.remove(getMyId());
if (localBucketSet != null) {
isSelf = true;
}
final HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
for (InternalDistributedMember recip : dest) {
FunctionRemoteContext context = new FunctionRemoteContext(function, execution.getArgumentsForMember(recip.getId()), null, memberToBuckets.get(recip), execution.isReExecute(), execution.isFnSerializationReqd());
recipMap.put(recip, context);
}
final LocalResultCollector<?, ?> localRC = execution.getLocalResultCollector(function, rc);
final DM dm = getDistributionManager();
final PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, this, 0L, localRC, execution.getServerResultSender(), recipMap.isEmpty(), !isSelf, execution.isForwardExceptions(), function, localBucketSet);
// execute locally and collect the result
if (isSelf && this.dataStore != null) {
final RegionFunctionContextImpl prContext = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(getMyId().getId()), null, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, localBucketSet), localBucketSet, resultSender, execution.isReExecute());
execution.executeFunctionOnLocalNode(function, prContext, resultSender, dm, isTX());
}
PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), localRC, function, resultSender);
return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
}
use of org.apache.geode.internal.cache.execute.PartitionedRegionFunctionResultSender in project geode by apache.
the class PartitionedRegion method executeFunctionOnRemoteNode.
private ResultCollector executeFunctionOnRemoteNode(InternalDistributedMember targetNode, final Function function, final Object object, final Set routingKeys, ResultCollector rc, Set bucketSet, ServerToClientFunctionResultSender sender, AbstractExecution execution) {
PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(null, this, 0, rc, sender, false, true, execution.isForwardExceptions(), function, bucketSet);
PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), rc, function, resultSender);
FunctionRemoteContext context = new FunctionRemoteContext(function, object, routingKeys, bucketSet, execution.isReExecute(), execution.isFnSerializationReqd());
HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
recipMap.put(targetNode, context);
return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
}
use of org.apache.geode.internal.cache.execute.PartitionedRegionFunctionResultSender in project geode by apache.
the class PartitionedRegion method executeOnSingleNode.
/**
* Single key execution on single node
*
* @since GemFire 6.0
*/
private ResultCollector executeOnSingleNode(final Function function, final PartitionedRegionFunctionExecutor execution, ResultCollector rc, boolean isPRSingleHop, boolean isBucketSetAsFilter) {
final Set routingKeys = execution.getFilter();
final Object key = routingKeys.iterator().next();
final Integer bucketId;
if (isBucketSetAsFilter) {
bucketId = (Integer) key;
} else {
bucketId = PartitionedRegionHelper.getHashKey(this, Operation.FUNCTION_EXECUTION, key, null, null);
}
InternalDistributedMember targetNode = null;
if (function.optimizeForWrite()) {
targetNode = createBucket(bucketId, 0, null);
HeapMemoryMonitor hmm = ((InternalResourceManager) cache.getResourceManager()).getHeapMonitor();
if (hmm.isMemberHeapCritical(targetNode) && !MemoryThresholds.isLowMemoryExceptionDisabled()) {
Set<DistributedMember> sm = Collections.singleton((DistributedMember) targetNode);
throw new LowMemoryException(LocalizedStrings.ResourceManager_LOW_MEMORY_FOR_0_FUNCEXEC_MEMBERS_1.toLocalizedString(function.getId(), sm), sm);
}
} else {
targetNode = getOrCreateNodeForBucketRead(bucketId);
}
final DistributedMember localVm = getMyId();
if (targetNode != null && isPRSingleHop && !localVm.equals(targetNode)) {
Set<ServerBucketProfile> profiles = this.getRegionAdvisor().getClientBucketProfiles(bucketId);
if (profiles != null) {
for (ServerBucketProfile profile : profiles) {
if (profile.getDistributedMember().equals(targetNode)) {
if (logger.isDebugEnabled()) {
logger.debug("FunctionServiceSingleHop: Found remote node.{}", localVm);
}
throw new InternalFunctionInvocationTargetException(LocalizedStrings.PartitionedRegion_MULTIPLE_TARGET_NODE_FOUND_FOR.toLocalizedString());
}
}
}
}
if (targetNode == null) {
throw new FunctionException(LocalizedStrings.PartitionedRegion_NO_TARGET_NODE_FOUND_FOR_KEY_0.toLocalizedString(key));
}
if (logger.isDebugEnabled()) {
logger.debug("Executing Function: {} setArguments={} on {}", function.getId(), execution.getArguments(), targetNode);
}
while (!execution.getFailedNodes().isEmpty()) {
RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
if (execution.getFailedNodes().contains(targetNode.getId())) {
/*
* if (retryTime.overMaximum()) { PRHARedundancyProvider.timedOut(this, null, null,
* "doing function execution", this.retryTimeout); // NOTREACHED }
*/
// Fix for Bug # 40083
targetNode = null;
while (targetNode == null) {
if (retryTime.overMaximum()) {
PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
// NOTREACHED
}
retryTime.waitToRetryNode();
if (function.optimizeForWrite()) {
targetNode = getOrCreateNodeForBucketWrite(bucketId, retryTime);
} else {
targetNode = getOrCreateNodeForBucketRead(bucketId);
}
}
if (targetNode == null) {
throw new FunctionException(LocalizedStrings.PartitionedRegion_NO_TARGET_NODE_FOUND_FOR_KEY_0.toLocalizedString(key));
}
} else {
execution.clearFailedNodes();
}
}
final HashSet<Integer> buckets = new HashSet<Integer>();
buckets.add(bucketId);
final Set<InternalDistributedMember> singleMember = Collections.singleton(targetNode);
execution.validateExecution(function, singleMember);
execution.setExecutionNodes(singleMember);
LocalResultCollector<?, ?> localRC = execution.getLocalResultCollector(function, rc);
if (targetNode.equals(localVm)) {
final DM dm = getDistributionManager();
PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, PartitionedRegion.this, 0, localRC, execution.getServerResultSender(), true, false, execution.isForwardExceptions(), function, buckets);
final FunctionContext context = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(localVm.getId()), routingKeys, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, buckets), buckets, resultSender, execution.isReExecute());
execution.executeFunctionOnLocalPRNode(function, context, resultSender, dm, isTX());
return localRC;
} else {
return executeFunctionOnRemoteNode(targetNode, function, execution.getArgumentsForMember(targetNode.getId()), routingKeys, function.isHA() ? rc : localRC, buckets, execution.getServerResultSender(), execution);
}
}
use of org.apache.geode.internal.cache.execute.PartitionedRegionFunctionResultSender in project geode by apache.
the class PartitionedRegion method executeOnAllBuckets.
/**
* Executes function on all bucket nodes
*
* @return ResultCollector
* @since GemFire 6.0
*/
private ResultCollector executeOnAllBuckets(final Function function, final PartitionedRegionFunctionExecutor execution, ResultCollector rc, boolean isPRSingleHop) {
Set<Integer> bucketSet = new HashSet<Integer>();
Iterator<Integer> itr = this.getRegionAdvisor().getBucketSet().iterator();
while (itr.hasNext()) {
try {
bucketSet.add(itr.next());
} catch (NoSuchElementException ignore) {
}
}
HashMap<InternalDistributedMember, HashSet<Integer>> memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
if (memberToBuckets.isEmpty()) {
throw new EmptyRegionFunctionException(LocalizedStrings.PartitionedRegion_FUNCTION_NOT_EXECUTED_AS_REGION_IS_EMPTY.toLocalizedString());
}
while (!execution.getFailedNodes().isEmpty()) {
Set memberKeySet = memberToBuckets.keySet();
RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
Iterator iterator = memberKeySet.iterator();
boolean hasRemovedNode = false;
while (iterator.hasNext()) {
if (execution.getFailedNodes().contains(((DistributedMember) iterator.next()).getId())) {
hasRemovedNode = true;
}
}
if (hasRemovedNode) {
if (retryTime.overMaximum()) {
PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
// NOTREACHED
}
retryTime.waitToRetryNode();
memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
} else {
execution.clearFailedNodes();
}
}
Set<InternalDistributedMember> dest = memberToBuckets.keySet();
execution.validateExecution(function, dest);
execution.setExecutionNodes(dest);
boolean isSelf = false;
final Set<Integer> localBucketSet = memberToBuckets.remove(getMyId());
if (localBucketSet != null) {
isSelf = true;
}
final HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
for (InternalDistributedMember recip : memberToBuckets.keySet()) {
FunctionRemoteContext context = new FunctionRemoteContext(function, execution.getArgumentsForMember(recip.getId()), null, memberToBuckets.get(recip), execution.isReExecute(), execution.isFnSerializationReqd());
recipMap.put(recip, context);
}
final LocalResultCollector<?, ?> localResultCollector = execution.getLocalResultCollector(function, rc);
final DM dm = getDistributionManager();
final PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, this, 0L, localResultCollector, execution.getServerResultSender(), recipMap.isEmpty(), !isSelf, execution.isForwardExceptions(), function, localBucketSet);
// execute locally and collect the result
if (isSelf && this.dataStore != null) {
final RegionFunctionContextImpl prContext = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(getMyId().getId()), null, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, localBucketSet), localBucketSet, resultSender, execution.isReExecute());
execution.executeFunctionOnLocalPRNode(function, prContext, resultSender, dm, isTX());
}
PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), localResultCollector, function, resultSender);
return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
}
Aggregations