Examples with ServerBucketProfile org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile used on opensource projects

Example 1 with ServerBucketProfile

use of org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile in project geode by apache.

the class PartitionedRegion method executeOnMultipleNodes.

/**
   * Executes function on multiple nodes
   */
private ResultCollector executeOnMultipleNodes(final Function function, final PartitionedRegionFunctionExecutor execution, ResultCollector rc, boolean isPRSingleHop, boolean isBucketSetAsFilter) {
    final Set routingKeys = execution.getFilter();
    final boolean primaryMembersNeeded = function.optimizeForWrite();
    HashMap<Integer, HashSet> bucketToKeysMap = FunctionExecutionNodePruner.groupByBucket(this, routingKeys, primaryMembersNeeded, false, isBucketSetAsFilter);
    HashMap<InternalDistributedMember, HashSet> memberToKeysMap = new HashMap<InternalDistributedMember, HashSet>();
    HashMap<InternalDistributedMember, HashSet<Integer>> memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketToKeysMap.keySet(), primaryMembersNeeded);
    if (isPRSingleHop && (memberToBuckets.size() > 1)) {
        // memberToBuckets.remove(getMyId()); // don't remove
        for (InternalDistributedMember targetNode : memberToBuckets.keySet()) {
            if (!targetNode.equals(getMyId())) {
                for (Integer bucketId : memberToBuckets.get(targetNode)) {
                    Set<ServerBucketProfile> profiles = this.getRegionAdvisor().getClientBucketProfiles(bucketId);
                    if (profiles != null) {
                        for (ServerBucketProfile profile : profiles) {
                            if (profile.getDistributedMember().equals(targetNode)) {
                                if (logger.isDebugEnabled()) {
                                    logger.debug("FunctionServiceSingleHop: Found multiple nodes.{}", getMyId());
                                }
                                throw new InternalFunctionInvocationTargetException(LocalizedStrings.PartitionedRegion_MULTIPLE_TARGET_NODE_FOUND_FOR.toLocalizedString());
                            }
                        }
                    }
                }
            }
        }
    }
    while (!execution.getFailedNodes().isEmpty()) {
        Set memberKeySet = memberToBuckets.keySet();
        RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
        Iterator iterator = memberKeySet.iterator();
        boolean hasRemovedNode = false;
        while (iterator.hasNext()) {
            if (execution.getFailedNodes().contains(((DistributedMember) iterator.next()).getId())) {
                hasRemovedNode = true;
            }
        }
        if (hasRemovedNode) {
            if (retryTime.overMaximum()) {
                PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
            // NOTREACHED
            }
            retryTime.waitToRetryNode();
            memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketToKeysMap.keySet(), primaryMembersNeeded);
        } else {
            execution.clearFailedNodes();
        }
    }
    for (Map.Entry entry : memberToBuckets.entrySet()) {
        InternalDistributedMember member = (InternalDistributedMember) entry.getKey();
        HashSet<Integer> buckets = (HashSet) entry.getValue();
        for (Integer bucket : buckets) {
            HashSet keys = memberToKeysMap.get(member);
            if (keys == null) {
                keys = new HashSet();
            }
            keys.addAll(bucketToKeysMap.get(bucket));
            memberToKeysMap.put(member, keys);
        }
    }
    // memberToKeysMap.keySet().retainAll(memberToBuckets.keySet());
    if (memberToKeysMap.isEmpty()) {
        throw new FunctionException(LocalizedStrings.PartitionedRegion_NO_TARGET_NODE_FOUND_FOR_KEY_0.toLocalizedString(routingKeys));
    }
    Set<InternalDistributedMember> dest = memberToKeysMap.keySet();
    execution.validateExecution(function, dest);
    // added for the data aware procedure.
    execution.setExecutionNodes(dest);
    // end
    final HashSet localKeys = memberToKeysMap.remove(getMyId());
    HashSet<Integer> localBucketSet = null;
    boolean remoteOnly = false;
    if (localKeys == null) {
        remoteOnly = true;
    } else {
        localBucketSet = FunctionExecutionNodePruner.getBucketSet(PartitionedRegion.this, localKeys, false, isBucketSetAsFilter);
        remoteOnly = false;
    }
    final LocalResultCollector<?, ?> localResultCollector = execution.getLocalResultCollector(function, rc);
    final DM dm = getDistributionManager();
    final PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, this, 0L, localResultCollector, execution.getServerResultSender(), memberToKeysMap.isEmpty(), remoteOnly, execution.isForwardExceptions(), function, localBucketSet);
    if (localKeys != null) {
        final RegionFunctionContextImpl prContext = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(getMyId().getId()), localKeys, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, localBucketSet), localBucketSet, resultSender, execution.isReExecute());
        if (logger.isDebugEnabled()) {
            logger.debug("FunctionService: Executing on local node with keys.{}", localKeys);
        }
        execution.executeFunctionOnLocalPRNode(function, prContext, resultSender, dm, isTX());
    }
    if (!memberToKeysMap.isEmpty()) {
        HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
        for (Map.Entry me : memberToKeysMap.entrySet()) {
            InternalDistributedMember recip = (InternalDistributedMember) me.getKey();
            HashSet memKeys = (HashSet) me.getValue();
            FunctionRemoteContext context = new FunctionRemoteContext(function, execution.getArgumentsForMember(recip.getId()), memKeys, FunctionExecutionNodePruner.getBucketSet(this, memKeys, false, isBucketSetAsFilter), execution.isReExecute(), execution.isFnSerializationReqd());
            recipMap.put(recip, context);
        }
        if (logger.isDebugEnabled()) {
            logger.debug("FunctionService: Executing on remote nodes with member to keys map.{}", memberToKeysMap);
        }
        PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), localResultCollector, function, resultSender);
        return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
    }
    return localResultCollector;
}

Example 2 with ServerBucketProfile

use of org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile in project geode by apache.

the class PartitionedRegion method executeOnBucketSet.

public ResultCollector executeOnBucketSet(final Function function, PartitionedRegionFunctionExecutor execution, ResultCollector rc, Set<Integer> bucketSet) {
    Set<Integer> actualBucketSet = this.getRegionAdvisor().getBucketSet();
    try {
        bucketSet.retainAll(actualBucketSet);
    } catch (NoSuchElementException ignore) {
    // done
    }
    HashMap<InternalDistributedMember, HashSet<Integer>> memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
    if (memberToBuckets.isEmpty()) {
        if (logger.isDebugEnabled()) {
            logger.debug("Executing on bucketset : {} executeOnBucketSet Member to buckets map is : {} bucketSet is empty", bucketSet, memberToBuckets);
        }
        throw new EmptyRegionFunctionException(LocalizedStrings.PartitionedRegion_FUNCTION_NOT_EXECUTED_AS_REGION_IS_EMPTY.toLocalizedString());
    } else {
        if (logger.isDebugEnabled()) {
            logger.debug("Executing on bucketset : {} executeOnBucketSet Member to buckets map is : {}", bucketSet, memberToBuckets);
        }
    }
    if (memberToBuckets.size() > 1) {
        for (InternalDistributedMember targetNode : memberToBuckets.keySet()) {
            if (!targetNode.equals(getMyId())) {
                for (Integer bucketId : memberToBuckets.get(targetNode)) {
                    Set<ServerBucketProfile> profiles = this.getRegionAdvisor().getClientBucketProfiles(bucketId);
                    if (profiles != null) {
                        for (ServerBucketProfile profile : profiles) {
                            if (profile.getDistributedMember().equals(targetNode)) {
                                if (logger.isDebugEnabled()) {
                                    logger.debug("FunctionServiceSingleHop: Found multiple nodes for executing on bucket set.{}", getMyId());
                                }
                                throw new InternalFunctionInvocationTargetException(LocalizedStrings.PartitionedRegion_MULTIPLE_TARGET_NODE_FOUND_FOR.toLocalizedString());
                            }
                        }
                    }
                }
            }
        }
    }
    execution = (PartitionedRegionFunctionExecutor) execution.withFilter(new HashSet());
    while (!execution.getFailedNodes().isEmpty()) {
        Set memberKeySet = memberToBuckets.keySet();
        RetryTimeKeeper retryTime = new RetryTimeKeeper(this.retryTimeout);
        Iterator iterator = memberKeySet.iterator();
        boolean hasRemovedNode = false;
        while (iterator.hasNext()) {
            if (execution.getFailedNodes().contains(((DistributedMember) iterator.next()).getId())) {
                hasRemovedNode = true;
            }
        }
        if (hasRemovedNode) {
            if (retryTime.overMaximum()) {
                PRHARedundancyProvider.timedOut(this, null, null, "doing function execution", this.retryTimeout);
            // NOTREACHED
            }
            retryTime.waitToRetryNode();
            memberToBuckets = FunctionExecutionNodePruner.groupByMemberToBuckets(this, bucketSet, function.optimizeForWrite());
        } else {
            execution.clearFailedNodes();
        }
    }
    Set<InternalDistributedMember> dest = memberToBuckets.keySet();
    if (function.optimizeForWrite() && cache.getInternalResourceManager().getHeapMonitor().containsHeapCriticalMembers(dest) && !MemoryThresholds.isLowMemoryExceptionDisabled()) {
        Set<InternalDistributedMember> hcm = cache.getResourceAdvisor().adviseCritialMembers();
        Set<DistributedMember> sm = SetUtils.intersection(hcm, dest);
        throw new LowMemoryException(LocalizedStrings.ResourceManager_LOW_MEMORY_FOR_0_FUNCEXEC_MEMBERS_1.toLocalizedString(function.getId(), sm), sm);
    }
    boolean isSelf = false;
    execution.setExecutionNodes(dest);
    final Set localBucketSet = memberToBuckets.remove(getMyId());
    if (localBucketSet != null) {
        isSelf = true;
    }
    final HashMap<InternalDistributedMember, FunctionRemoteContext> recipMap = new HashMap<InternalDistributedMember, FunctionRemoteContext>();
    for (InternalDistributedMember recip : dest) {
        FunctionRemoteContext context = new FunctionRemoteContext(function, execution.getArgumentsForMember(recip.getId()), null, memberToBuckets.get(recip), execution.isReExecute(), execution.isFnSerializationReqd());
        recipMap.put(recip, context);
    }
    final LocalResultCollector<?, ?> localRC = execution.getLocalResultCollector(function, rc);
    final DM dm = getDistributionManager();
    final PartitionedRegionFunctionResultSender resultSender = new PartitionedRegionFunctionResultSender(dm, this, 0L, localRC, execution.getServerResultSender(), recipMap.isEmpty(), !isSelf, execution.isForwardExceptions(), function, localBucketSet);
    // execute locally and collect the result
    if (isSelf && this.dataStore != null) {
        final RegionFunctionContextImpl prContext = new RegionFunctionContextImpl(function.getId(), PartitionedRegion.this, execution.getArgumentsForMember(getMyId().getId()), null, ColocationHelper.constructAndGetAllColocatedLocalDataSet(PartitionedRegion.this, localBucketSet), localBucketSet, resultSender, execution.isReExecute());
        execution.executeFunctionOnLocalNode(function, prContext, resultSender, dm, isTX());
    }
    PartitionedRegionFunctionResultWaiter resultReciever = new PartitionedRegionFunctionResultWaiter(getSystem(), this.getPRId(), localRC, function, resultSender);
    return resultReciever.getPartitionedDataFrom(recipMap, this, execution);
}

Example 3 with ServerBucketProfile

use of org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile in project geode by apache.

the class RegionAdvisor method getAllClientBucketProfiles.

/**
   * Returns an unmodifiable map of bucket IDs to locations hosting the bucket.
   */
public Map<Integer, List<BucketServerLocation66>> getAllClientBucketProfiles() {
    Map<Integer, List<BucketServerLocation66>> bucketToServerLocations = new HashMap<Integer, List<BucketServerLocation66>>();
    for (Integer bucketId : this.clientBucketProfilesMap.keySet()) {
        ArrayList<BucketServerLocation66> clientBucketProfiles = new ArrayList<BucketServerLocation66>();
        for (BucketProfile profile : this.clientBucketProfilesMap.get(bucketId)) {
            if (profile.isHosting) {
                ServerBucketProfile cProfile = (ServerBucketProfile) profile;
                Set<BucketServerLocation66> bucketServerLocations = cProfile.getBucketServerLocations();
                // Either we can make BucketServeLocation having ServerLocation with them
                // Or we can create bucketServerLocation as it is by iterating over the set of servers
                clientBucketProfiles.addAll(bucketServerLocations);
            }
        }
        bucketToServerLocations.put(bucketId, clientBucketProfiles);
    }
    if (getPartitionedRegion().isDataStore()) {
        for (Integer bucketId : getPartitionedRegion().getDataStore().getAllLocalBucketIds()) {
            BucketProfile profile = getBucketAdvisor(bucketId).getLocalProfile();
            if (logger.isDebugEnabled()) {
                logger.debug("The local profile is : {}", profile);
            }
            if (profile != null) {
                List<BucketServerLocation66> clientBucketProfiles = bucketToServerLocations.get(bucketId);
                if (clientBucketProfiles == null) {
                    clientBucketProfiles = new ArrayList<BucketServerLocation66>();
                    bucketToServerLocations.put(bucketId, clientBucketProfiles);
                }
                if ((profile instanceof ServerBucketProfile) && profile.isHosting) {
                    ServerBucketProfile cProfile = (ServerBucketProfile) profile;
                    Set<BucketServerLocation66> bucketServerLocations = cProfile.getBucketServerLocations();
                    // Either we can make BucketServeLocation having ServerLocation with
                    // them
                    // Or we can create bucketServerLocation as it is by iterating over
                    // the set of servers
                    clientBucketProfiles.removeAll(bucketServerLocations);
                    clientBucketProfiles.addAll(bucketServerLocations);
                }
            }
        }
    }
    return bucketToServerLocations;
}

Example 4 with ServerBucketProfile

use of org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile in project geode by apache.

the class RegionAdvisor method getAllClientBucketProfilesTest.

public ConcurrentHashMap<Integer, Set<ServerBucketProfile>> getAllClientBucketProfilesTest() {
    ConcurrentHashMap<Integer, Set<ServerBucketProfile>> map = new ConcurrentHashMap<Integer, Set<ServerBucketProfile>>();
    Map<Integer, List<BucketServerLocation66>> testMap = new HashMap<>(this.getAllClientBucketProfiles());
    for (Integer bucketId : testMap.keySet()) {
        Set<ServerBucketProfile> parr = new HashSet<>(this.clientBucketProfilesMap.get(bucketId));
        map.put(bucketId, parr);
    }
    if (getPartitionedRegion().isDataStore()) {
        for (Integer bucketId : getPartitionedRegion().getDataStore().getAllLocalBucketIds()) {
            BucketProfile profile = getBucketAdvisor(bucketId).getLocalProfile();
            if ((profile instanceof ServerBucketProfile) && profile.isHosting) {
                map.get(bucketId).add((ServerBucketProfile) profile);
            }
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug("This maps is sksk {} and size is {}", map, map.keySet().size());
    }
    return map;
}

Example 5 with ServerBucketProfile

use of org.apache.geode.internal.cache.BucketAdvisor.ServerBucketProfile in project geode by apache.

the class Shipment method testMetadataIsSameOnAllServersAndClientsHA.

@Test
public void testMetadataIsSameOnAllServersAndClientsHA() {
    Integer port0 = (Integer) member0.invoke(() -> PartitionedRegionSingleHopDUnitTest.createServer(2, 4));
    Integer port1 = (Integer) member1.invoke(() -> PartitionedRegionSingleHopDUnitTest.createServer(2, 4));
    createClient(port0, port1, port0, port1);
    put();
    ClientMetadataService cms = ((GemFireCacheImpl) cache).getClientMetadataService();
    cms.getClientPRMetadata((LocalRegion) region);
    final Map<String, ClientPartitionAdvisor> regionMetaData = cms.getClientPRMetadata_TEST_ONLY();
    Awaitility.waitAtMost(60, TimeUnit.SECONDS).until(() -> (regionMetaData.size() == 1));
    assertEquals(1, regionMetaData.size());
    assertTrue(regionMetaData.containsKey(region.getFullPath()));
    member0.invoke(new CacheSerializableRunnable("aba") {

        @Override
        public void run2() throws CacheException {
            final PartitionedRegion pr = (PartitionedRegion) region;
            ConcurrentHashMap<Integer, Set<ServerBucketProfile>> serverMap = pr.getRegionAdvisor().getAllClientBucketProfilesTest();
        }
    });
    member1.invoke(new CacheSerializableRunnable("aba") {

        @Override
        public void run2() throws CacheException {
            final PartitionedRegion pr = (PartitionedRegion) region;
            ConcurrentHashMap<Integer, Set<ServerBucketProfile>> serverMap = pr.getRegionAdvisor().getAllClientBucketProfilesTest();
        }
    });
    ClientPartitionAdvisor prMetaData = regionMetaData.get(region.getFullPath());
    final Map<Integer, List<BucketServerLocation66>> clientMap = prMetaData.getBucketServerLocationsMap_TEST_ONLY();
    Awaitility.waitAtMost(60, TimeUnit.SECONDS).until(() -> (clientMap.size() == 4));
    for (Entry entry : clientMap.entrySet()) {
        assertEquals(2, ((List) entry.getValue()).size());
    }
    member0.invoke(() -> PartitionedRegionSingleHopDUnitTest.verifyMetadata(clientMap));
    member1.invoke(() -> PartitionedRegionSingleHopDUnitTest.verifyMetadata(clientMap));
    member0.invoke(() -> PartitionedRegionSingleHopDUnitTest.stopServer());
    put();
    cms = ((GemFireCacheImpl) cache).getClientMetadataService();
    cms.getClientPRMetadata((LocalRegion) region);
    // member1.invoke(() -> PartitionedRegionSingleHopDUnitTest.verifyMetadata(clientMap));
    assertEquals(4, /* numBuckets */
    clientMap.size());
    for (Entry entry : clientMap.entrySet()) {
        assertEquals(1, ((List) entry.getValue()).size());
    }
    assertEquals(1, regionMetaData.size());
    assertTrue(regionMetaData.containsKey(region.getFullPath()));
    assertEquals(4, /* numBuckets */
    clientMap.size());
    Awaitility.waitAtMost(60, TimeUnit.SECONDS).until(() -> {
        int bucketId = -1;
        int size = -1;
        List globalList = new ArrayList();
        boolean finished = true;
        for (Entry entry : clientMap.entrySet()) {
            List list = (List) entry.getValue();
            if (list.size() != 1) {
                size = list.size();
                globalList = list;
                bucketId = (Integer) entry.getKey();
                finished = false;
                System.out.println("bucket copies are not created, the locations size for bucket id : " + bucketId + " size : " + size + " the list is " + globalList);
            }
        }
        return finished;
    });
}