Examples with PRLoad org.apache.geode.internal.cache.partitioned.PRLoad used on opensource projects

Example 1 with PRLoad

use of org.apache.geode.internal.cache.partitioned.PRLoad in project geode by apache.

the class ResourceManagerDUnitTest method testGetInternalPRDetails.

/**
   * Creates partitioned regions in multiple vms and fully exercises the internal-only
   * getInternalPRDetails API on ResourceManager.
   */
@Test
public void testGetInternalPRDetails() {
    // two regions
    final String[] regionPath = new String[] { getUniqueName() + "-PR-0", getUniqueName() + "-PR-1" };
    // numBuckets config for the two regions
    final int[] numBuckets = new int[] { 100, 90 };
    // redundantCopies config for the two regions
    final int[] redundantCopies = new int[] { 1, 0 };
    // localMaxMemory config to use for three members
    final int[] localMaxMemory = new int[] { 50, 100, 0 };
    // bucketKeys to use for making three bckets in first PR
    final Integer[] bucketKeys = new Integer[] { Integer.valueOf(0), Integer.valueOf(42), Integer.valueOf(76) };
    assertEquals(0, bucketKeys[0].hashCode());
    assertEquals(42, bucketKeys[1].hashCode());
    assertEquals(76, bucketKeys[2].hashCode());
    createRegion(Host.getHost(0).getVM(0), regionPath[0], localMaxMemory[0], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(1), regionPath[0], localMaxMemory[1], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(2), regionPath[0], localMaxMemory[2], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(0), regionPath[1], localMaxMemory[0], numBuckets[1], redundantCopies[1]);
    // 2 MB in size
    final byte[] value = new byte[1024 * 1024 * 2];
    createBuckets(0, regionPath[0], bucketKeys, value);
    // identify the members and their config values
    final InternalDistributedMember[] members = new InternalDistributedMember[3];
    final long[] memberSizes = new long[members.length];
    final int[] memberBucketCounts = new int[members.length];
    final int[] memberPrimaryCounts = new int[members.length];
    for (int i = 0; i < members.length; i++) {
        final int vm = i;
        members[vm] = (InternalDistributedMember) Host.getHost(0).getVM(vm).invoke(new SerializableCallable() {

            public Object call() {
                return getSystem().getDistributedMember();
            }
        });
        memberSizes[vm] = ((Long) Host.getHost(0).getVM(vm).invoke(new SerializableCallable() {

            public Object call() {
                PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(regionPath[0]);
                PartitionedRegionDataStore ds = pr.getDataStore();
                if (ds == null) {
                    return Long.valueOf(0);
                } else {
                    return Long.valueOf(getSize(ds));
                }
            }
        })).longValue();
        memberBucketCounts[vm] = ((Integer) Host.getHost(0).getVM(vm).invoke(new SerializableCallable() {

            public Object call() {
                PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(regionPath[0]);
                PartitionedRegionDataStore ds = pr.getDataStore();
                if (ds == null) {
                    return new Integer(0);
                } else {
                    return new Integer(ds.getBucketsManaged());
                }
            }
        })).intValue();
        memberPrimaryCounts[vm] = ((Integer) Host.getHost(0).getVM(vm).invoke(new SerializableCallable() {

            public Object call() {
                PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(regionPath[0]);
                PartitionedRegionDataStore ds = pr.getDataStore();
                if (ds == null) {
                    return new Integer(0);
                } else {
                    return new Integer(ds.getNumberOfPrimaryBucketsManaged());
                }
            }
        })).intValue();
    }
    // test everything here
    for (int i = 0; i < localMaxMemory.length; i++) {
        final int vm = i;
        Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {

            public void run() {
                Set<InternalPRInfo> detailsSet = new HashSet<InternalPRInfo>();
                GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
                for (PartitionedRegion pr : cache.getPartitionedRegions()) {
                    InternalPRInfo info = pr.getRedundancyProvider().buildPartitionedRegionInfo(true, cache.getInternalResourceManager().getLoadProbe());
                    detailsSet.add(info);
                }
                if (vm == 0) {
                    assertEquals(2, detailsSet.size());
                } else {
                    assertEquals(1, detailsSet.size());
                }
                // iterate over each InternalPRDetails
                for (Iterator<InternalPRInfo> prIter = detailsSet.iterator(); prIter.hasNext(); ) {
                    InternalPRInfo details = prIter.next();
                    // NOTE: getRegionPath() contains the Region.SEPARATOR + regionPath
                    assertTrue("Unknown regionPath=" + details.getRegionPath(), details.getRegionPath().contains(regionPath[0]) || details.getRegionPath().contains(regionPath[1]));
                    if (details.getRegionPath().contains(regionPath[0])) {
                        assertEquals(numBuckets[0], details.getConfiguredBucketCount());
                        assertEquals(0, details.getLowRedundancyBucketCount());
                        assertEquals(redundantCopies[0], details.getConfiguredRedundantCopies());
                        assertEquals(redundantCopies[0], details.getActualRedundantCopies());
                        assertNull(details.getColocatedWith());
                        Set<InternalPartitionDetails> memberDetails = details.getInternalPartitionDetails();
                        assertNotNull(memberDetails);
                        assertEquals(localMaxMemory.length - 1, memberDetails.size());
                        // iterate over each InternalPartitionDetails (datastores only)
                        for (Iterator<InternalPartitionDetails> mbrIter = memberDetails.iterator(); mbrIter.hasNext(); ) {
                            InternalPartitionDetails mbrDetails = mbrIter.next();
                            assertNotNull(mbrDetails);
                            DistributedMember mbr = mbrDetails.getDistributedMember();
                            assertNotNull(mbr);
                            int membersIdx = -1;
                            for (int idx = 0; idx < members.length; idx++) {
                                if (mbr.equals(members[idx])) {
                                    membersIdx = idx;
                                }
                            }
                            assertEquals(localMaxMemory[membersIdx] * 1024 * 1024, mbrDetails.getConfiguredMaxMemory());
                            assertEquals(memberSizes[membersIdx], mbrDetails.getSize());
                            assertEquals(memberBucketCounts[membersIdx], mbrDetails.getBucketCount());
                            assertEquals(memberPrimaryCounts[membersIdx], mbrDetails.getPrimaryCount());
                            PRLoad load = mbrDetails.getPRLoad();
                            assertNotNull(load);
                            assertEquals((float) localMaxMemory[membersIdx], load.getWeight(), 0);
                            int totalBucketBytes = 0;
                            int primaryCount = 0;
                            for (int bid = 0; bid < numBuckets[0]; bid++) {
                                long bucketBytes = mbrDetails.getBucketSize(bid);
                                assertTrue(bucketBytes >= 0);
                                totalBucketBytes += bucketBytes;
                                // validate against the PRLoad
                                assertEquals((float) bucketBytes, load.getReadLoad(bid), 0);
                                if (load.getWriteLoad(bid) > 0) {
                                    // found a primary
                                    primaryCount++;
                                }
                            }
                            // assertIndexDetailsEquals(memberSizes[membersIdx] * (1024* 1024),
                            // totalBucketBytes);
                            assertEquals(memberPrimaryCounts[membersIdx], primaryCount);
                            if (mbr.equals(getSystem().getDistributedMember())) {
                                // PartitionMemberDetails represents the local member
                                PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(details.getRegionPath());
                                assertEquals(pr.getLocalMaxMemory() * (1024L * 1024L), mbrDetails.getConfiguredMaxMemory());
                                PartitionedRegionDataStore ds = pr.getDataStore();
                                assertNotNull(ds);
                                assertEquals(getSize(ds), mbrDetails.getSize());
                                assertEquals(ds.getBucketsManaged(), mbrDetails.getBucketCount());
                                assertEquals(ds.getNumberOfPrimaryBucketsManaged(), mbrDetails.getPrimaryCount());
                            }
                        }
                    } else {
                        // found the other PR which has only one datastore and we know
                        // this system memberId is the only entry in mbrDetails
                        assertEquals(numBuckets[1], details.getConfiguredBucketCount());
                        assertEquals(0, details.getLowRedundancyBucketCount());
                        assertEquals(redundantCopies[1], details.getConfiguredRedundantCopies());
                        assertEquals(redundantCopies[1], details.getActualRedundantCopies());
                        assertNull(details.getColocatedWith());
                        Set<PartitionMemberInfo> memberDetails = details.getPartitionMemberInfo();
                        assertNotNull(memberDetails);
                        assertEquals(1, memberDetails.size());
                        PartitionMemberInfo mbrDetails = memberDetails.iterator().next();
                        assertEquals(getSystem().getDistributedMember(), mbrDetails.getDistributedMember());
                        PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(details.getRegionPath());
                        assertEquals(pr.getLocalMaxMemory() * (1024L * 1024L), mbrDetails.getConfiguredMaxMemory());
                        PartitionedRegionDataStore ds = pr.getDataStore();
                        assertNotNull(ds);
                        assertEquals(getSize(ds), mbrDetails.getSize());
                        assertEquals(ds.getBucketsManaged(), mbrDetails.getBucketCount());
                        assertEquals(ds.getNumberOfPrimaryBucketsManaged(), mbrDetails.getPrimaryCount());
                    }
                }
            }
        });
    }
    destroyRegions(0, regionPath);
}

Example 2 with PRLoad

use of org.apache.geode.internal.cache.partitioned.PRLoad in project geode by apache.

the class PartitionedRegionLoadModel method addRegion.

/**
   * Add a region to the model. All regions that are added are assumed to be colocated. The first
   * region added to the model should be the parent region. The parent region is expected to have at
   * least as many members as child regions; it may have more. If the parent has more members than
   * child regions those members will be considered invalid.
   * 
   * @param region
   * @param memberDetailSet A set of details about each member.
   * @param offlineDetails
   */
public void addRegion(String region, Collection<? extends InternalPartitionDetails> memberDetailSet, OfflineMemberDetails offlineDetails, boolean enforceLocalMaxMemory) {
    this.allColocatedRegions.add(region);
    // build up a list of members and an array of buckets for this
    // region. Each bucket has a reference to all of the members
    // that host it and each member has a reference to all of the buckets
    // it hosts
    Map<InternalDistributedMember, Member> regionMember = new HashMap<InternalDistributedMember, Member>();
    Bucket[] regionBuckets = new Bucket[this.buckets.length];
    for (InternalPartitionDetails memberDetails : memberDetailSet) {
        InternalDistributedMember memberId = (InternalDistributedMember) memberDetails.getDistributedMember();
        boolean isCritical = criticalMembers.contains(memberId);
        Member member = new Member(memberId, memberDetails.getPRLoad().getWeight(), memberDetails.getConfiguredMaxMemory(), isCritical, enforceLocalMaxMemory);
        regionMember.put(memberId, member);
        PRLoad load = memberDetails.getPRLoad();
        for (int i = 0; i < regionBuckets.length; i++) {
            if (load.getReadLoad(i) > 0) {
                Bucket bucket = regionBuckets[i];
                if (bucket == null) {
                    Set<PersistentMemberID> offlineMembers = offlineDetails.getOfflineMembers(i);
                    bucket = new Bucket(i, load.getReadLoad(i), memberDetails.getBucketSize(i), offlineMembers);
                    regionBuckets[i] = bucket;
                }
                bucket.addMember(member);
                if (load.getWriteLoad(i) > 0) {
                    if (bucket.getPrimary() == null) {
                        bucket.setPrimary(member, load.getWriteLoad(i));
                    } else if (!bucket.getPrimary().equals(member)) {
                        bucket.setPrimary(INVALID_MEMBER, 1);
                    }
                }
            }
        }
    }
    // regions
    for (Member member : regionMember.values()) {
        InternalDistributedMember memberId = member.getDistributedMember();
        MemberRollup memberSum = this.members.get(memberId);
        boolean isCritical = criticalMembers.contains(memberId);
        if (memberSum == null) {
            memberSum = new MemberRollup(memberId, isCritical, enforceLocalMaxMemory);
            this.members.put(memberId, memberSum);
        }
        memberSum.addColocatedMember(region, member);
    }
    // regions and buckets
    for (int i = 0; i < this.buckets.length; i++) {
        if (regionBuckets[i] == null) {
            // do nothing, this bucket is not hosted for this region.
            // [sumedh] remove from buckets array too to be consistent since
            // this method will be invoked repeatedly for all colocated regions,
            // and then we may miss some colocated regions for a bucket leading
            // to all kinds of issues later
            this.buckets[i] = null;
            continue;
        }
        if (this.buckets[i] == null) {
            // If this is the first region we have seen that is hosting this bucket, create a bucket
            // rollup
            this.buckets[i] = new BucketRollup(i);
        }
        // Add all of the members hosting the bucket to the rollup
        for (Member member : regionBuckets[i].getMembersHosting()) {
            InternalDistributedMember memberId = member.getDistributedMember();
            this.buckets[i].addMember(this.members.get(memberId));
        }
        // set the primary for the rollup
        if (regionBuckets[i].getPrimary() != null) {
            if (this.buckets[i].getPrimary() == null) {
                InternalDistributedMember memberId = regionBuckets[i].getPrimary().getDistributedMember();
                this.buckets[i].setPrimary(this.members.get(memberId), 0);
            } else {
                if (!(this.buckets[i].getPrimary() == INVALID_MEMBER)) {
                    if (!this.buckets[i].getPrimary().getDistributedMember().equals(regionBuckets[i].getPrimary().getDistributedMember())) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("PartitionedRegionLoadModel - Setting bucket {} to INVALID because it is the primary on two members.This could just be a race in the collocation of data. member1={} member2={}", this.buckets[i], this.buckets[i].getPrimary(), regionBuckets[i].getPrimary());
                        }
                        this.buckets[i].setPrimary(INVALID_MEMBER, 0);
                    }
                }
            }
        }
        this.buckets[i].addColocatedBucket(region, regionBuckets[i]);
    }
    // If any members don't have this new region, remove them.
    for (Iterator<Entry<InternalDistributedMember, MemberRollup>> itr = members.entrySet().iterator(); itr.hasNext(); ) {
        MemberRollup memberRollup = itr.next().getValue();
        if (!memberRollup.getColocatedMembers().keySet().equals(this.allColocatedRegions)) {
            itr.remove();
            if (logger.isDebugEnabled()) {
                logger.debug("PartitionedRegionLoadModel - removing member {} from the consideration because it doesn't have all of the colocated regions. Expected={}, was={}", memberRollup, allColocatedRegions, memberRollup.getColocatedMembers());
            }
            // This state should never happen
            if (!memberRollup.getBuckets().isEmpty()) {
                logger.warn(LocalizedMessage.create(LocalizedStrings.PartitionedRegionLoadModel_INCOMPLETE_COLOCATION, new Object[] { memberRollup, this.allColocatedRegions, memberRollup.getColocatedMembers().keySet(), memberRollup.getBuckets() }));
            }
            for (Bucket bucket : new HashSet<Bucket>(memberRollup.getBuckets())) {
                bucket.removeMember(memberRollup);
            }
        }
    }
}