Examples with IntBigArray com.facebook.presto.common.array.IntBigArray used on opensource projects

Example 1 with IntBigArray

use of com.facebook.presto.common.array.IntBigArray in project presto by prestodb.

the class ValueStore method rehash.

@VisibleForTesting
void rehash() {
    ++rehashCount;
    long newBucketCountLong = bucketCount * 2L;
    if (newBucketCountLong > Integer.MAX_VALUE) {
        throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed " + Integer.MAX_VALUE + " entries (" + newBucketCountLong + ")");
    }
    int newBucketCount = (int) newBucketCountLong;
    int newMask = newBucketCount - 1;
    IntBigArray newBuckets = new IntBigArray(-1);
    newBuckets.ensureCapacity(newBucketCount);
    for (int i = 0; i < values.getPositionCount(); i++) {
        long valueHash = valueHashes.get(i);
        int bucketId = getBucketId(valueHash, newMask);
        int probeCount = 1;
        while (newBuckets.get(bucketId) != EMPTY_BUCKET) {
            int probe = nextProbe(probeCount);
            bucketId = nextBucketId(bucketId, newMask, probe);
            probeCount++;
        }
        // record the mapping
        newBuckets.set(bucketId, i);
    }
    buckets = newBuckets;
    // worst case is every bucket has a unique value, so pre-emptively keep this large enough to have a value for ever bucket
    // TODO: could optimize the growth algorithm to be resize this only when necessary; this wastes memory but guarantees that if every value has a distinct hash, we have space
    valueHashes.ensureCapacity(newBucketCount);
    bucketCount = newBucketCount;
    maxFill = calculateMaxFill(newBucketCount, MAX_FILL_RATIO);
    mask = newMask;
}

Example 2 with IntBigArray

use of com.facebook.presto.common.array.IntBigArray in project presto by prestodb.

the class InMemoryHashAggregationBuilder method hashSortedGroupIds.

private IntIterator hashSortedGroupIds() {
    IntBigArray groupIds = new IntBigArray();
    groupIds.ensureCapacity(groupByHash.getGroupCount());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        groupIds.set(i, i);
    }
    groupIds.sort(0, groupByHash.getGroupCount(), (leftGroupId, rightGroupId) -> Long.compare(groupByHash.getRawHash(leftGroupId), groupByHash.getRawHash(rightGroupId)));
    return new AbstractIntIterator() {

        private final int totalPositions = groupByHash.getGroupCount();

        private int position;

        @Override
        public boolean hasNext() {
            return position < totalPositions;
        }

        @Override
        public int nextInt() {
            return groupIds.get(position++);
        }
    };
}

Example 3 with IntBigArray

use of com.facebook.presto.common.array.IntBigArray in project presto by prestodb.

the class GroupedTypedHistogram method rehash.

private void rehash() {
    long newBucketCountLong = bucketCount * 2L;
    if (newBucketCountLong > Integer.MAX_VALUE) {
        throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed " + Integer.MAX_VALUE + " entries (" + newBucketCountLong + ")");
    }
    int newBucketCount = computeBucketCount((int) newBucketCountLong, MAX_FILL_RATIO);
    int newMask = newBucketCount - 1;
    IntBigArray newBuckets = new IntBigArray(-1);
    newBuckets.ensureCapacity(newBucketCount);
    for (int i = 0; i < nextNodePointer; i++) {
        // find the old one
        int bucketId = getBucketIdForNode(i, newMask);
        int probeCount = 1;
        int originalBucket = bucketId;
        // find new one
        while (newBuckets.get(bucketId) != -1) {
            int probe = nextProbe(probeCount);
            bucketId = nextBucketId(originalBucket, newMask, probe);
            probeCount++;
        }
        // record the mapping
        newBuckets.set(bucketId, i);
    }
    buckets = newBuckets;
    bucketCount = newBucketCount;
    maxFill = calculateMaxFill(newBucketCount, MAX_FILL_RATIO);
    mask = newMask;
    resizeNodeArrays(newBucketCount);
}

Example 4 with IntBigArray

use of com.facebook.presto.common.array.IntBigArray in project presto by prestodb.

the class BigintGroupByHash method tryRehash.

private boolean tryRehash() {
    long newCapacityLong = hashCapacity * 2L;
    if (newCapacityLong > Integer.MAX_VALUE) {
        throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed 1 billion entries");
    }
    int newCapacity = toIntExact(newCapacityLong);
    // An estimate of how much extra memory is needed before we can go ahead and expand the hash table.
    // This includes the new capacity for values, groupIds, and valuesByGroupId as well as the size of the current page
    preallocatedMemoryInBytes = (newCapacity - hashCapacity) * (long) (Long.BYTES + Integer.BYTES) + (calculateMaxFill(newCapacity) - maxFill) * Long.BYTES + currentPageSizeInBytes;
    if (!updateMemory.update()) {
        // reserved memory but has exceeded the limit
        return false;
    }
    preallocatedMemoryInBytes = 0;
    expectedHashCollisions += estimateNumberOfHashCollisions(getGroupCount(), hashCapacity);
    int newMask = newCapacity - 1;
    LongBigArray newValues = new LongBigArray();
    newValues.ensureCapacity(newCapacity);
    IntBigArray newGroupIds = new IntBigArray(-1);
    newGroupIds.ensureCapacity(newCapacity);
    for (int groupId = 0; groupId < nextGroupId; groupId++) {
        if (groupId == nullGroupId) {
            continue;
        }
        long value = valuesByGroupId.get(groupId);
        // find an empty slot for the address
        long hashPosition = getHashPosition(value, newMask);
        while (newGroupIds.get(hashPosition) != -1) {
            hashPosition = (hashPosition + 1) & newMask;
            hashCollisions++;
        }
        // record the mapping
        newValues.set(hashPosition, value);
        newGroupIds.set(hashPosition, groupId);
    }
    mask = newMask;
    hashCapacity = newCapacity;
    maxFill = calculateMaxFill(hashCapacity);
    values = newValues;
    groupIds = newGroupIds;
    this.valuesByGroupId.ensureCapacity(maxFill);
    return true;
}

Example 5 with IntBigArray

use of com.facebook.presto.common.array.IntBigArray in project presto by prestodb.

the class SingleTypedHistogram method rehash.

private void rehash() {
    long newCapacityLong = hashCapacity * 2L;
    if (newCapacityLong > Integer.MAX_VALUE) {
        throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed 1 billion entries");
    }
    int newCapacity = (int) newCapacityLong;
    int newMask = newCapacity - 1;
    IntBigArray newHashPositions = new IntBigArray(-1);
    newHashPositions.ensureCapacity(newCapacity);
    for (int i = 0; i < values.getPositionCount(); i++) {
        // find an empty slot for the address
        int hashPosition = getBucketId(TypeUtils.hashPosition(type, values, i), newMask);
        while (newHashPositions.get(hashPosition) != -1) {
            hashPosition = (hashPosition + 1) & newMask;
        }
        // record the mapping
        newHashPositions.set(hashPosition, i);
    }
    hashCapacity = newCapacity;
    mask = newMask;
    maxFill = calculateMaxFill(newCapacity);
    hashPositions = newHashPositions;
    this.counts.ensureCapacity(maxFill);
}