Examples with LongBigArray com.facebook.presto.common.array.LongBigArray used on opensource projects

Example 1 with LongBigArray

use of com.facebook.presto.common.array.LongBigArray in project presto by prestodb.

the class BigintGroupByHash method tryRehash.

private boolean tryRehash() {
    long newCapacityLong = hashCapacity * 2L;
    if (newCapacityLong > Integer.MAX_VALUE) {
        throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed 1 billion entries");
    }
    int newCapacity = toIntExact(newCapacityLong);
    // An estimate of how much extra memory is needed before we can go ahead and expand the hash table.
    // This includes the new capacity for values, groupIds, and valuesByGroupId as well as the size of the current page
    preallocatedMemoryInBytes = (newCapacity - hashCapacity) * (long) (Long.BYTES + Integer.BYTES) + (calculateMaxFill(newCapacity) - maxFill) * Long.BYTES + currentPageSizeInBytes;
    if (!updateMemory.update()) {
        // reserved memory but has exceeded the limit
        return false;
    }
    preallocatedMemoryInBytes = 0;
    expectedHashCollisions += estimateNumberOfHashCollisions(getGroupCount(), hashCapacity);
    int newMask = newCapacity - 1;
    LongBigArray newValues = new LongBigArray();
    newValues.ensureCapacity(newCapacity);
    IntBigArray newGroupIds = new IntBigArray(-1);
    newGroupIds.ensureCapacity(newCapacity);
    for (int groupId = 0; groupId < nextGroupId; groupId++) {
        if (groupId == nullGroupId) {
            continue;
        }
        long value = valuesByGroupId.get(groupId);
        // find an empty slot for the address
        long hashPosition = getHashPosition(value, newMask);
        while (newGroupIds.get(hashPosition) != -1) {
            hashPosition = (hashPosition + 1) & newMask;
            hashCollisions++;
        }
        // record the mapping
        newValues.set(hashPosition, value);
        newGroupIds.set(hashPosition, groupId);
    }
    mask = newMask;
    hashCapacity = newCapacity;
    maxFill = calculateMaxFill(hashCapacity);
    values = newValues;
    groupIds = newGroupIds;
    this.valuesByGroupId.ensureCapacity(maxFill);
    return true;
}

Example 2 with LongBigArray

use of com.facebook.presto.common.array.LongBigArray in project presto by prestodb.

the class StreamSummary method compact.

private synchronized void compact() {
    BlockBuilder newHeapBlockBuilder = type.createBlockBuilder(null, heapBlockBuilder.getPositionCount());
    // since block positions are changed, we need to update all data structures which are using block position as reference
    LongBigArray newBlockPositionToCount = new LongBigArray();
    hashCapacity = arraySize(heapCapacity, FILL_RATIO);
    maxFill = calculateMaxFill(hashCapacity);
    newBlockPositionToCount.ensureCapacity(hashCapacity);
    IntBigArray newBlockToHeapIndex = new IntBigArray();
    newBlockToHeapIndex.ensureCapacity(hashCapacity);
    for (int heapPosition = 0; heapPosition < getHeapSize(); heapPosition++) {
        int newBlockPos = newHeapBlockBuilder.getPositionCount();
        StreamDataEntity heapEntry = minHeap.get(heapPosition);
        int oldBlockPosition = getBlockPosition(heapEntry);
        type.appendTo(heapBlockBuilder, oldBlockPosition, newHeapBlockBuilder);
        newBlockPositionToCount.set(newBlockPos, blockPositionToCount.get(oldBlockPosition));
        newBlockToHeapIndex.set(newBlockPos, heapPosition);
        hashToBlockPosition.set(heapEntry.getHashPosition(), newBlockPos);
    }
    blockPositionToCount = newBlockPositionToCount;
    heapBlockBuilder = newHeapBlockBuilder;
    blockToHeapIndex = newBlockToHeapIndex;
    rehash();
}

Example 3 with LongBigArray

use of com.facebook.presto.common.array.LongBigArray in project presto by prestodb.

the class BenchmarkGroupByHash method baselineBigArray.

@Benchmark
@OperationsPerInvocation(POSITIONS)
public long baselineBigArray(BaselinePagesData data) {
    int hashSize = arraySize(GROUP_COUNT, 0.9f);
    int mask = hashSize - 1;
    LongBigArray table = new LongBigArray(-1);
    table.ensureCapacity(hashSize);
    long groupIds = 0;
    for (Page page : data.getPages()) {
        Block block = page.getBlock(0);
        int positionCount = block.getPositionCount();
        for (int position = 0; position < positionCount; position++) {
            long value = BIGINT.getLong(block, position);
            int tablePosition = (int) XxHash64.hash(value) & mask;
            while (table.get(tablePosition) != -1 && table.get(tablePosition) != value) {
                tablePosition++;
            }
            if (table.get(tablePosition) == -1) {
                table.set(tablePosition, value);
                groupIds++;
            }
        }
    }
    return groupIds;
}