Examples with IntArraySet it.unimi.dsi.fastutil.ints.IntArraySet used on opensource projects

Example 1 with IntArraySet

use of it.unimi.dsi.fastutil.ints.IntArraySet in project presto by prestodb.

the class HivePageSink method writePage.

private void writePage(Page page) {
    int[] writerIndexes = getWriterIndexes(page);
    // record which positions are used by which writer
    for (int position = 0; position < page.getPositionCount(); position++) {
        int writerIndex = writerIndexes[position];
        writerPositions.get(writerIndex).add(position);
    }
    // invoke the writers
    Page dataPage = getDataPage(page);
    IntSet writersUsed = new IntArraySet(writerIndexes);
    for (IntIterator iterator = writersUsed.iterator(); iterator.hasNext(); ) {
        int writerIndex = iterator.nextInt();
        WriterPositions currentWriterPositions = writerPositions.get(writerIndex);
        if (currentWriterPositions.isEmpty()) {
            continue;
        }
        // If write is partitioned across multiple writers, filter page using dictionary blocks
        Page pageForWriter = dataPage;
        if (currentWriterPositions.size() != dataPage.getPositionCount()) {
            Block[] blocks = new Block[dataPage.getChannelCount()];
            for (int channel = 0; channel < dataPage.getChannelCount(); channel++) {
                blocks[channel] = new DictionaryBlock(currentWriterPositions.size(), dataPage.getBlock(channel), currentWriterPositions.getPositionsArray());
            }
            pageForWriter = new Page(currentWriterPositions.size(), blocks);
        }
        HiveWriter writer = writers.get(writerIndex);
        long currentMemory = writer.getSystemMemoryUsage();
        writer.append(pageForWriter);
        systemMemoryUsage += (writer.getSystemMemoryUsage() - currentMemory);
        currentWriterPositions.clear();
    }
}