Examples with FastBitSet javolution.util.FastBitSet used on opensource projects

Example 6 with FastBitSet

use of javolution.util.FastBitSet in project hive by apache.

the class GroupByOperator method process.

@Override
public void process(Object row, int tag) throws HiveException {
    firstRow = false;
    ObjectInspector rowInspector = inputObjInspectors[0];
    // Total number of input rows is needed for hash aggregation only
    if (hashAggr) {
        numRowsInput++;
        // if hash aggregation is not behaving properly, disable it
        if (numRowsInput == numRowsCompareHashAggr) {
            numRowsCompareHashAggr += groupbyMapAggrInterval;
            long numRowsProcessed = groupingSetsPresent ? numRowsInput * groupingSets.size() : numRowsInput;
            // map-side aggregation should reduce the entries by at-least half
            if (numRowsHashTbl > numRowsProcessed * minReductionHashAggr) {
                LOG.warn("Disable Hash Aggr: #hash table = " + numRowsHashTbl + " #numRowsInput = " + numRowsInput + " reduction = " + 1.0 * (numRowsHashTbl / numRowsProcessed) + " minReduction = " + minReductionHashAggr + " groupingSetsPresent = " + groupingSetsPresent + " numRowsProcessed = " + numRowsProcessed);
                flushHashTable(true);
                hashAggr = false;
            } else {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Hash Aggr Enabled: #hash table = " + numRowsHashTbl + " #numRowsInput = " + numRowsInput + " reduction = " + 1.0 * (numRowsHashTbl / numRowsProcessed) + " minReduction = " + minReductionHashAggr + " groupingSetsPresent = " + groupingSetsPresent + " numRowsProcessed = " + numRowsProcessed);
                }
            }
        }
    }
    try {
        countAfterReport++;
        newKeys.getNewKey(row, rowInspector);
        if (groupingSetsPresent) {
            Object[] newKeysArray = newKeys.getKeyArray();
            Object[] cloneNewKeysArray = new Object[newKeysArray.length];
            System.arraycopy(newKeysArray, 0, cloneNewKeysArray, 0, groupingSetsPosition);
            for (int groupingSetPos = 0; groupingSetPos < groupingSets.size(); groupingSetPos++) {
                Arrays.fill(newKeysArray, 0, groupingSetsPosition, null);
                FastBitSet bitset = groupingSetsBitSet[groupingSetPos];
                // Some keys need to be left to null corresponding to that grouping set.
                for (int keyPos = bitset.nextClearBit(0); keyPos < groupingSetsPosition; keyPos = bitset.nextClearBit(keyPos + 1)) {
                    newKeysArray[keyPos] = cloneNewKeysArray[keyPos];
                }
                newKeysArray[groupingSetsPosition] = newKeysGroupingSets[groupingSetPos];
                processKey(row, rowInspector);
            }
        } else {
            processKey(row, rowInspector);
        }
    } catch (HiveException e) {
        throw e;
    } catch (Exception e) {
        throw new HiveException(e);
    }
}

Example 7 with FastBitSet

use of javolution.util.FastBitSet in project hive by apache.

the class FMSketchUtils method readBitVector.

private static FastBitSet readBitVector(InputStream in) throws IOException {
    FastBitSet fastBitSet = new FastBitSet();
    fastBitSet.clear();
    for (int i = 0; i < 4; i++) {
        byte b = (byte) in.read();
        for (int j = 0; j < 8; j++) {
            if ((b & (1 << j)) != 0) {
                fastBitSet.set(j + 8 * i);
            }
        }
    }
    return fastBitSet;
}