Examples with DATA_RANGE_IN_MB org.apache.tez.runtime.library.utils.DATA_RANGE_IN_MB used on opensource projects

Example 1 with DATA_RANGE_IN_MB

use of org.apache.tez.runtime.library.utils.DATA_RANGE_IN_MB in project tez by apache.

the class ShuffleVertexManagerBase method parsePartitionStats.

@VisibleForTesting
void parsePartitionStats(SourceVertexInfo srcInfo, RoaringBitmap partitionStats) {
    Preconditions.checkState(srcInfo.statsInMB != null, "Stats should be initialized");
    Iterator<Integer> it = partitionStats.iterator();
    final DATA_RANGE_IN_MB[] RANGES = DATA_RANGE_IN_MB.values();
    final int RANGE_LEN = RANGES.length;
    while (it.hasNext()) {
        int pos = it.next();
        int index = ((pos) / RANGE_LEN);
        int rangeIndex = ((pos) % RANGE_LEN);
        // Add to aggregated stats and normalize to DATA_RANGE_IN_MB.
        if (RANGES[rangeIndex].getSizeInMB() > 0) {
            srcInfo.statsInMB[index] += RANGES[rangeIndex].getSizeInMB();
        }
    }
}

Example 2 with DATA_RANGE_IN_MB

use of org.apache.tez.runtime.library.utils.DATA_RANGE_IN_MB in project tez by apache.

the class TestUnorderedPartitionedKVWriter method getPartitionStats.

private int[] getPartitionStats(VertexManagerEvent vme) throws IOException {
    RoaringBitmap partitionStats = new RoaringBitmap();
    VertexManagerEventPayloadProto payload = VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vme.getUserPayload()));
    if (!reportPartitionStats.isEnabled()) {
        assertFalse(payload.hasPartitionStats());
        assertFalse(payload.hasDetailedPartitionStats());
        return null;
    }
    if (reportPartitionStats.isPrecise()) {
        assertTrue(payload.hasDetailedPartitionStats());
        List<Integer> sizeInMBList = payload.getDetailedPartitionStats().getSizeInMbList();
        int[] stats = new int[sizeInMBList.size()];
        for (int i = 0; i < sizeInMBList.size(); i++) {
            stats[i] += sizeInMBList.get(i);
        }
        return stats;
    } else {
        assertTrue(payload.hasPartitionStats());
        ByteString compressedPartitionStats = payload.getPartitionStats();
        byte[] rawData = TezCommonUtils.decompressByteStringToByteArray(compressedPartitionStats);
        ByteArrayInputStream bin = new ByteArrayInputStream(rawData);
        partitionStats.deserialize(new DataInputStream(bin));
        int[] stats = new int[partitionStats.getCardinality()];
        Iterator<Integer> it = partitionStats.iterator();
        final DATA_RANGE_IN_MB[] RANGES = DATA_RANGE_IN_MB.values();
        final int RANGE_LEN = RANGES.length;
        while (it.hasNext()) {
            int pos = it.next();
            int index = ((pos) / RANGE_LEN);
            int rangeIndex = ((pos) % RANGE_LEN);
            if (RANGES[rangeIndex].getSizeInMB() > 0) {
                stats[index] += RANGES[rangeIndex].getSizeInMB();
            }
        }
        return stats;
    }
}