Examples with Memory org.apache.datasketches.memory.Memory used on opensource projects

Example 26 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ItemsSketch method getInstance.

/**
 * Heapifies the given srcMem, which must be a Memory image of a ItemsSketch
 * @param <T> type of item
 * @param srcMem a Memory image of a sketch.
 * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
 * @param comparator to compare items
 * @param serDe an instance of ArrayOfItemsSerDe
 * @return a ItemsSketch on the Java heap.
 */
public static <T> ItemsSketch<T> getInstance(final Memory srcMem, final Comparator<? super T> comparator, final ArrayOfItemsSerDe<T> serDe) {
    final long memCapBytes = srcMem.getCapacity();
    if (memCapBytes < 8) {
        throw new SketchesArgumentException("Memory too small: " + memCapBytes);
    }
    final int preambleLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final int familyID = extractFamilyID(srcMem);
    final int flags = extractFlags(srcMem);
    final int k = extractK(srcMem);
    ItemsUtil.checkItemsSerVer(serVer);
    if (serVer == 3 && (flags & COMPACT_FLAG_MASK) == 0) {
        throw new SketchesArgumentException("Non-compact Memory images are not supported.");
    }
    final boolean empty = Util.checkPreLongsFlagsCap(preambleLongs, flags, memCapBytes);
    Util.checkFamilyID(familyID);
    // checks k
    final ItemsSketch<T> qs = getInstance(k, comparator);
    if (empty) {
        return qs;
    }
    // Not empty, must have valid preamble + min, max
    final long n = extractN(srcMem);
    // can't check memory capacity here, not enough information
    // for min, max
    final int extra = 2;
    final int numMemItems = Util.computeRetainedItems(k, n) + extra;
    // set class members
    qs.n_ = n;
    qs.combinedBufferItemCapacity_ = Util.computeCombinedBufferItemCapacity(k, n);
    qs.baseBufferCount_ = computeBaseBufferItems(k, n);
    qs.bitPattern_ = computeBitPattern(k, n);
    qs.combinedBuffer_ = new Object[qs.combinedBufferItemCapacity_];
    final int srcMemItemsOffsetBytes = preambleLongs * Long.BYTES;
    final Memory mReg = srcMem.region(srcMemItemsOffsetBytes, srcMem.getCapacity() - srcMemItemsOffsetBytes);
    final T[] itemsArray = serDe.deserializeFromMemory(mReg, numMemItems);
    qs.itemsArrayToCombinedBuffer(itemsArray);
    return qs;
}

Example 27 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class VarOptItemsUnion method heapify.

/**
 * Instantiates a Union from Memory
 *
 * @param <T> The type of item this sketch contains
 * @param srcMem Memory object containing a serialized union
 * @param serDe An instance of ArrayOfItemsSerDe
 * @return A VarOptItemsUnion created from the provided Memory
 */
public static <T> VarOptItemsUnion<T> heapify(final Memory srcMem, final ArrayOfItemsSerDe<T> serDe) {
    Family.VAROPT_UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    long n = 0;
    double outerTauNum = 0.0;
    long outerTauDenom = 0;
    final int numPreLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    final int maxK = extractMaxK(srcMem);
    if (!isEmpty) {
        n = extractN(srcMem);
        outerTauNum = extractOuterTauNumerator(srcMem);
        outerTauDenom = extractOuterTauDenominator(srcMem);
    }
    if (serVer != SER_VER) {
        throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
    }
    final boolean preLongsEqMin = (numPreLongs == Family.VAROPT_UNION.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.VAROPT_UNION.getMaxPreLongs());
    if (!preLongsEqMin && !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty union with only " + Family.VAROPT_UNION.getMinPreLongs() + "preLongs");
    }
    final VarOptItemsUnion<T> viu = new VarOptItemsUnion<>(maxK);
    if (isEmpty) {
        viu.gadget_ = VarOptItemsSketch.newInstanceAsGadget(maxK);
    } else {
        viu.n_ = n;
        viu.outerTauNumer = outerTauNum;
        viu.outerTauDenom = outerTauDenom;
        final int preLongBytes = numPreLongs << 3;
        final Memory sketchMem = srcMem.region(preLongBytes, srcMem.getCapacity() - preLongBytes);
        viu.gadget_ = VarOptItemsSketch.heapify(sketchMem, serDe);
    }
    return viu;
}

Example 28 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class CompactOperations method componentsToCompact.

static // No error checking
CompactSketch componentsToCompact(final long thetaLong, final int curCount, final short seedHash, final boolean srcEmpty, final boolean srcCompact, final boolean srcOrdered, final boolean dstOrdered, final WritableMemory dstMem, // may not be compacted, ordered or unordered, may be null
final long[] hashArr) {
    final boolean direct = dstMem != null;
    final boolean empty = srcEmpty || ((curCount == 0) && (thetaLong == Long.MAX_VALUE));
    final boolean single = (curCount == 1) && (thetaLong == Long.MAX_VALUE);
    final long[] hashArrOut;
    if (!srcCompact) {
        hashArrOut = CompactOperations.compactCache(hashArr, curCount, thetaLong, dstOrdered);
    } else {
        hashArrOut = hashArr;
    }
    if (!srcOrdered && dstOrdered && !empty && !single) {
        Arrays.sort(hashArrOut);
    }
    // Note: for empty or single we always output the ordered form.
    final boolean dstOrderedOut = (empty || single) ? true : dstOrdered;
    if (direct) {
        final int preLongs = computeCompactPreLongs(empty, curCount, thetaLong);
        // always LE
        int flags = READ_ONLY_FLAG_MASK | COMPACT_FLAG_MASK;
        flags |= empty ? EMPTY_FLAG_MASK : 0;
        flags |= dstOrderedOut ? ORDERED_FLAG_MASK : 0;
        flags |= single ? SINGLEITEM_FLAG_MASK : 0;
        final Memory mem = loadCompactMemory(hashArrOut, seedHash, curCount, thetaLong, dstMem, (byte) flags, preLongs);
        return new DirectCompactSketch(mem);
    } else {
        // Heap
        if (empty) {
            return EmptyCompactSketch.getInstance();
        }
        if (single) {
            return new SingleItemSketch(hashArrOut[0], seedHash);
        }
        return new HeapCompactSketch(hashArrOut, empty, seedHash, curCount, thetaLong, dstOrderedOut);
    }
}

Example 29 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class SketchesTest method getMemoryFromCompactSketch.

private static Memory getMemoryFromCompactSketch(final CompactSketch csk) {
    final byte[] sk1bytes = csk.toByteArray();
    final Memory mem = Memory.wrap(sk1bytes);
    return mem;
}

Example 30 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class SketchesTest method getCompactSketchMemory.

private static Memory getCompactSketchMemory(final int k, final int from, final int to) {
    final UpdateSketch sk1 = updateSketchBuilder().setNominalEntries(k).build();
    for (int i = from; i < to; i++) {
        sk1.update(i);
    }
    final CompactSketch csk = sk1.compact(true, null);
    final byte[] sk1bytes = csk.toByteArray();
    final Memory mem = Memory.wrap(sk1bytes);
    return mem;
}