Examples with IntegerSerializerDeserializer org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer used on opensource projects

Example 1 with IntegerSerializerDeserializer

use of org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer in project asterixdb by apache.

the class DataGenOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
    final RecordDescriptor recDesc = outRecDescs[0];
    final ArrayTupleBuilder tb = new ArrayTupleBuilder(recDesc.getFields().length);
    final Random rnd = new Random(randomSeed);
    final int maxUniqueAttempts = 20;
    return new AbstractUnaryOutputSourceOperatorNodePushable() {

        // for quick & dirty exclusion of duplicates
        // WARNING: could contain numRecord entries and use a lot of memory
        HashSet<String> stringHs = new HashSet<String>();

        HashSet<Integer> intHs = new HashSet<Integer>();

        @Override
        public void initialize() throws HyracksDataException {
            try {
                writer.open();
                for (int i = 0; i < numRecords; i++) {
                    tb.reset();
                    for (int j = 0; j < recDesc.getFieldCount(); j++) {
                        genField(tb, j);
                    }
                    if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                        appender.write(writer, true);
                        if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                            throw new HyracksDataException("Record size (" + tb.getSize() + ") larger than frame size (" + appender.getBuffer().capacity() + ")");
                        }
                    }
                }
                appender.write(writer, true);
            } catch (Throwable th) {
                writer.fail();
                throw new HyracksDataException(th);
            } finally {
                writer.close();
            }
        }

        private void genField(ArrayTupleBuilder tb, int fieldIndex) throws HyracksDataException {
            DataOutput dos = tb.getDataOutput();
            if (recDesc.getFields()[fieldIndex] instanceof IntegerSerializerDeserializer) {
                int val = -1;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        int tmp = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                        if (intHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            intHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else if (recDesc.getFields()[fieldIndex] instanceof UTF8StringSerializerDeserializer) {
                String val = null;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        String tmp = randomString(maxStrLen, rnd);
                        if (stringHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            stringHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = randomString(maxStrLen, rnd);
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else {
                throw new HyracksDataException("Type unsupported in data generator. Only integers and strings allowed");
            }
        }

        private String randomString(int length, Random random) {
            String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
            StringBuilder strBuilder = new StringBuilder();
            for (int i = 0; i < s.length() && i < length; i++) {
                strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
            }
            return strBuilder.toString();
        }
    };
}

Example 2 with IntegerSerializerDeserializer

use of org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer in project asterixdb by apache.

the class LSMBTreeFilterMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
    OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType, true);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, true, getRandom());
    } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, true, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_BTREE_MAX_TREES_TO_MERGE;
    ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
    IBinaryComparator comp = ctx.getComparatorFactories()[0].createBinaryComparator();
    for (int i = 0; i < maxTreesToMerge; i++) {
        int flushed = 0;
        for (; flushed < i; flushed++) {
            Pair<ITupleReference, ITupleReference> minMax = null;
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                minMax = orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, true, getRandom());
            } else {
                minMax = orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, true, getRandom());
            }
            if (minMax != null) {
                ILSMComponentFilter f = ((LSMBTree) ctx.getIndex()).getCurrentMemoryComponent().getLSMComponentFilter();
                Pair<ITupleReference, ITupleReference> obsMinMax = filterToMinMax(f);
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp));
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getRight(), minMax.getRight(), comp));
            }
            StubIOOperationCallback stub = new StubIOOperationCallback();
            BlockingIOOperationCallbackWrapper waiter = new BlockingIOOperationCallbackWrapper(stub);
            accessor.scheduleFlush(waiter);
            waiter.waitForIO();
            if (minMax != null) {
                Pair<ITupleReference, ITupleReference> obsMinMax = filterToMinMax(stub.getLastNewComponent().getLSMComponentFilter());
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp));
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getRight(), minMax.getRight(), comp));
            }
        }
        List<ILSMDiskComponent> flushedComponents = ((LSMBTree) ctx.getIndex()).getImmutableComponents();
        MutablePair<ITupleReference, ITupleReference> expectedMergeMinMax = null;
        for (ILSMDiskComponent f : flushedComponents) {
            Pair<ITupleReference, ITupleReference> componentMinMax = filterToMinMax(f.getLSMComponentFilter());
            if (expectedMergeMinMax == null) {
                expectedMergeMinMax = MutablePair.of(componentMinMax.getLeft(), componentMinMax.getRight());
            }
            if (TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getLeft(), componentMinMax.getLeft(), comp) > 0) {
                expectedMergeMinMax.setLeft(componentMinMax.getLeft());
            }
            if (TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getRight(), componentMinMax.getRight(), comp) < 0) {
                expectedMergeMinMax.setRight(componentMinMax.getRight());
            }
        }
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((LSMBTree) ctx.getIndex()).getImmutableComponents());
        flushedComponents = ((LSMBTree) ctx.getIndex()).getImmutableComponents();
        Pair<ITupleReference, ITupleReference> mergedMinMax = filterToMinMax(flushedComponents.get(0).getLSMComponentFilter());
        Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getLeft(), mergedMinMax.getLeft(), comp));
        Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getRight(), mergedMinMax.getRight(), comp));
        orderedIndexTestUtils.checkPointSearches(ctx);
        orderedIndexTestUtils.checkScan(ctx);
        orderedIndexTestUtils.checkDiskOrderScan(ctx);
        orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
        if (prefixLowKey != null && prefixHighKey != null) {
            orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
        }
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}

Example 3 with IntegerSerializerDeserializer

use of org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer in project asterixdb by apache.

the class LSMBTreeMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
    OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType, false);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
    } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_BTREE_MAX_TREES_TO_MERGE;
    for (int i = 0; i < maxTreesToMerge; i++) {
        for (int j = 0; j < i; j++) {
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
                orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            }
        }
        ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((LSMBTree) ctx.getIndex()).getImmutableComponents());
        orderedIndexTestUtils.checkPointSearches(ctx);
        orderedIndexTestUtils.checkScan(ctx);
        orderedIndexTestUtils.checkDiskOrderScan(ctx);
        orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
        if (prefixLowKey != null && prefixHighKey != null) {
            orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
        }
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}

Example 4 with IntegerSerializerDeserializer

use of org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer in project asterixdb by apache.

the class LSMRTreeMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key, RTreePolicyType rtreePolicyType) throws Exception {
    AbstractRTreeTestContext ctx = createTestContext(fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
    } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
        rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_RTREE_BULKLOAD_ROUNDS;
    for (int i = 0; i < maxTreesToMerge; i++) {
        for (int j = 0; j < i; j++) {
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                rTreeTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
                rTreeTestUtils.insertDoubleTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            }
        }
        ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((AbstractLSMRTree) ctx.getIndex()).getImmutableComponents());
        rTreeTestUtils.checkScan(ctx);
        rTreeTestUtils.checkDiskOrderScan(ctx);
        rTreeTestUtils.checkRangeSearch(ctx, key);
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}