Examples with VSizeFrame org.apache.hyracks.api.comm.VSizeFrame used on opensource projects

Example 1 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class DataGenOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
    final RecordDescriptor recDesc = outRecDescs[0];
    final ArrayTupleBuilder tb = new ArrayTupleBuilder(recDesc.getFields().length);
    final Random rnd = new Random(randomSeed);
    final int maxUniqueAttempts = 20;
    return new AbstractUnaryOutputSourceOperatorNodePushable() {

        // for quick & dirty exclusion of duplicates
        // WARNING: could contain numRecord entries and use a lot of memory
        HashSet<String> stringHs = new HashSet<String>();

        HashSet<Integer> intHs = new HashSet<Integer>();

        @Override
        public void initialize() throws HyracksDataException {
            try {
                writer.open();
                for (int i = 0; i < numRecords; i++) {
                    tb.reset();
                    for (int j = 0; j < recDesc.getFieldCount(); j++) {
                        genField(tb, j);
                    }
                    if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                        appender.write(writer, true);
                        if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                            throw new HyracksDataException("Record size (" + tb.getSize() + ") larger than frame size (" + appender.getBuffer().capacity() + ")");
                        }
                    }
                }
                appender.write(writer, true);
            } catch (Throwable th) {
                writer.fail();
                throw new HyracksDataException(th);
            } finally {
                writer.close();
            }
        }

        private void genField(ArrayTupleBuilder tb, int fieldIndex) throws HyracksDataException {
            DataOutput dos = tb.getDataOutput();
            if (recDesc.getFields()[fieldIndex] instanceof IntegerSerializerDeserializer) {
                int val = -1;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        int tmp = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                        if (intHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            intHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else if (recDesc.getFields()[fieldIndex] instanceof UTF8StringSerializerDeserializer) {
                String val = null;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        String tmp = randomString(maxStrLen, rnd);
                        if (stringHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            stringHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = randomString(maxStrLen, rnd);
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else {
                throw new HyracksDataException("Type unsupported in data generator. Only integers and strings allowed");
            }
        }

        private String randomString(int length, Random random) {
            String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
            StringBuilder strBuilder = new StringBuilder();
            for (int i = 0; i < s.length() && i < length; i++) {
                strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
            }
            return strBuilder.toString();
        }
    };
}

Example 2 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FeedFrameUtil method removeBadTuple.

public static ByteBuffer removeBadTuple(IHyracksTaskContext ctx, int tupleIndex, FrameTupleAccessor fta) throws HyracksDataException {
    FrameTupleAppender appender = new FrameTupleAppender();
    IFrame slicedFrame = new VSizeFrame(ctx);
    appender.reset(slicedFrame, true);
    int totalTuples = fta.getTupleCount();
    for (int ti = 0; ti < totalTuples; ti++) {
        if (ti != tupleIndex) {
            appender.append(fta, ti);
        }
    }
    return slicedFrame.getBuffer();
}

Example 3 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class CounterTimerTupleForwarder method initialize.

@Override
public void initialize(IHyracksTaskContext ctx, IFrameWriter writer) throws HyracksDataException {
    this.appender = new FrameTupleAppender();
    this.frame = new VSizeFrame(ctx);
    appender.reset(frame, true);
    this.writer = writer;
    if (activeTimer) {
        this.timer = new Timer();
        this.flushTask = new TimeBasedFlushTask(writer, lock);
        timer.scheduleAtFixedRate(flushTask, 0, batchInterval);
    }
}

Example 4 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FeedSpillerUnitTest method testWriteFixedSizeSpill.

/*
     * Spiller spills each 1024 frames to a file
     */
/*
     * The following tests:
     * 1. Test writer only.
     * Write 1023 frames.
     * Check only 1 file exist.
     * Write 1 more frame
     * Check two files exist.
     * Insert 1023 more frames.
     * Check that we still have 2 files.
     * Write 1 more frame.
     * Check that we have 3 files.
     * Check that we have 2048 frames to read.
     * Close the spiller
     * Check files were deleted.
     */
@org.junit.Test
public void testWriteFixedSizeSpill() {
    try {
        removeSpillFiles();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        FrameSpiller spiller = new FrameSpiller(ctx, TEST_DATAVERSE + "_" + TEST_FEED + "_" + TEST_DATASET, new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE));
        spiller.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        for (int i = 0; i < 1022; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(1023, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1024, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        for (int i = 0; i < 1023; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(2047, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(2048, spiller.remaining());
        Assert.assertEquals(3, countSpillFiles());
        spiller.close();
        Assert.assertEquals(0, spiller.remaining());
        Assert.assertEquals(0, countSpillFiles());
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail(th.getMessage());
    }
}

Example 5 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Slow next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        writer.setNextDuration(1);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that nextFrame was called
        writer.validate(false);
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}