Examples with FrameTupleAppender org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender used on opensource projects

Example 1 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class DataGenOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
    final RecordDescriptor recDesc = outRecDescs[0];
    final ArrayTupleBuilder tb = new ArrayTupleBuilder(recDesc.getFields().length);
    final Random rnd = new Random(randomSeed);
    final int maxUniqueAttempts = 20;
    return new AbstractUnaryOutputSourceOperatorNodePushable() {

        // for quick & dirty exclusion of duplicates
        // WARNING: could contain numRecord entries and use a lot of memory
        HashSet<String> stringHs = new HashSet<String>();

        HashSet<Integer> intHs = new HashSet<Integer>();

        @Override
        public void initialize() throws HyracksDataException {
            try {
                writer.open();
                for (int i = 0; i < numRecords; i++) {
                    tb.reset();
                    for (int j = 0; j < recDesc.getFieldCount(); j++) {
                        genField(tb, j);
                    }
                    if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                        appender.write(writer, true);
                        if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                            throw new HyracksDataException("Record size (" + tb.getSize() + ") larger than frame size (" + appender.getBuffer().capacity() + ")");
                        }
                    }
                }
                appender.write(writer, true);
            } catch (Throwable th) {
                writer.fail();
                throw new HyracksDataException(th);
            } finally {
                writer.close();
            }
        }

        private void genField(ArrayTupleBuilder tb, int fieldIndex) throws HyracksDataException {
            DataOutput dos = tb.getDataOutput();
            if (recDesc.getFields()[fieldIndex] instanceof IntegerSerializerDeserializer) {
                int val = -1;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        int tmp = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                        if (intHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            intHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else if (recDesc.getFields()[fieldIndex] instanceof UTF8StringSerializerDeserializer) {
                String val = null;
                if (fieldIndex == uniqueField) {
                    int attempt = 0;
                    while (attempt < maxUniqueAttempts) {
                        String tmp = randomString(maxStrLen, rnd);
                        if (stringHs.contains(tmp))
                            attempt++;
                        else {
                            val = tmp;
                            stringHs.add(val);
                            break;
                        }
                    }
                    if (attempt == maxUniqueAttempts)
                        throw new HyracksDataException("MaxUnique attempts reached in datagen");
                } else {
                    val = randomString(maxStrLen, rnd);
                }
                recDesc.getFields()[fieldIndex].serialize(val, dos);
                tb.addFieldEndOffset();
            } else {
                throw new HyracksDataException("Type unsupported in data generator. Only integers and strings allowed");
            }
        }

        private String randomString(int length, Random random) {
            String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
            StringBuilder strBuilder = new StringBuilder();
            for (int i = 0; i < s.length() && i < length; i++) {
                strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
            }
            return strBuilder.toString();
        }
    };
}

Example 2 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class AbstractTupleMemoryManagerTest method prepareVariableSizeTuples.

protected Map<Integer, Integer> prepareVariableSizeTuples() throws HyracksDataException {
    Map<Integer, Integer> dataSet = new HashMap<>();
    ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
    FixedSizeFrame frame = new FixedSizeFrame(buffer);
    FrameTupleAppender appender = new FrameTupleAppender();
    appender.reset(frame, true);
    for (int i = 0; true; i++) {
        tupleBuilder.reset();
        tupleBuilder.addField(fieldsSerDer[0], i);
        tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', i));
        if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
            break;
        }
        dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
    }
    inFTA.reset(buffer);
    return dataSet;
}

Example 3 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class VariableTupleMemoryManagerTest method deleteRandomSelectedTuples.

private ByteBuffer deleteRandomSelectedTuples(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted, int minNumOfRecordTobeDeleted) throws HyracksDataException {
    ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
    FixedSizeFrame frame = new FixedSizeFrame(buffer);
    FrameTupleAppender appender = new FrameTupleAppender();
    appender.reset(frame, true);
    assert (minNumOfRecordTobeDeleted < mapInserted.size());
    int countDeleted = minNumOfRecordTobeDeleted + random.nextInt(mapInserted.size() - minNumOfRecordTobeDeleted);
    ITuplePointerAccessor accessor = tupleMemoryManager.createTuplePointerAccessor();
    for (int i = 0; i < countDeleted; i++) {
        Iterator<Map.Entry<TuplePointer, Integer>> iter = mapInserted.entrySet().iterator();
        assert (iter.hasNext());
        Map.Entry<TuplePointer, Integer> pair = iter.next();
        accessor.reset(pair.getKey());
        appender.append(accessor.getBuffer().array(), accessor.getTupleStartOffset(), accessor.getTupleLength());
        map.remove(pair.getValue());
        tupleMemoryManager.deleteTuple(pair.getKey());
        iter.remove();
    }
    return buffer;
}

Example 4 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method prepareRandomTuples.

private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
    frameList.clear();
    HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
    int[] fieldSlot = { 0 };
    int id = 0;
    int size = 0;
    while (size < BUDGET) {
        int tupleLength = random.nextInt(BUDGET / 3) + 4;
        IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
        IFrameTupleAppender appender = new FrameTupleAppender();
        appender.reset(frame, true);
        //            System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
        ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
        buffer.putInt(0, id);
        assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
        set.put(id++, tupleLength);
        size += frame.getFrameSize();
        frameList.add(frame);
    }
    return set;
}

Example 5 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method prepareTuples.

private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
    frameList.clear();
    HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
    for (int i = 0; i < NUM_MIN_FRAME; ++i) {
        IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
        IFrameTupleAppender appender = new FrameTupleAppender();
        appender.reset(frame, true);
        int[] fieldSlot = { 0 };
        ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
        buffer.putInt(0, i);
        appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
        set.put(i, buffer.capacity());
        frameList.add(frame);
    }
    return set;
}