Examples with ISerializerDeserializer org.apache.hyracks.api.dataflow.value.ISerializerDeserializer used on opensource projects

Example 76 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testMessageFitsWithTuples.

@Test
public void testMessageFitsWithTuples() throws Exception {
    try {
        // Routing will be round robin
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        FieldType[] types = { FieldType.Integer64, FieldType.Double, FieldType.Boolean, FieldType.String };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        PartitionWithMessageDataWriter partitioner = (PartitionWithMessageDataWriter) connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        partitioner.open();
        FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
        List<TestFrameWriter> recipients = new ArrayList<>();
        for (int i = 0; i < partitionWriterFactory.getWriters().values().size(); i++) {
            recipients.add(partitionWriterFactory.getWriters().get(i));
        }
        TestTupleGenerator ttg = new TestTupleGenerator(types, STRING_FIELD_SIZES, true);
        VSizeFrame frame = new VSizeFrame(ctx);
        FrameTupleAppender appender = new FrameTupleAppender(frame);
        for (int count = 0; count < NUMBER_OF_CONSUMERS; count++) {
            ITupleReference tuple = ttg.next();
            appender.append(tuple);
        }
        partitioner.nextFrame(frame.getBuffer());
        partitioner.flush();
        Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(1).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(2).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(3).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(4).nextFrameCount(), 1);
        for (TestFrameWriter writer : recipients) {
            fta.reset(writer.getLastFrame());
            Assert.assertEquals(fta.getTupleCount(), 2);
            FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
            Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
        }
        partitioner.close();
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}

Example 77 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AInt8ConstructorDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable inputArg = new VoidPointable();

                private IScalarEvaluator eval = args[0].createScalarEvaluator(ctx);

                private byte value;

                private int offset;

                private boolean positive;

                private AMutableInt8 aInt8 = new AMutableInt8((byte) 0);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInt8> int8Serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT8);

                private final UTF8StringPointable utf8Ptr = new UTF8StringPointable();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    try {
                        resultStorage.reset();
                        eval.evaluate(tuple, inputArg);
                        byte[] serString = inputArg.getByteArray();
                        int startOffset = inputArg.getStartOffset();
                        int len = inputArg.getLength();
                        if (serString[startOffset] == ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
                            utf8Ptr.set(serString, startOffset + 1, len - 1);
                            offset = utf8Ptr.getCharStartOffset();
                            //accumulating value in negative domain
                            //otherwise Byte.MIN_VALUE = -(Byte.MAX_VALUE + 1) would have caused overflow
                            value = 0;
                            positive = true;
                            byte limit = -Byte.MAX_VALUE;
                            if (serString[offset] == '+') {
                                offset++;
                            } else if (serString[offset] == '-') {
                                offset++;
                                positive = false;
                                limit = Byte.MIN_VALUE;
                            }
                            int end = startOffset + len;
                            for (; offset < end; offset++) {
                                int digit;
                                if (serString[offset] >= '0' && serString[offset] <= '9') {
                                    value = (byte) (value * 10);
                                    digit = serString[offset] - '0';
                                } else if (serString[offset] == 'i' && serString[offset + 1] == '8' && offset + 2 == end) {
                                    break;
                                } else {
                                    throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INT8_TYPE_TAG);
                                }
                                if (value < limit + digit) {
                                    throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INT8_TYPE_TAG);
                                }
                                value = (byte) (value - digit);
                            }
                            if (value > 0) {
                                throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INT8_TYPE_TAG);
                            }
                            if (value < 0 && positive) {
                                value *= -1;
                            }
                            aInt8.setValue(value);
                            int8Serde.serialize(aInt8, out);
                        } else {
                            throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INT8_TYPE_TAG);
                        }
                        result.set(resultStorage);
                    } catch (IOException e1) {
                        throw new InvalidDataFormatException(getIdentifier(), e1, ATypeTag.SERIALIZED_INT8_TYPE_TAG);
                    }
                }
            };
        }
    };
}

Example 78 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AIntervalConstructorDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr0 = new VoidPointable();

                private IPointable argPtr1 = new VoidPointable();

                private IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);

                private IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);

                private AMutableInterval aInterval = new AMutableInterval(0L, 0L, (byte) 0);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInterval> intervalSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINTERVAL);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval0.evaluate(tuple, argPtr0);
                    eval1.evaluate(tuple, argPtr1);
                    byte[] bytes0 = argPtr0.getByteArray();
                    int offset0 = argPtr0.getStartOffset();
                    byte[] bytes1 = argPtr1.getByteArray();
                    int offset1 = argPtr1.getStartOffset();
                    try {
                        if (bytes0[offset0] != bytes1[offset1]) {
                            throw new IncompatibleTypeException(getIdentifier(), bytes0[offset0], bytes1[offset1]);
                        }
                        long intervalStart, intervalEnd;
                        ATypeTag intervalType = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes0[offset0]);
                        switch(intervalType) {
                            case DATE:
                                intervalStart = ADateSerializerDeserializer.getChronon(bytes0, offset0 + 1);
                                intervalEnd = ADateSerializerDeserializer.getChronon(bytes1, offset1 + 1);
                                break;
                            case TIME:
                                intervalStart = ATimeSerializerDeserializer.getChronon(bytes0, offset0 + 1);
                                intervalEnd = ATimeSerializerDeserializer.getChronon(bytes1, offset1 + 1);
                                break;
                            case DATETIME:
                                intervalStart = ADateTimeSerializerDeserializer.getChronon(bytes0, offset0 + 1);
                                intervalEnd = ADateTimeSerializerDeserializer.getChronon(bytes1, offset1 + 1);
                                break;
                            default:
                                throw new UnsupportedItemTypeException(getIdentifier(), bytes0[offset0]);
                        }
                        if (intervalEnd < intervalStart) {
                            throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                        }
                        aInterval.setValue(intervalStart, intervalEnd, intervalType.serialize());
                        intervalSerde.serialize(aInterval, out);
                    } catch (IOException e) {
                        throw new InvalidDataFormatException(getIdentifier(), e, ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                    }
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 79 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AIntervalStartFromDateConstructorDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr0 = new VoidPointable();

                private IPointable argPtr1 = new VoidPointable();

                private IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);

                private IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);

                private AMutableInterval aInterval = new AMutableInterval(0L, 0L, (byte) 0);

                private AMutableDuration aDuration = new AMutableDuration(0, 0L);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInterval> intervalSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINTERVAL);

                private final UTF8StringPointable utf8Ptr = new UTF8StringPointable();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval0.evaluate(tuple, argPtr0);
                    eval1.evaluate(tuple, argPtr1);
                    byte[] bytes0 = argPtr0.getByteArray();
                    int offset0 = argPtr0.getStartOffset();
                    int len0 = argPtr0.getLength();
                    byte[] bytes1 = argPtr1.getByteArray();
                    int offset1 = argPtr1.getStartOffset();
                    int len1 = argPtr1.getLength();
                    try {
                        long intervalStart = 0, intervalEnd = 0;
                        if (bytes0[offset0] == ATypeTag.SERIALIZED_DATE_TYPE_TAG) {
                            intervalStart = ADateSerializerDeserializer.getChronon(bytes0, offset0 + 1) * GregorianCalendarSystem.CHRONON_OF_DAY;
                        } else if (bytes0[offset0] == ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
                            utf8Ptr.set(bytes0, offset0 + 1, len0 - 1);
                            // start date
                            int stringLength = utf8Ptr.getUTF8Length();
                            int startOffset = utf8Ptr.getCharStartOffset();
                            intervalStart = ADateParserFactory.parseDatePart(bytes0, startOffset, stringLength);
                        } else {
                            throw new TypeMismatchException(getIdentifier(), 0, bytes0[offset0], ATypeTag.SERIALIZED_DATE_TYPE_TAG, ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                        }
                        if (bytes1[offset1] == ATypeTag.SERIALIZED_DURATION_TYPE_TAG) {
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, ADurationSerializerDeserializer.getYearMonth(bytes1, offset1 + 1), ADurationSerializerDeserializer.getDayTime(bytes1, offset1 + 1), false);
                        } else if (bytes1[offset1] == ATypeTag.SERIALIZED_DAY_TIME_DURATION_TYPE_TAG) {
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, 0, ADayTimeDurationSerializerDeserializer.getDayTime(bytes1, offset1 + 1), false);
                        } else if (bytes1[offset1] == ATypeTag.SERIALIZED_YEAR_MONTH_DURATION_TYPE_TAG) {
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, AYearMonthDurationSerializerDeserializer.getYearMonth(bytes1, offset1 + 1), 0, false);
                        } else if (bytes1[offset1] == ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
                            // duration
                            utf8Ptr.set(bytes1, offset1 + 1, len1 - 1);
                            int stringLength = utf8Ptr.getUTF8Length();
                            ADurationParserFactory.parseDuration(bytes1, utf8Ptr.getCharStartOffset(), stringLength, aDuration, ADurationParseOption.All);
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, aDuration.getMonths(), aDuration.getMilliseconds(), false);
                        } else {
                            throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1], ATypeTag.SERIALIZED_DATE_TYPE_TAG, ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                        }
                        intervalStart = GregorianCalendarSystem.getChrononInDays(intervalStart);
                        intervalEnd = GregorianCalendarSystem.getChrononInDays(intervalEnd);
                        if (intervalEnd < intervalStart) {
                            throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                        }
                        aInterval.setValue(intervalStart, intervalEnd, ATypeTag.SERIALIZED_DATE_TYPE_TAG);
                        intervalSerde.serialize(aInterval, out);
                    } catch (IOException e) {
                        throw new InvalidDataFormatException(getIdentifier(), e, ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                    }
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 80 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AIntervalStartFromTimeConstructorDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr0 = new VoidPointable();

                private IPointable argPtr1 = new VoidPointable();

                private IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);

                private IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);

                private AMutableInterval aInterval = new AMutableInterval(0L, 0L, (byte) 0);

                private AMutableDuration aDuration = new AMutableDuration(0, 0L);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInterval> intervalSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINTERVAL);

                private final UTF8StringPointable utf8Ptr = new UTF8StringPointable();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval0.evaluate(tuple, argPtr0);
                    eval1.evaluate(tuple, argPtr1);
                    byte[] bytes0 = argPtr0.getByteArray();
                    int offset0 = argPtr0.getStartOffset();
                    int len0 = argPtr0.getLength();
                    byte[] bytes1 = argPtr1.getByteArray();
                    int offset1 = argPtr1.getStartOffset();
                    int len1 = argPtr1.getLength();
                    try {
                        long intervalStart = 0, intervalEnd = 0;
                        if (bytes0[offset0] == ATypeTag.SERIALIZED_TIME_TYPE_TAG) {
                            intervalStart = ATimeSerializerDeserializer.getChronon(bytes0, offset0 + 1);
                        } else if (bytes0[offset0] == ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
                            utf8Ptr.set(bytes0, offset0 + 1, len0 - 1);
                            int stringLength = utf8Ptr.getUTF8Length();
                            intervalStart = ATimeParserFactory.parseTimePart(bytes0, utf8Ptr.getCharStartOffset(), stringLength);
                        } else {
                            throw new TypeMismatchException(getIdentifier(), 0, bytes0[offset0], ATypeTag.SERIALIZED_TIME_TYPE_TAG, ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                        }
                        if (intervalStart < 0) {
                            intervalStart += GregorianCalendarSystem.CHRONON_OF_DAY;
                        }
                        if (bytes1[offset1] == ATypeTag.SERIALIZED_DURATION_TYPE_TAG) {
                            if (ADurationSerializerDeserializer.getYearMonth(bytes1, offset1 + 1) != 0) {
                                throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                            }
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, 0, ADurationSerializerDeserializer.getDayTime(bytes1, offset1 + 1), false);
                        } else if (bytes1[offset1] == ATypeTag.SERIALIZED_DAY_TIME_DURATION_TYPE_TAG) {
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, 0, ADayTimeDurationSerializerDeserializer.getDayTime(bytes1, offset1 + 1), false);
                        } else if (bytes1[offset1] == ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
                            // duration
                            utf8Ptr.set(bytes1, offset1 + 1, len1 - 1);
                            int stringLength = utf8Ptr.getUTF8Length();
                            ADurationParserFactory.parseDuration(bytes1, utf8Ptr.getCharStartOffset(), stringLength, aDuration, ADurationParseOption.All);
                            if (aDuration.getMonths() != 0) {
                                throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                            }
                            intervalEnd = DurationArithmeticOperations.addDuration(intervalStart, 0, aDuration.getMilliseconds(), false);
                        } else {
                            throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1], ATypeTag.SERIALIZED_DURATION_TYPE_TAG, ATypeTag.SERIALIZED_DAY_TIME_DURATION_TYPE_TAG, ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                        }
                        if (intervalEnd > GregorianCalendarSystem.CHRONON_OF_DAY) {
                            intervalEnd = intervalEnd % (int) (GregorianCalendarSystem.CHRONON_OF_DAY);
                        }
                        if (intervalEnd < intervalStart) {
                            throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                        }
                        aInterval.setValue(intervalStart, intervalEnd, ATypeTag.SERIALIZED_TIME_TYPE_TAG);
                        intervalSerde.serialize(aInterval, out);
                    } catch (IOException e) {
                        throw new InvalidDataFormatException(getIdentifier(), e, ATypeTag.SERIALIZED_INTERVAL_TYPE_TAG);
                    }
                    result.set(resultStorage);
                }
            };
        }
    };
}