Examples with IFrameTupleReference org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference used on opensource projects

Example 96 with IFrameTupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference in project asterixdb by apache.

the class IntegerEqFunctionEvaluatorFactory method createScalarEvaluator.

@Override
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
    return new IScalarEvaluator() {

        private IPointable out1 = new VoidPointable();

        private IPointable out2 = new VoidPointable();

        private IScalarEvaluator eval1 = arg1Factory.createScalarEvaluator(ctx);

        private IScalarEvaluator eval2 = arg2Factory.createScalarEvaluator(ctx);

        private byte[] resultData = new byte[1];

        @Override
        public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
            eval1.evaluate(tuple, out1);
            eval2.evaluate(tuple, out2);
            int v1 = IntegerPointable.getInteger(out1.getByteArray(), out1.getStartOffset());
            int v2 = IntegerPointable.getInteger(out2.getByteArray(), out2.getStartOffset());
            boolean r = v1 == v2;
            resultData[0] = r ? (byte) 1 : (byte) 0;
            result.set(resultData, 0, 1);
        }
    };
}

Example 97 with IFrameTupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference in project asterixdb by apache.

the class RecordMergeDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            final PointableAllocator pa = new PointableAllocator();
            final IVisitablePointable vp0 = pa.allocateRecordValue(inRecType0);
            final IVisitablePointable vp1 = pa.allocateRecordValue(inRecType1);
            final IPointable argPtr0 = new VoidPointable();
            final IPointable argPtr1 = new VoidPointable();
            final IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);
            final IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);
            final List<RecordBuilder> rbStack = new ArrayList<>();
            final ArrayBackedValueStorage tabvs = new ArrayBackedValueStorage();
            return new IScalarEvaluator() {

                private final RuntimeRecordTypeInfo runtimeRecordTypeInfo = new RuntimeRecordTypeInfo();

                private final DeepEqualAssessor deepEqualAssesor = new DeepEqualAssessor();

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval0.evaluate(tuple, argPtr0);
                    eval1.evaluate(tuple, argPtr1);
                    vp0.set(argPtr0);
                    vp1.set(argPtr1);
                    ARecordVisitablePointable rp0 = (ARecordVisitablePointable) vp0;
                    ARecordVisitablePointable rp1 = (ARecordVisitablePointable) vp1;
                    try {
                        mergeFields(outRecType, rp0, rp1, true, 0);
                        rbStack.get(0).write(out, true);
                    } catch (IOException | AsterixException e) {
                        throw new HyracksDataException(e);
                    }
                    result.set(resultStorage);
                }

                private void mergeFields(ARecordType combinedType, ARecordVisitablePointable leftRecord, ARecordVisitablePointable rightRecord, boolean openFromParent, int nestedLevel) throws IOException, AsterixException, HyracksDataException {
                    if (rbStack.size() < (nestedLevel + 1)) {
                        rbStack.add(new RecordBuilder());
                    }
                    rbStack.get(nestedLevel).reset(combinedType);
                    rbStack.get(nestedLevel).init();
                    //Add all fields from left record
                    for (int i = 0; i < leftRecord.getFieldNames().size(); i++) {
                        IVisitablePointable leftName = leftRecord.getFieldNames().get(i);
                        IVisitablePointable leftValue = leftRecord.getFieldValues().get(i);
                        IVisitablePointable leftType = leftRecord.getFieldTypeTags().get(i);
                        boolean foundMatch = false;
                        for (int j = 0; j < rightRecord.getFieldNames().size(); j++) {
                            IVisitablePointable rightName = rightRecord.getFieldNames().get(j);
                            IVisitablePointable rightValue = rightRecord.getFieldValues().get(j);
                            IVisitablePointable rightType = rightRecord.getFieldTypeTags().get(j);
                            // Check if same fieldname
                            if (PointableHelper.isEqual(leftName, rightName) && !deepEqualAssesor.isEqual(leftValue, rightValue)) {
                                //Field was found on the right and are subrecords, merge them
                                if (PointableHelper.sameType(ATypeTag.OBJECT, rightType) && PointableHelper.sameType(ATypeTag.OBJECT, leftType)) {
                                    //We are merging two sub records
                                    addFieldToSubRecord(combinedType, leftName, leftValue, rightValue, openFromParent, nestedLevel);
                                    foundMatch = true;
                                } else {
                                    throw new RuntimeDataException(ErrorCode.DUPLICATE_FIELD_NAME, getIdentifier());
                                }
                            }
                        }
                        if (!foundMatch) {
                            addFieldToSubRecord(combinedType, leftName, leftValue, null, openFromParent, nestedLevel);
                        }
                    }
                    //Repeat for right side (ignoring duplicates this time)
                    for (int j = 0; j < rightRecord.getFieldNames().size(); j++) {
                        IVisitablePointable rightName = rightRecord.getFieldNames().get(j);
                        IVisitablePointable rightValue = rightRecord.getFieldValues().get(j);
                        boolean foundMatch = false;
                        for (int i = 0; i < leftRecord.getFieldNames().size(); i++) {
                            IVisitablePointable leftName = leftRecord.getFieldNames().get(i);
                            if (rightName.equals(leftName)) {
                                foundMatch = true;
                            }
                        }
                        if (!foundMatch) {
                            addFieldToSubRecord(combinedType, rightName, rightValue, null, openFromParent, nestedLevel);
                        }
                    }
                }

                /*
                     * Takes in a record type, field name, and the field values (which are record) from two records
                     * Merges them into one record of combinedType
                     * And adds that record as a field to the Record in subrb
                     * the second value can be null, indicated that you just add the value of left as a field to subrb
                     *
                     */
                private void addFieldToSubRecord(ARecordType combinedType, IVisitablePointable fieldNamePointable, IVisitablePointable leftValue, IVisitablePointable rightValue, boolean openFromParent, int nestedLevel) throws IOException, AsterixException, HyracksDataException {
                    runtimeRecordTypeInfo.reset(combinedType);
                    int pos = runtimeRecordTypeInfo.getFieldIndex(fieldNamePointable.getByteArray(), fieldNamePointable.getStartOffset() + 1, fieldNamePointable.getLength() - 1);
                    //Add the merged field
                    if (combinedType != null && pos >= 0) {
                        if (rightValue == null) {
                            rbStack.get(nestedLevel).addField(pos, leftValue);
                        } else {
                            mergeFields((ARecordType) combinedType.getFieldTypes()[pos], (ARecordVisitablePointable) leftValue, (ARecordVisitablePointable) rightValue, false, nestedLevel + 1);
                            tabvs.reset();
                            rbStack.get(nestedLevel + 1).write(tabvs.getDataOutput(), true);
                            rbStack.get(nestedLevel).addField(pos, tabvs);
                        }
                    } else {
                        if (rightValue == null) {
                            rbStack.get(nestedLevel).addField(fieldNamePointable, leftValue);
                        } else {
                            mergeFields(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, (ARecordVisitablePointable) leftValue, (ARecordVisitablePointable) rightValue, false, nestedLevel + 1);
                            tabvs.reset();
                            rbStack.get(nestedLevel + 1).write(tabvs.getDataOutput(), true);
                            rbStack.get(nestedLevel).addField(fieldNamePointable, tabvs);
                        }
                    }
                }
            };
        }
    };
}

Example 98 with IFrameTupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference in project asterixdb by apache.

the class RecordPairsDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            // For writing each individual output record.
            final ArrayBackedValueStorage itemStorage = new ArrayBackedValueStorage();
            final DataOutput itemOutput = itemStorage.getDataOutput();
            final RecordBuilder recBuilder = new RecordBuilder();
            recBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
            // For writing the resulting list of records.
            final OrderedListBuilder listBuilder = new OrderedListBuilder();
            final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
            final DataOutput resultOut = resultStorage.getDataOutput();
            // Sets up the constant field names, "name" for the key field, "value" for the value field.
            final ArrayBackedValueStorage nameStorage = new ArrayBackedValueStorage();
            final ArrayBackedValueStorage valueStorage = new ArrayBackedValueStorage();
            AObjectSerializerDeserializer serde = AObjectSerializerDeserializer.INSTANCE;
            try {
                serde.serialize(new AString("name"), nameStorage.getDataOutput());
                serde.serialize(new AString("value"), valueStorage.getDataOutput());
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
            return new IScalarEvaluator() {

                private final IScalarEvaluator argEvaluator = args[0].createScalarEvaluator(ctx);

                private final IPointable argPtr = new VoidPointable();

                private final ARecordVisitablePointable recordVisitablePointable = new ARecordVisitablePointable(recType);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    // Resets the result storage.
                    resultStorage.reset();
                    // Gets the input record.
                    argEvaluator.evaluate(tuple, argPtr);
                    byte inputTypeTag = argPtr.getByteArray()[argPtr.getStartOffset()];
                    if (inputTypeTag != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                        throw new TypeMismatchException(getIdentifier(), 0, inputTypeTag, ATypeTag.SERIALIZED_RECORD_TYPE_TAG);
                    }
                    recordVisitablePointable.set(argPtr);
                    listBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
                    List<IVisitablePointable> fieldNames = recordVisitablePointable.getFieldNames();
                    List<IVisitablePointable> fieldValues = recordVisitablePointable.getFieldValues();
                    // Adds each field of the input record as a key-value pair into the result.
                    int numFields = recordVisitablePointable.getFieldNames().size();
                    for (int fieldIndex = 0; fieldIndex < numFields; ++fieldIndex) {
                        itemStorage.reset();
                        recBuilder.init();
                        recBuilder.addField(nameStorage, fieldNames.get(fieldIndex));
                        recBuilder.addField(valueStorage, fieldValues.get(fieldIndex));
                        recBuilder.write(itemOutput, true);
                        listBuilder.addItem(itemStorage);
                    }
                    // Writes the result and sets the result pointable.
                    listBuilder.write(resultOut, true);
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 99 with IFrameTupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference in project asterixdb by apache.

the class RecordRemoveFieldsEvalFactory method createScalarEvaluator.

@Override
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
    final PointableAllocator pa = new PointableAllocator();
    final IVisitablePointable vp0 = pa.allocateRecordValue(inputRecType);
    final IVisitablePointable vp1 = pa.allocateListValue(inputListType);
    final IPointable inputArg0 = new VoidPointable();
    final IPointable inputArg1 = new VoidPointable();
    final IScalarEvaluator eval0 = inputRecordEvalFactory.createScalarEvaluator(ctx);
    final IScalarEvaluator eval1 = removeFieldPathsFactory.createScalarEvaluator(ctx);
    return new IScalarEvaluator() {

        private final RuntimeRecordTypeInfo runtimeRecordTypeInfo = new RuntimeRecordTypeInfo();

        private final List<RecordBuilder> rbStack = new ArrayList<>();

        private final ArrayBackedValueStorage tabvs = new ArrayBackedValueStorage();

        private final Deque<IVisitablePointable> recordPath = new ArrayDeque<>();

        private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

        private DataOutput out = resultStorage.getDataOutput();

        @Override
        public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
            resultStorage.reset();
            eval0.evaluate(tuple, inputArg0);
            eval1.evaluate(tuple, inputArg1);
            byte inputTypeTag0 = inputArg0.getByteArray()[inputArg0.getStartOffset()];
            if (inputTypeTag0 != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                throw new TypeMismatchException(BuiltinFunctions.REMOVE_FIELDS, 0, inputTypeTag0, ATypeTag.SERIALIZED_INT32_TYPE_TAG);
            }
            byte inputTypeTag1 = inputArg1.getByteArray()[inputArg1.getStartOffset()];
            if (inputTypeTag1 != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
                throw new TypeMismatchException(BuiltinFunctions.REMOVE_FIELDS, 1, inputTypeTag1, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
            }
            vp0.set(inputArg0);
            vp1.set(inputArg1);
            ARecordVisitablePointable recordPointable = (ARecordVisitablePointable) vp0;
            AListVisitablePointable listPointable = (AListVisitablePointable) vp1;
            try {
                recordPath.clear();
                rbStack.clear();
                processRecord(requiredRecType, recordPointable, listPointable, 0);
                rbStack.get(0).write(out, true);
            } catch (IOException | AsterixException e) {
                throw new HyracksDataException(e);
            }
            result.set(resultStorage);
        }

        private void processRecord(ARecordType requiredType, ARecordVisitablePointable srp, AListVisitablePointable inputList, int nestedLevel) throws IOException, AsterixException, HyracksDataException {
            if (rbStack.size() < (nestedLevel + 1)) {
                rbStack.add(new RecordBuilder());
            }
            rbStack.get(nestedLevel).reset(requiredType);
            rbStack.get(nestedLevel).init();
            List<IVisitablePointable> fieldNames = srp.getFieldNames();
            List<IVisitablePointable> fieldValues = srp.getFieldValues();
            List<IVisitablePointable> fieldTypes = srp.getFieldTypeTags();
            for (int i = 0; i < fieldNames.size(); i++) {
                IVisitablePointable subRecFieldName = fieldNames.get(i);
                recordPath.push(subRecFieldName);
                if (isValidPath(inputList)) {
                    if (requiredType != null && requiredType.getTypeTag() != ATypeTag.ANY) {
                        addKeptFieldToSubRecord(requiredType, subRecFieldName, fieldValues.get(i), fieldTypes.get(i), inputList, nestedLevel);
                    } else {
                        addKeptFieldToSubRecord(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, subRecFieldName, fieldValues.get(i), fieldTypes.get(i), inputList, nestedLevel);
                    }
                }
                recordPath.pop();
            }
        }

        private void addKeptFieldToSubRecord(ARecordType requiredType, IVisitablePointable fieldNamePointable, IVisitablePointable fieldValuePointable, IVisitablePointable fieldTypePointable, AListVisitablePointable inputList, int nestedLevel) throws IOException, AsterixException, HyracksDataException {
            runtimeRecordTypeInfo.reset(requiredType);
            int pos = runtimeRecordTypeInfo.getFieldIndex(fieldNamePointable.getByteArray(), fieldNamePointable.getStartOffset() + 1, fieldNamePointable.getLength() - 1);
            if (pos >= 0) {
                // Closed field
                if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable)) {
                    processRecord((ARecordType) requiredType.getFieldTypes()[pos], (ARecordVisitablePointable) fieldValuePointable, inputList, nestedLevel + 1);
                    tabvs.reset();
                    rbStack.get(nestedLevel + 1).write(tabvs.getDataOutput(), true);
                    rbStack.get(nestedLevel).addField(pos, tabvs);
                } else {
                    rbStack.get(nestedLevel).addField(pos, fieldValuePointable);
                }
            } else {
                // Open field
                if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable)) {
                    processRecord(null, (ARecordVisitablePointable) fieldValuePointable, inputList, nestedLevel + 1);
                    tabvs.reset();
                    rbStack.get(nestedLevel + 1).write(tabvs.getDataOutput(), true);
                    rbStack.get(nestedLevel).addField(fieldNamePointable, tabvs);
                } else {
                    rbStack.get(nestedLevel).addField(fieldNamePointable, fieldValuePointable);
                }
            }
        }

        private boolean isValidPath(AListVisitablePointable inputList) throws HyracksDataException {
            List<IVisitablePointable> items = inputList.getItems();
            List<IVisitablePointable> typeTags = inputList.getItemTags();
            int pathLen = recordPath.size();
            for (int i = 0; i < items.size(); i++) {
                IVisitablePointable item = items.get(i);
                if (PointableHelper.sameType(ATypeTag.ARRAY, typeTags.get(i))) {
                    List<IVisitablePointable> inputPathItems = ((AListVisitablePointable) item).getItems();
                    if (pathLen == inputPathItems.size()) {
                        boolean match = true;
                        Iterator<IVisitablePointable> fpi = recordPath.iterator();
                        for (int j = inputPathItems.size() - 1; j >= 0; j--) {
                            match &= PointableHelper.isEqual(inputPathItems.get(j), fpi.next());
                            if (!match) {
                                break;
                            }
                        }
                        if (match) {
                            // Not a valid path for the output record
                            return false;
                        }
                    }
                } else {
                    if (PointableHelper.isEqual(recordPath.getFirst(), item)) {
                        return false;
                    }
                }
            }
            return true;
        }
    };
}

Example 100 with IFrameTupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference in project asterixdb by apache.

the class PrintBinaryDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new AbstractBinaryScalarEvaluator(ctx, args) {

                private StringBuilder stringBuilder = new StringBuilder();

                private final ByteArrayPointable byteArrayPtr = new ByteArrayPointable();

                private final UTF8StringPointable formatPointable = new UTF8StringPointable();

                private final UTF8StringWriter writer = new UTF8StringWriter();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    evaluators[0].evaluate(tuple, pointables[0]);
                    evaluators[1].evaluate(tuple, pointables[1]);
                    try {
                        ATypeTag arg0Tag = ATypeTag.VALUE_TYPE_MAPPING[pointables[0].getByteArray()[pointables[0].getStartOffset()]];
                        ATypeTag arg1Tag = ATypeTag.VALUE_TYPE_MAPPING[pointables[1].getByteArray()[pointables[1].getStartOffset()]];
                        checkTypeMachingThrowsIfNot(getIdentifier().getName(), EXPECTED_INPUT_TAGS, arg0Tag, arg1Tag);
                        byteArrayPtr.set(pointables[0].getByteArray(), pointables[0].getStartOffset() + 1, pointables[0].getLength());
                        formatPointable.set(pointables[1].getByteArray(), pointables[1].getStartOffset() + 1, pointables[1].getLength());
                        int lengthBinary = byteArrayPtr.getContentLength();
                        stringBuilder.setLength(0);
                        if (HEX_FORMAT.ignoreCaseCompareTo(formatPointable) == 0) {
                            HexPrinter.printHexString(byteArrayPtr.getByteArray(), byteArrayPtr.getContentStartOffset(), lengthBinary, stringBuilder);
                        } else if (BASE64_FORMAT.ignoreCaseCompareTo(formatPointable) == 0) {
                            Base64Printer.printBase64Binary(byteArrayPtr.getByteArray(), byteArrayPtr.getContentStartOffset(), lengthBinary, stringBuilder);
                        } else {
                            throw new UnsupportedItemTypeException(getIdentifier(), arg1Tag.serialize());
                        }
                        dataOutput.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                        writer.writeUTF8(stringBuilder.toString(), dataOutput);
                    } catch (IOException e) {
                        throw new HyracksDataException(e);
                    }
                    result.set(resultStorage);
                }
            };
        }
    };
}