Examples with AUnionType org.apache.asterix.om.types.AUnionType used on opensource projects

Example 21 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class ARecordCaster method loadRequiredType.

private void loadRequiredType(ARecordType reqType) throws IOException {
    reqFieldNames.clear();
    reqFieldTypeTags.clear();
    allocator.reset();
    cachedReqType = reqType;
    int numSchemaFields = reqType.getFieldTypes().length;
    IAType[] fieldTypes = reqType.getFieldTypes();
    String[] fieldNames = reqType.getFieldNames();
    fieldPermutation = new int[numSchemaFields];
    optionalFields = new boolean[numSchemaFields];
    for (int i = 0; i < optionalFields.length; i++) {
        optionalFields[i] = false;
    }
    bos.reset(nullTypeTag.getStartOffset() + nullTypeTag.getLength());
    for (int i = 0; i < numSchemaFields; i++) {
        ATypeTag ftypeTag = fieldTypes[i].getTypeTag();
        String fname = fieldNames[i];
        // add type tag pointable
        if (NonTaggedFormatUtil.isOptional(fieldTypes[i])) {
            // optional field: add the embedded non-null type tag
            ftypeTag = ((AUnionType) fieldTypes[i]).getActualType().getTypeTag();
            optionalFields[i] = true;
        }
        int tagStart = bos.size();
        dos.writeByte(ftypeTag.serialize());
        int tagEnd = bos.size();
        IVisitablePointable typeTagPointable = allocator.allocateEmpty();
        typeTagPointable.set(bos.getByteArray(), tagStart, tagEnd - tagStart);
        reqFieldTypeTags.add(typeTagPointable);
        // add type name pointable (including a string type tag)
        int nameStart = bos.size();
        dos.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
        utf8Writer.writeUTF8(fname, dos);
        int nameEnd = bos.size();
        IVisitablePointable typeNamePointable = allocator.allocateEmpty();
        typeNamePointable.set(bos.getByteArray(), nameStart, nameEnd - nameStart);
        reqFieldNames.add(typeNamePointable);
    }
    reqFieldNamesSortedIndex = new int[reqFieldNames.size()];
    for (int i = 0; i < reqFieldNamesSortedIndex.length; i++) {
        reqFieldNamesSortedIndex[i] = i;
    }
    // sort the field name index
    quickSort(reqFieldNamesSortedIndex, reqFieldNames, 0, reqFieldNamesSortedIndex.length - 1);
}

Example 22 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class ARecordCaster method writeOutput.

private void writeOutput(List<IVisitablePointable> fieldNames, List<IVisitablePointable> fieldTypeTags, List<IVisitablePointable> fieldValues, DataOutput output, ACastVisitor visitor) throws HyracksDataException {
    // reset the states of the record builder
    recBuilder.reset(cachedReqType);
    recBuilder.init();
    // write the closed part
    for (int i = 0; i < fieldPermutation.length; i++) {
        final int pos = fieldPermutation[i];
        final IVisitablePointable field = pos >= 0 ? fieldValues.get(pos) : missingTypeTag;
        final IAType fType = cachedReqType.getFieldTypes()[i];
        nestedVisitorArg.second = fType;
        // as flat
        if (optionalFields[i]) {
            //the field is optional in the input record
            IVisitablePointable fieldTypeTag = pos >= 0 ? fieldTypeTags.get(pos) : null;
            if (fieldTypeTag == null || fieldTypeTag.equals(missingTypeTag)) {
                nestedVisitorArg.second = BuiltinType.AMISSING;
            } else if (fieldTypeTag.equals(nullTypeTag)) {
                nestedVisitorArg.second = BuiltinType.ANULL;
            } else {
                nestedVisitorArg.second = ((AUnionType) fType).getActualType();
            }
        }
        field.accept(visitor, nestedVisitorArg);
        recBuilder.addField(i, nestedVisitorArg.first);
    }
    // write the open part
    for (int i = 0; i < numInputFields; i++) {
        if (openFields[i]) {
            IVisitablePointable name = fieldNames.get(i);
            IVisitablePointable field = fieldValues.get(i);
            IVisitablePointable fieldTypeTag = fieldTypeTags.get(i);
            ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(fieldTypeTag.getByteArray()[fieldTypeTag.getStartOffset()]);
            nestedVisitorArg.second = DefaultOpenFieldType.getDefaultOpenFieldType(typeTag);
            field.accept(visitor, nestedVisitorArg);
            recBuilder.addField(name, nestedVisitorArg.first);
        }
    }
    recBuilder.write(output, true);
}

Example 23 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class TweetParserTest method openRecordTypeTest.

@Test
public void openRecordTypeTest() throws IOException, URISyntaxException {
    String[] ids = { "720549057849114629", "668950503552864257", "668945640186101761", "263602997047730177", "668948268605403136", "741701526859567104" };
    // contruct type
    IAType geoFieldType = new ARecordType("GeoType", new String[] { "coordinates" }, new IAType[] { new AOrderedListType(AFLOAT, "point") }, true);
    List<IAType> unionTypeList = new ArrayList<>();
    unionTypeList.add(geoFieldType);
    unionTypeList.add(ANULL);
    unionTypeList.add(AMISSING);
    IAType geoUnionType = new AUnionType(unionTypeList, "GeoType?");
    ARecordType tweetRecordType = new ARecordType("TweetType", new String[] { "id", "geo" }, new IAType[] { AINT64, geoUnionType }, true);
    TweetParser parser = new TweetParser(tweetRecordType);
    List<String> lines = Files.readAllLines(Paths.get(getClass().getResource("/test_tweets.txt").toURI()));
    ByteArrayOutputStream is = new ByteArrayOutputStream();
    DataOutput output = new DataOutputStream(is);
    for (int iter1 = 0; iter1 < lines.size(); iter1++) {
        GenericRecord<String> record = new GenericRecord<>();
        record.set(lines.get(iter1));
        try {
            parser.parse(record, output);
        } catch (HyracksDataException e) {
            e.printStackTrace();
            Assert.fail("Unexpected failure in parser.");
        }
        Assert.assertTrue((PA.getValue(parser, "aInt64")).toString().equals(ids[iter1]));
    }
}

Example 24 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class NonTaggedDataFormat method registerTypeInferers.

void registerTypeInferers() {
    functionTypeInferers.put(BuiltinFunctions.LISTIFY, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            fd.setImmutableStates(context.getType(expr));
        }
    });
    functionTypeInferers.put(BuiltinFunctions.RECORD_MERGE, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
            IAType outType = (IAType) context.getType(expr);
            IAType type0 = (IAType) context.getType(f.getArguments().get(0).getValue());
            IAType type1 = (IAType) context.getType(f.getArguments().get(1).getValue());
            fd.setImmutableStates(outType, type0, type1);
        }
    });
    functionTypeInferers.put(BuiltinFunctions.DEEP_EQUAL, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
            IAType type0 = (IAType) context.getType(f.getArguments().get(0).getValue());
            IAType type1 = (IAType) context.getType(f.getArguments().get(1).getValue());
            fd.setImmutableStates(type0, type1);
        }
    });
    functionTypeInferers.put(BuiltinFunctions.ADD_FIELDS, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
            IAType outType = (IAType) context.getType(expr);
            IAType type0 = (IAType) context.getType(f.getArguments().get(0).getValue());
            ILogicalExpression listExpr = f.getArguments().get(1).getValue();
            IAType type1 = (IAType) context.getType(listExpr);
            if (type0.getTypeTag().equals(ATypeTag.ANY)) {
                type0 = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
            }
            if (type1.getTypeTag().equals(ATypeTag.ANY)) {
                type1 = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
            }
            fd.setImmutableStates(outType, type0, type1);
        }
    });
    functionTypeInferers.put(BuiltinFunctions.REMOVE_FIELDS, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
            IAType outType = (IAType) context.getType(expr);
            IAType type0 = (IAType) context.getType(f.getArguments().get(0).getValue());
            ILogicalExpression le = f.getArguments().get(1).getValue();
            IAType type1 = (IAType) context.getType(le);
            if (type0.getTypeTag().equals(ATypeTag.ANY)) {
                type0 = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
            }
            if (type1.getTypeTag().equals(ATypeTag.ANY)) {
                type1 = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
            }
            fd.setImmutableStates(outType, type0, type1);
        }
    });
    functionTypeInferers.put(BuiltinFunctions.CAST_TYPE, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
            IAType rt = TypeCastUtils.getRequiredType(funcExpr);
            IAType it = (IAType) context.getType(funcExpr.getArguments().get(0).getValue());
            fd.setImmutableStates(rt, it);
        }
    });
    functionTypeInferers.put(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            ARecordType rt = (ARecordType) context.getType(expr);
            fd.setImmutableStates(rt, computeOpenFields((AbstractFunctionCallExpression) expr, rt));
        }

        private boolean[] computeOpenFields(AbstractFunctionCallExpression expr, ARecordType recType) {
            int n = expr.getArguments().size() / 2;
            boolean[] open = new boolean[n];
            for (int i = 0; i < n; i++) {
                Mutable<ILogicalExpression> argRef = expr.getArguments().get(2 * i);
                ILogicalExpression arg = argRef.getValue();
                open[i] = true;
                final String fn = ConstantExpressionUtil.getStringConstant(arg);
                if (fn != null) {
                    for (String s : recType.getFieldNames()) {
                        if (s.equals(fn)) {
                            open[i] = false;
                            break;
                        }
                    }
                }
            }
            return open;
        }
    });
    functionTypeInferers.put(BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            fd.setImmutableStates(context.getType(expr));
        }
    });
    functionTypeInferers.put(BuiltinFunctions.ORDERED_LIST_CONSTRUCTOR, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            fd.setImmutableStates(context.getType(expr));
        }
    });
    functionTypeInferers.put(BuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            fd.setImmutableStates(context.getType(expr));
        }
    });
    functionTypeInferers.put(BuiltinFunctions.FIELD_ACCESS_BY_INDEX, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            IAType t = (IAType) context.getType(fce.getArguments().get(0).getValue());
            switch(t.getTypeTag()) {
                case OBJECT:
                    {
                        fd.setImmutableStates(t);
                        break;
                    }
                case UNION:
                    {
                        AUnionType unionT = (AUnionType) t;
                        if (unionT.isUnknownableType()) {
                            IAType t2 = unionT.getActualType();
                            if (t2.getTypeTag() == ATypeTag.OBJECT) {
                                fd.setImmutableStates(t2);
                                break;
                            }
                        }
                        throw new NotImplementedException("field-access-by-index for data of type " + t);
                    }
                default:
                    {
                        throw new NotImplementedException("field-access-by-index for data of type " + t);
                    }
            }
        }
    });
    functionTypeInferers.put(BuiltinFunctions.FIELD_ACCESS_NESTED, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            IAType t = (IAType) context.getType(fce.getArguments().get(0).getValue());
            AOrderedList fieldPath = (AOrderedList) (((AsterixConstantValue) ((ConstantExpression) fce.getArguments().get(1).getValue()).getValue()).getObject());
            List<String> listFieldPath = new ArrayList<String>();
            for (int i = 0; i < fieldPath.size(); i++) {
                listFieldPath.add(((AString) fieldPath.getItem(i)).getStringValue());
            }
            switch(t.getTypeTag()) {
                case OBJECT:
                    {
                        fd.setImmutableStates(t, listFieldPath);
                        break;
                    }
                case ANY:
                    fd.setImmutableStates(RecordUtil.FULLY_OPEN_RECORD_TYPE, listFieldPath);
                    break;
                default:
                    {
                        throw new NotImplementedException("field-access-nested for data of type " + t);
                    }
            }
        }
    });
    functionTypeInferers.put(BuiltinFunctions.GET_RECORD_FIELDS, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            IAType t = (IAType) context.getType(fce.getArguments().get(0).getValue());
            ATypeTag typeTag = t.getTypeTag();
            if (typeTag.equals(ATypeTag.OBJECT)) {
                fd.setImmutableStates(t);
            } else if (typeTag.equals(ATypeTag.ANY)) {
                fd.setImmutableStates(RecordUtil.FULLY_OPEN_RECORD_TYPE);
            } else {
                throw new NotImplementedException("get-record-fields for data of type " + t);
            }
        }
    });
    functionTypeInferers.put(BuiltinFunctions.GET_RECORD_FIELD_VALUE, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            IAType t = (IAType) context.getType(fce.getArguments().get(0).getValue());
            ATypeTag typeTag = t.getTypeTag();
            if (typeTag.equals(ATypeTag.OBJECT)) {
                fd.setImmutableStates(t);
            } else if (typeTag.equals(ATypeTag.ANY)) {
                fd.setImmutableStates(RecordUtil.FULLY_OPEN_RECORD_TYPE);
            } else {
                throw new NotImplementedException("get-record-field-value for data of type " + t);
            }
        }
    });
    functionTypeInferers.put(BuiltinFunctions.RECORD_PAIRS, new FunctionTypeInferer() {

        @Override
        public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) throws AlgebricksException {
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            IAType t = (IAType) context.getType(fce.getArguments().get(0).getValue());
            ATypeTag typeTag = t.getTypeTag();
            if (typeTag.equals(ATypeTag.OBJECT)) {
                fd.setImmutableStates(t);
            } else if (typeTag.equals(ATypeTag.ANY)) {
                fd.setImmutableStates(RecordUtil.FULLY_OPEN_RECORD_TYPE);
            } else {
                throw new NotImplementedException("record-fields with data of type " + t);
            }
        }
    });
}

Example 25 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class ADMDataParser method getComplexType.

private IAType getComplexType(IAType aObjectType, ATypeTag tag) {
    if (aObjectType == null) {
        return null;
    }
    if (aObjectType.getTypeTag() == tag) {
        return aObjectType;
    }
    if (aObjectType.getTypeTag() == ATypeTag.UNION) {
        AUnionType unionType = (AUnionType) aObjectType;
        IAType type = unionType.getActualType();
        if (type.getTypeTag() == tag) {
            return type;
        }
    }
    // wont get here
    return null;
}