Examples with AvroLiteral com.linkedin.avroutil1.model.AvroLiteral used on opensource projects

Example 1 with AvroLiteral

use of com.linkedin.avroutil1.model.AvroLiteral in project avro-util by linkedin.

the class AvscSchemaWriter method emitRecordFields.

protected void emitRecordFields(AvroRecordSchema schema, AvscWriterContext context, AvscWriterConfig config, JsonObjectBuilder output) {
    JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
    List<AvroSchemaField> fields = schema.getFields();
    for (AvroSchemaField field : fields) {
        JsonObjectBuilder fieldBuilder = Json.createObjectBuilder();
        fieldBuilder.add("name", field.getName());
        if (field.hasDoc()) {
            fieldBuilder.add("doc", field.getDoc());
        }
        AvroSchema fieldSchema = field.getSchema();
        fieldBuilder.add("type", writeSchema(fieldSchema, context, config));
        if (field.hasDefaultValue()) {
            AvroLiteral defaultValue = field.getDefaultValue();
            JsonValue defaultValueLiteral = writeDefaultValue(fieldSchema, defaultValue);
            fieldBuilder.add("default", defaultValueLiteral);
        }
        // TODO - order
        // TODO - aliases
        arrayBuilder.add(fieldBuilder);
    }
    output.add("fields", arrayBuilder);
}

Example 2 with AvroLiteral

use of com.linkedin.avroutil1.model.AvroLiteral in project avro-util by linkedin.

the class AvscSchemaWriter method writeDefaultValue.

protected JsonValue writeDefaultValue(AvroSchema fieldSchema, AvroLiteral literal) {
    AvroType type = fieldSchema.type();
    String temp;
    switch(type) {
        case NULL:
            // noinspection unused (kept as a sanity check)
            AvroNullLiteral nullLiteral = (AvroNullLiteral) literal;
            return JsonValue.NULL;
        case BOOLEAN:
            AvroBooleanLiteral boolLiteral = (AvroBooleanLiteral) literal;
            return boolLiteral.getValue() ? JsonValue.TRUE : JsonValue.FALSE;
        case INT:
            AvroIntegerLiteral intLiteral = (AvroIntegerLiteral) literal;
            return Json.createValue(intLiteral.getValue());
        case LONG:
            AvroLongLiteral longLiteral = (AvroLongLiteral) literal;
            return Json.createValue(longLiteral.getValue());
        case FLOAT:
            AvroFloatLiteral floatLiteral = (AvroFloatLiteral) literal;
            return Json.createValue(floatLiteral.getValue());
        case DOUBLE:
            AvroDoubleLiteral doubleLiteral = (AvroDoubleLiteral) literal;
            return Json.createValue(doubleLiteral.getValue());
        case STRING:
            AvroStringLiteral stringLiteral = (AvroStringLiteral) literal;
            return Json.createValue(stringLiteral.getValue());
        case BYTES:
            AvroBytesLiteral bytesLiteral = (AvroBytesLiteral) literal;
            // spec  says "values for bytes and fixed fields are JSON strings, where Unicode code points
            // 0-255 are mapped to unsigned 8-bit byte values 0-255", and this is how its done
            temp = new String(bytesLiteral.getValue(), StandardCharsets.ISO_8859_1);
            return Json.createValue(temp);
        case ENUM:
            AvroEnumLiteral enumLiteral = (AvroEnumLiteral) literal;
            return Json.createValue(enumLiteral.getValue());
        case FIXED:
            AvroFixedLiteral fixedLiteral = (AvroFixedLiteral) literal;
            // spec  says "values for bytes and fixed fields are JSON strings, where Unicode code points
            // 0-255 are mapped to unsigned 8-bit byte values 0-255", and this is how its done
            temp = new String(fixedLiteral.getValue(), StandardCharsets.ISO_8859_1);
            return Json.createValue(temp);
        case ARRAY:
            AvroArrayLiteral arrayLiteral = (AvroArrayLiteral) literal;
            List<AvroLiteral> array = arrayLiteral.getValue();
            AvroArraySchema arraySchema = (AvroArraySchema) arrayLiteral.getSchema();
            JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
            for (AvroLiteral element : array) {
                arrayBuilder.add(writeDefaultValue(arraySchema.getValueSchema(), element));
            }
            return arrayBuilder.build();
        default:
            throw new UnsupportedOperationException("writing default values for " + type + " not implemented yet");
    }
}

Example 3 with AvroLiteral

use of com.linkedin.avroutil1.model.AvroLiteral in project avro-util by linkedin.

the class AvscParser method parseNamedSchema.

private AvroNamedSchema parseNamedSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer extraProps) {
    AvroName schemaName = parseSchemaName(objectNode, context, avroType);
    List<AvroName> aliases = parseAliases(objectNode, context, avroType, schemaName);
    // technically the avro spec does not allow "doc" on type fixed, but screw that
    Located<String> docStr = getOptionalString(objectNode, "doc");
    String doc = docStr != null ? docStr.getValue() : null;
    boolean namespaceChanged = false;
    // check if context namespace changed
    if (!context.getCurrentNamespace().equals(schemaName.getNamespace())) {
        context.pushNamespace(schemaName.getNamespace());
        namespaceChanged = true;
    }
    AvroNamedSchema namedSchema;
    switch(avroType) {
        case RECORD:
            AvroRecordSchema recordSchema = new AvroRecordSchema(codeLocation, schemaName, aliases, doc, extraProps);
            JsonArrayExt fieldsNode = getRequiredArray(objectNode, "fields", () -> "all avro records must have fields");
            List<AvroSchemaField> fields = new ArrayList<>(fieldsNode.size());
            for (int fieldNum = 0; fieldNum < fieldsNode.size(); fieldNum++) {
                // !=null
                JsonValueExt fieldDeclNode = (JsonValueExt) fieldsNode.get(fieldNum);
                JsonValue.ValueType fieldNodeType = fieldDeclNode.getValueType();
                if (fieldNodeType != JsonValue.ValueType.OBJECT) {
                    throw new AvroSyntaxException("field " + fieldNum + " for record " + schemaName.getSimpleName() + " at " + fieldDeclNode.getStartLocation() + " expected to be an OBJECT, not a " + JsonPUtil.describe(fieldNodeType) + " (" + fieldDeclNode + ")");
                }
                TextLocation fieldStartLocation = Util.convertLocation(fieldDeclNode.getStartLocation());
                TextLocation fieldEndLocation = Util.convertLocation(fieldDeclNode.getEndLocation());
                CodeLocation fieldCodeLocation = new CodeLocation(context.getUri(), fieldStartLocation, fieldEndLocation);
                JsonObjectExt fieldDecl = (JsonObjectExt) fieldDeclNode;
                Located<String> fieldName = getRequiredString(fieldDecl, "name", () -> "all record fields must have a name");
                JsonValueExt fieldTypeNode = getRequiredNode(fieldDecl, "type", () -> "all record fields must have a type");
                Located<String> locatedDocField = getOptionalString(fieldDecl, "doc");
                String docField = locatedDocField == null ? null : locatedDocField.getValue();
                SchemaOrRef fieldSchema = parseSchemaDeclOrRef(fieldTypeNode, context, false);
                JsonValueExt fieldDefaultValueNode = fieldDecl.get("default");
                AvroLiteral defaultValue = null;
                if (fieldDefaultValueNode != null) {
                    if (fieldSchema.isResolved()) {
                        LiteralOrIssue defaultValurOrIssue = parseLiteral(fieldDefaultValueNode, fieldSchema.getSchema(), fieldName.getValue(), context);
                        if (defaultValurOrIssue.getIssue() == null) {
                            defaultValue = defaultValurOrIssue.getLiteral();
                        }
                    // TODO - handle issues
                    } else {
                        // TODO - implement delayed default value parsing
                        throw new UnsupportedOperationException("delayed parsing of default value for " + fieldName.getValue() + " TBD");
                    }
                }
                LinkedHashMap<String, JsonValueExt> props = parseExtraProps(fieldDecl, CORE_FIELD_PROPERTIES);
                JsonPropertiesContainer propsContainer = props.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(props);
                AvroSchemaField field = new AvroSchemaField(fieldCodeLocation, fieldName.getValue(), docField, fieldSchema, defaultValue, propsContainer);
                fields.add(field);
            }
            recordSchema.setFields(fields);
            namedSchema = recordSchema;
            break;
        case ENUM:
            JsonArrayExt symbolsNode = getRequiredArray(objectNode, "symbols", () -> "all avro enums must have symbols");
            List<String> symbols = new ArrayList<>(symbolsNode.size());
            for (int ordinal = 0; ordinal < symbolsNode.size(); ordinal++) {
                JsonValueExt symbolNode = (JsonValueExt) symbolsNode.get(ordinal);
                JsonValue.ValueType symbolNodeType = symbolNode.getValueType();
                if (symbolNodeType != JsonValue.ValueType.STRING) {
                    throw new AvroSyntaxException("symbol " + ordinal + " for enum " + schemaName.getSimpleName() + " at " + symbolNode.getStartLocation() + " expected to be a STRING, not a " + JsonPUtil.describe(symbolNodeType) + " (" + symbolNode + ")");
                }
                symbols.add(symbolNode.toString());
            }
            String defaultSymbol = null;
            Located<String> defaultStr = getOptionalString(objectNode, "default");
            if (defaultStr != null) {
                defaultSymbol = defaultStr.getValue();
                if (!symbols.contains(defaultSymbol)) {
                    context.addIssue(AvscIssues.badEnumDefaultValue(locationOf(context.getUri(), defaultStr), defaultSymbol, schemaName.getSimpleName(), symbols));
                    // TODO - support "fixing" by selecting 1st symbol as default?
                    defaultSymbol = null;
                }
            }
            namedSchema = new AvroEnumSchema(codeLocation, schemaName, aliases, doc, symbols, defaultSymbol, extraProps);
            break;
        case FIXED:
            JsonValueExt sizeNode = getRequiredNode(objectNode, "size", () -> "fixed types must have a size property");
            if (sizeNode.getValueType() != JsonValue.ValueType.NUMBER || !(((JsonNumberExt) sizeNode).isIntegral())) {
                throw new AvroSyntaxException("size for fixed " + schemaName.getSimpleName() + " at " + sizeNode.getStartLocation() + " expected to be an INTEGER, not a " + JsonPUtil.describe(sizeNode.getValueType()) + " (" + sizeNode + ")");
            }
            int fixedSize = ((JsonNumberExt) sizeNode).intValue();
            Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(objectNode, context, avroType, codeLocation);
            if (logicalTypeResult.hasIssues()) {
                context.addIssues(logicalTypeResult.getIssues());
            }
            namedSchema = new AvroFixedSchema(codeLocation, schemaName, aliases, doc, fixedSize, logicalTypeResult.getData(), extraProps);
            break;
        default:
            throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
    }
    if (namespaceChanged) {
        context.popNamespace();
    }
    return namedSchema;
}

Example 4 with AvroLiteral

use of com.linkedin.avroutil1.model.AvroLiteral in project avro-util by linkedin.

the class AvscParser method parseLiteral.

private LiteralOrIssue parseLiteral(JsonValueExt literalNode, AvroSchema schema, String fieldName, AvscFileParseContext context) {
    AvroType avroType = schema.type();
    JsonValue.ValueType jsonType = literalNode.getValueType();
    AvscIssue issue;
    BigInteger bigIntegerValue;
    BigDecimal bigDecimalValue;
    byte[] bytes;
    switch(avroType) {
        case NULL:
            if (jsonType != JsonValue.ValueType.NULL) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroNullLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode)));
        case BOOLEAN:
            if (jsonType != JsonValue.ValueType.FALSE && jsonType != JsonValue.ValueType.TRUE) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            boolean boolValue = jsonType == JsonValue.ValueType.TRUE;
            return new LiteralOrIssue(new AvroBooleanLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), boolValue));
        case INT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt intNode = (JsonNumberExt) literalNode;
            if (!intNode.isIntegral()) {
                // TODO - be more specific about this error (int vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = intNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_INT) > 0 || bigIntegerValue.compareTo(MIN_INT) < 0) {
                // TODO - be more specific about this error (out of signed int range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroIntegerLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.intValueExact()));
        case LONG:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt longNode = (JsonNumberExt) literalNode;
            if (!longNode.isIntegral()) {
                // TODO - be more specific about this error (long vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = longNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_LONG) > 0 || bigIntegerValue.compareTo(MIN_LONG) < 0) {
                // TODO - be more specific about this error (out of signed long range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroLongLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.longValueExact()));
        case FLOAT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt floatNode = (JsonNumberExt) literalNode;
            bigDecimalValue = floatNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_FLOAT) > 0) {
                // TODO - be more specific about this error (out of float range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFloatLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.floatValue()));
        case DOUBLE:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt doubleNode = (JsonNumberExt) literalNode;
            bigDecimalValue = doubleNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_DOUBLE) > 0) {
                // TODO - be more specific about this error (out of double range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroDoubleLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.doubleValue()));
        case BYTES:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt bytesNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = bytesNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            return new LiteralOrIssue(new AvroBytesLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bytes));
        case FIXED:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
            JsonStringExt fixedNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = fixedNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            if (bytes.length != fixedSchema.getSize()) {
                // TODO - be more specific about this error (wrong length)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFixedLiteral(fixedSchema, locationOf(context.getUri(), literalNode), bytes));
        case STRING:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt stringNode = (JsonStringExt) literalNode;
            return new LiteralOrIssue(new AvroStringLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), stringNode.getString()));
        case ENUM:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
            JsonStringExt enumNode = (JsonStringExt) literalNode;
            String enumValue = enumNode.getString();
            if (!enumSchema.getSymbols().contains(enumValue)) {
                // TODO - be more specific about this error (unknown symbol)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroEnumLiteral(enumSchema, locationOf(context.getUri(), literalNode), enumValue));
        case ARRAY:
            if (jsonType != JsonValue.ValueType.ARRAY) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroArraySchema arraySchema = (AvroArraySchema) schema;
            AvroSchema valueSchema = arraySchema.getValueSchema();
            JsonArrayExt arrayNode = (JsonArrayExt) literalNode;
            ArrayList<AvroLiteral> values = new ArrayList<>(arrayNode.size());
            for (int i = 0; i < arrayNode.size(); i++) {
                JsonValueExt valueNode = (JsonValueExt) arrayNode.get(i);
                LiteralOrIssue value = parseLiteral(valueNode, valueSchema, fieldName, context);
                // issues parsing any member value mean a failure to parse the array as a whole
                if (value.getIssue() != null) {
                    // TODO - be more specific about this error (unparsable array element i)
                    // TODO - add "causedBy" to AvscIssue and use it here
                    issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                    context.addIssue(issue);
                    return new LiteralOrIssue(issue);
                }
                values.add(value.getLiteral());
            }
            return new LiteralOrIssue(new AvroArrayLiteral(arraySchema, locationOf(context.getUri(), literalNode), values));
        default:
            throw new UnsupportedOperationException("dont know how to parse a " + avroType + " at " + literalNode.getStartLocation() + " out of a " + literalNode.getValueType() + " (" + literalNode + ")");
    }
}