Examples with AvroPrimitiveSchema com.linkedin.avroutil1.model.AvroPrimitiveSchema used on opensource projects

Example 1 with AvroPrimitiveSchema

use of com.linkedin.avroutil1.model.AvroPrimitiveSchema in project avro-util by linkedin.

the class AvscParserTest method testParsingStringTypes.

@Test
public void testParsingStringTypes() throws Exception {
    String avsc = TestUtil.load("schemas/TestRecordWithStringTypes.avsc");
    AvscParser parser = new AvscParser();
    AvscParseResult result = parser.parse(avsc);
    Assert.assertNull(result.getParseError());
    AvroRecordSchema schema = (AvroRecordSchema) result.getTopLevelSchema();
    Assert.assertNotNull(schema);
    for (AvroSchemaField field : schema.getFields()) {
        Assert.assertEquals(field.getSchema().type(), AvroType.STRING);
        AvroPrimitiveSchema strSchema = (AvroPrimitiveSchema) field.getSchema();
        switch(field.getName()) {
            case "vanillaStringField":
                Assert.assertNull(strSchema.getJavaStringRepresentation());
                break;
            case "stringFieldWithStringJavaType":
                Assert.assertEquals(strSchema.getJavaStringRepresentation(), AvroJavaStringRepresentation.STRING);
                break;
            case "stringFieldWithMisplacedCharSequenceJavaType":
                Assert.assertNull(strSchema.getJavaStringRepresentation());
                // TODO - look for a warning about the misplaced value
                break;
            case "stringFieldWithCharSequenceJavaType":
                Assert.assertEquals(strSchema.getJavaStringRepresentation(), AvroJavaStringRepresentation.CHAR_SEQUENCE);
                break;
            case "stringFieldWithUtf8JavaType":
                Assert.assertEquals(strSchema.getJavaStringRepresentation(), AvroJavaStringRepresentation.UTF8);
                break;
        }
    }
}

Example 2 with AvroPrimitiveSchema

use of com.linkedin.avroutil1.model.AvroPrimitiveSchema in project avro-util by linkedin.

the class AvscParserTest method testParsingLogicalTypes.

@Test
public void testParsingLogicalTypes() throws Exception {
    String avsc = TestUtil.load("schemas/TestRecordWithLogicalTypes.avsc");
    AvscParser parser = new AvscParser();
    AvscParseResult result = parser.parse(avsc);
    Assert.assertNull(result.getParseError());
    AvroRecordSchema schema = (AvroRecordSchema) result.getTopLevelSchema();
    Assert.assertNotNull(schema);
    for (AvroSchemaField field : schema.getFields()) {
        switch(field.getName()) {
            case "bytesDecimalField":
                Assert.assertEquals(field.getSchema().type(), AvroType.BYTES);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.DECIMAL);
                Assert.assertEquals(((AvroPrimitiveSchema) field.getSchema()).getScale(), 2);
                Assert.assertEquals(((AvroPrimitiveSchema) field.getSchema()).getPrecision(), 4);
                break;
            case "fixedDecimalField":
                Assert.assertEquals(field.getSchema().type(), AvroType.FIXED);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.DECIMAL);
                break;
            case "fixedDurationField":
                Assert.assertEquals(field.getSchema().type(), AvroType.FIXED);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.DURATION);
                break;
            case "stringUUIDField":
                Assert.assertEquals(field.getSchema().type(), AvroType.STRING);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.UUID);
                break;
            case "intDateField":
                Assert.assertEquals(field.getSchema().type(), AvroType.INT);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.DATE);
                break;
            case "intTimeMillisField":
                Assert.assertEquals(field.getSchema().type(), AvroType.INT);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.TIME_MILLIS);
                break;
            case "longTimeMicrosField":
                Assert.assertEquals(field.getSchema().type(), AvroType.LONG);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.TIME_MICROS);
                break;
            case "longTimestampMillisField":
                Assert.assertEquals(field.getSchema().type(), AvroType.LONG);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.TIMESTAMP_MILLIS);
                break;
            case "longTimestampMicrosField":
                Assert.assertEquals(field.getSchema().type(), AvroType.LONG);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.TIMESTAMP_MICROS);
                break;
            case "longLocalTimestampMillisField":
                Assert.assertEquals(field.getSchema().type(), AvroType.LONG);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.LOCAL_TIMESTAMP_MILLIS);
                break;
            case "longLocalTimestampMicrosField":
                Assert.assertEquals(field.getSchema().type(), AvroType.LONG);
                Assert.assertEquals(field.getSchema().logicalType(), AvroLogicalType.LOCAL_TIMESTAMP_MICROS);
                break;
            default:
                break;
        }
    }
}

Example 3 with AvroPrimitiveSchema

use of com.linkedin.avroutil1.model.AvroPrimitiveSchema in project avro-util by linkedin.

the class AvscParser method parseDecoratedPrimitiveSchema.

private AvroPrimitiveSchema parseDecoratedPrimitiveSchema(JsonObjectExt primitiveNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer props) {
    AvroLogicalType logicalType = null;
    int scale = 0;
    int precision = 0;
    Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(primitiveNode, context, avroType, codeLocation);
    if (logicalTypeResult.hasIssues()) {
        context.addIssues(logicalTypeResult.getIssues());
    }
    // might be null
    logicalType = logicalTypeResult.getData();
    Parsed<AvroJavaStringRepresentation> stringRepResult = parseStringRepresentation(primitiveNode, context, avroType, codeLocation);
    AvroJavaStringRepresentation stringRep = null;
    if (stringRepResult.hasIssues()) {
        context.addIssues(stringRepResult.getIssues());
    }
    if (AvroType.STRING.equals(avroType) && stringRepResult.hasData()) {
        stringRep = stringRepResult.getData();
    }
    Located<Integer> precisionResult = getOptionalInteger(primitiveNode, "precision", context);
    Located<Integer> scaleResult = getOptionalInteger(primitiveNode, "scale", context);
    boolean scaleAndPrecisionExpected = AvroType.BYTES.equals(avroType) && AvroLogicalType.DECIMAL.equals(logicalType);
    if (scaleAndPrecisionExpected) {
        boolean scaleAndPrecisionOK = true;
        int precisionValue = 0;
        int scaleValue = 0;
        // precision is actually required
        if (precisionResult == null) {
            context.addIssue(AvscIssues.precisionRequiredAndNotSet(codeLocation, avroType, logicalType));
            scaleAndPrecisionOK = false;
        }
        if (scaleAndPrecisionOK && scaleResult != null) {
            precisionValue = precisionResult.getValue();
            scaleValue = scaleResult.getValue();
            if (precisionValue < scaleValue) {
                context.addIssue(AvscIssues.precisionSmallerThanScale(locationOf(context.getUri(), precisionResult), precisionValue, locationOf(context.getUri(), scaleResult), scaleValue));
                scaleAndPrecisionOK = false;
            }
        }
        if (scaleAndPrecisionOK) {
            scale = scaleValue;
            precision = precisionValue;
        } else {
            // "cancel" the logicalType
            logicalType = null;
        }
    }
    return new AvroPrimitiveSchema(codeLocation, avroType, logicalType, stringRep, scale, precision, props);
}

Example 4 with AvroPrimitiveSchema

use of com.linkedin.avroutil1.model.AvroPrimitiveSchema in project avro-util by linkedin.

the class AvscParser method parseLiteral.

private LiteralOrIssue parseLiteral(JsonValueExt literalNode, AvroSchema schema, String fieldName, AvscFileParseContext context) {
    AvroType avroType = schema.type();
    JsonValue.ValueType jsonType = literalNode.getValueType();
    AvscIssue issue;
    BigInteger bigIntegerValue;
    BigDecimal bigDecimalValue;
    byte[] bytes;
    switch(avroType) {
        case NULL:
            if (jsonType != JsonValue.ValueType.NULL) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroNullLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode)));
        case BOOLEAN:
            if (jsonType != JsonValue.ValueType.FALSE && jsonType != JsonValue.ValueType.TRUE) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            boolean boolValue = jsonType == JsonValue.ValueType.TRUE;
            return new LiteralOrIssue(new AvroBooleanLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), boolValue));
        case INT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt intNode = (JsonNumberExt) literalNode;
            if (!intNode.isIntegral()) {
                // TODO - be more specific about this error (int vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = intNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_INT) > 0 || bigIntegerValue.compareTo(MIN_INT) < 0) {
                // TODO - be more specific about this error (out of signed int range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroIntegerLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.intValueExact()));
        case LONG:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt longNode = (JsonNumberExt) literalNode;
            if (!longNode.isIntegral()) {
                // TODO - be more specific about this error (long vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = longNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_LONG) > 0 || bigIntegerValue.compareTo(MIN_LONG) < 0) {
                // TODO - be more specific about this error (out of signed long range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroLongLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.longValueExact()));
        case FLOAT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt floatNode = (JsonNumberExt) literalNode;
            bigDecimalValue = floatNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_FLOAT) > 0) {
                // TODO - be more specific about this error (out of float range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFloatLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.floatValue()));
        case DOUBLE:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt doubleNode = (JsonNumberExt) literalNode;
            bigDecimalValue = doubleNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_DOUBLE) > 0) {
                // TODO - be more specific about this error (out of double range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroDoubleLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.doubleValue()));
        case BYTES:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt bytesNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = bytesNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            return new LiteralOrIssue(new AvroBytesLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bytes));
        case FIXED:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
            JsonStringExt fixedNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = fixedNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            if (bytes.length != fixedSchema.getSize()) {
                // TODO - be more specific about this error (wrong length)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFixedLiteral(fixedSchema, locationOf(context.getUri(), literalNode), bytes));
        case STRING:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt stringNode = (JsonStringExt) literalNode;
            return new LiteralOrIssue(new AvroStringLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), stringNode.getString()));
        case ENUM:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
            JsonStringExt enumNode = (JsonStringExt) literalNode;
            String enumValue = enumNode.getString();
            if (!enumSchema.getSymbols().contains(enumValue)) {
                // TODO - be more specific about this error (unknown symbol)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroEnumLiteral(enumSchema, locationOf(context.getUri(), literalNode), enumValue));
        case ARRAY:
            if (jsonType != JsonValue.ValueType.ARRAY) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroArraySchema arraySchema = (AvroArraySchema) schema;
            AvroSchema valueSchema = arraySchema.getValueSchema();
            JsonArrayExt arrayNode = (JsonArrayExt) literalNode;
            ArrayList<AvroLiteral> values = new ArrayList<>(arrayNode.size());
            for (int i = 0; i < arrayNode.size(); i++) {
                JsonValueExt valueNode = (JsonValueExt) arrayNode.get(i);
                LiteralOrIssue value = parseLiteral(valueNode, valueSchema, fieldName, context);
                // issues parsing any member value mean a failure to parse the array as a whole
                if (value.getIssue() != null) {
                    // TODO - be more specific about this error (unparsable array element i)
                    // TODO - add "causedBy" to AvscIssue and use it here
                    issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                    context.addIssue(issue);
                    return new LiteralOrIssue(issue);
                }
                values.add(value.getLiteral());
            }
            return new LiteralOrIssue(new AvroArrayLiteral(arraySchema, locationOf(context.getUri(), literalNode), values));
        default:
            throw new UnsupportedOperationException("dont know how to parse a " + avroType + " at " + literalNode.getStartLocation() + " out of a " + literalNode.getValueType() + " (" + literalNode + ")");
    }
}