Search in sources :

Example 1 with ShortFormatInfo

use of org.apache.inlong.sort.formats.common.ShortFormatInfo in project incubator-inlong by apache.

the class CommonUtilsTest method testBuildAvroRecordSchemaInJsonForRecursiveFields.

@Test
public void testBuildAvroRecordSchemaInJsonForRecursiveFields() throws IOException {
    FieldInfo[] testFieldInfos = new FieldInfo[] { new FieldInfo("f1", new ArrayFormatInfo(new MapFormatInfo(new StringFormatInfo(), new ArrayFormatInfo(new ArrayFormatInfo(new ShortFormatInfo()))))), new FieldInfo("f2", new MapFormatInfo(new StringFormatInfo(), new MapFormatInfo(new StringFormatInfo(), new RowFormatInfo(new String[] { "f21", "f22" }, new FormatInfo[] { new IntFormatInfo(), new ArrayFormatInfo(new ByteFormatInfo()) })))), new FieldInfo("f3", new RowFormatInfo(new String[] { "f31", "f32" }, new FormatInfo[] { new ArrayFormatInfo(new StringFormatInfo()), new RowFormatInfo(new String[] { "f321", "f322" }, new FormatInfo[] { new ArrayFormatInfo(new IntFormatInfo()), new MapFormatInfo(new StringFormatInfo(), new ArrayFormatInfo(new ByteFormatInfo())) }) })) };
    JsonNode expectedJsonNode = objectMapper.readTree("{\n" + "    \"type\":\"record\",\n" + "    \"name\":\"record\",\n" + "    \"fields\":[\n" + "        {\n" + "            \"name\":\"f1\",\n" + "            \"type\":[\n" + "                \"null\",\n" + "                {\n" + "                    \"type\":\"array\",\n" + "                    \"items\":[\n" + "                        \"null\",\n" + "                        {\n" + "                            \"type\":\"map\",\n" + "                            \"values\":[\n" + "                                \"null\",\n" + "                                {\n" + "                                    \"type\":\"array\",\n" + "                                    \"items\":[\n" + "                                        \"null\",\n" + "                                        {\n" + "                                            \"type\":\"array\",\n" + "                                            \"items\":[\n" + "                                                \"null\",\n" + "                                                \"int\"\n" + "                                            ]\n" + "                                        }\n" + "                                    ]\n" + "                                }\n" + "                            ]\n" + "                        }\n" + "                    ]\n" + "                }\n" + "            ],\n" + "            \"default\":null\n" + "        },\n" + "        {\n" + "            \"name\":\"f2\",\n" + "            \"type\":[\n" + "                \"null\",\n" + "                {\n" + "                    \"type\":\"map\",\n" + "                    \"values\":[\n" + "                        \"null\",\n" + "                        {\n" + "                            \"type\":\"map\",\n" + "                            \"values\":[\n" + "                                \"null\",\n" + "                                {\n" + "                                    \"type\":\"record\",\n" + "                                    \"name\":\"record_f2\",\n" + "                                    \"fields\":[\n" + "                                        {\n" + "                                            \"name\":\"f21\",\n" + "                                            \"type\":[\n" + "                                                \"null\",\n" + "                                                \"int\"\n" + "                                            ],\n" + "                                            \"default\":null\n" + "                                        },\n" + "                                        {\n" + "                                            \"name\":\"f22\",\n" + "                                            \"type\":[\n" + "                                                \"null\",\n" + "                                                {\n" + "                                                    \"type\":\"array\",\n" + "                                                    \"items\":[\n" + "                                                        \"null\",\n" + "                                                        \"int\"\n" + "                                                    ]\n" + "                                                }\n" + "                                            ],\n" + "                                            \"default\":null\n" + "                                        }\n" + "                                    ]\n" + "                                }\n" + "                            ]\n" + "                        }\n" + "                    ]\n" + "                }\n" + "            ],\n" + "            \"default\":null\n" + "        },\n" + "        {\n" + "            \"name\":\"f3\",\n" + "            \"type\":[\n" + "                \"null\",\n" + "                {\n" + "                    \"type\":\"record\",\n" + "                    \"name\":\"record_f3\",\n" + "                    \"fields\":[\n" + "                        {\n" + "                            \"name\":\"f31\",\n" + "                            \"type\":[\n" + "                                \"null\",\n" + "                                {\n" + "                                    \"type\":\"array\",\n" + "                                    \"items\":[\n" + "                                        \"null\",\n" + "                                        \"string\"\n" + "                                    ]\n" + "                                }\n" + "                            ],\n" + "                            \"default\":null\n" + "                        },\n" + "                        {\n" + "                            \"name\":\"f32\",\n" + "                            \"type\":[\n" + "                                \"null\",\n" + "                                {\n" + "                                    \"type\":\"record\",\n" + "                                    \"name\":\"record_f3_f32\",\n" + "                                    \"fields\":[\n" + "                                        {\n" + "                                            \"name\":\"f321\",\n" + "                                            \"type\":[\n" + "                                                \"null\",\n" + "                                                {\n" + "                                                    \"type\":\"array\",\n" + "                                                    \"items\":[\n" + "                                                        \"null\",\n" + "                                                        \"int\"\n" + "                                                    ]\n" + "                                                }\n" + "                                            ],\n" + "                                            \"default\":null\n" + "                                        },\n" + "                                        {\n" + "                                            \"name\":\"f322\",\n" + "                                            \"type\":[\n" + "                                                \"null\",\n" + "                                                {\n" + "                                                    \"type\":\"map\",\n" + "                                                    \"values\":[\n" + "                                                        \"null\",\n" + "                                                        {\n" + "                                                            \"type\":\"array\",\n" + "                                                            \"items\":[\n" + "                                                                \"null\",\n" + "                                                                \"int\"\n" + "                                                            ]\n" + "                                                        }\n" + "                                                    ]\n" + "                                                }\n" + "                                            ],\n" + "                                            \"default\":null\n" + "                                        }\n" + "                                    ]\n" + "                                }\n" + "                            ],\n" + "                            \"default\":null\n" + "                        }\n" + "                    ]\n" + "                }\n" + "            ],\n" + "            \"default\":null\n" + "        }\n" + "    ]\n" + "}");
    String actualJson = buildAvroRecordSchemaInJson(testFieldInfos);
    JsonNode actualJsonNode = objectMapper.readTree(actualJson);
    assertEquals(expectedJsonNode, actualJsonNode);
}
Also used : MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) JsonNode(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) BuiltInFieldInfo(org.apache.inlong.sort.protocol.BuiltInFieldInfo) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) Test(org.junit.Test)

Example 2 with ShortFormatInfo

use of org.apache.inlong.sort.formats.common.ShortFormatInfo in project incubator-inlong by apache.

the class TableFormatUtils method deriveLogicalType.

/**
 * Derive the LogicalType for the given FormatInfo.
 */
public static LogicalType deriveLogicalType(FormatInfo formatInfo) {
    if (formatInfo instanceof StringFormatInfo) {
        return new VarCharType();
    } else if (formatInfo instanceof BooleanFormatInfo) {
        return new BooleanType();
    } else if (formatInfo instanceof ByteFormatInfo) {
        return new TinyIntType();
    } else if (formatInfo instanceof ShortFormatInfo) {
        return new SmallIntType();
    } else if (formatInfo instanceof IntFormatInfo) {
        return new IntType();
    } else if (formatInfo instanceof LongFormatInfo) {
        return new BigIntType();
    } else if (formatInfo instanceof FloatFormatInfo) {
        return new FloatType();
    } else if (formatInfo instanceof DoubleFormatInfo) {
        return new DoubleType();
    } else if (formatInfo instanceof DecimalFormatInfo) {
        return new DecimalType();
    } else if (formatInfo instanceof TimeFormatInfo) {
        return new TimeType();
    } else if (formatInfo instanceof DateFormatInfo) {
        return new DateType();
    } else if (formatInfo instanceof TimestampFormatInfo) {
        return new TimestampType(DEFAULT_PRECISION_FOR_TIMESTAMP);
    } else if (formatInfo instanceof LocalZonedTimestampFormatInfo) {
        return new LocalZonedTimestampType();
    } else if (formatInfo instanceof ArrayFormatInfo) {
        FormatInfo elementFormatInfo = ((ArrayFormatInfo) formatInfo).getElementFormatInfo();
        return new ArrayType(deriveLogicalType(elementFormatInfo));
    } else if (formatInfo instanceof MapFormatInfo) {
        MapFormatInfo mapFormatInfo = (MapFormatInfo) formatInfo;
        FormatInfo keyFormatInfo = mapFormatInfo.getKeyFormatInfo();
        FormatInfo valueFormatInfo = mapFormatInfo.getValueFormatInfo();
        return new MapType(deriveLogicalType(keyFormatInfo), deriveLogicalType(valueFormatInfo));
    } else if (formatInfo instanceof RowFormatInfo) {
        RowFormatInfo rowFormatInfo = (RowFormatInfo) formatInfo;
        FormatInfo[] formatInfos = rowFormatInfo.getFieldFormatInfos();
        int formatInfosSize = formatInfos.length;
        LogicalType[] logicalTypes = new LogicalType[formatInfosSize];
        for (int i = 0; i < formatInfosSize; ++i) {
            logicalTypes[i] = deriveLogicalType(formatInfos[i]);
        }
        return RowType.of(logicalTypes, rowFormatInfo.getFieldNames());
    } else if (formatInfo instanceof BinaryFormatInfo) {
        return new BinaryType();
    } else if (formatInfo instanceof NullFormatInfo) {
        return new NullType();
    } else {
        throw new UnsupportedOperationException();
    }
}
Also used : MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) BigIntType(org.apache.flink.table.types.logical.BigIntType) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) MapType(org.apache.flink.table.types.logical.MapType) TinyIntType(org.apache.flink.table.types.logical.TinyIntType) IntType(org.apache.flink.table.types.logical.IntType) BigIntType(org.apache.flink.table.types.logical.BigIntType) SmallIntType(org.apache.flink.table.types.logical.SmallIntType) FloatType(org.apache.flink.table.types.logical.FloatType) TimeType(org.apache.flink.table.types.logical.TimeType) ArrayType(org.apache.flink.table.types.logical.ArrayType) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) TimestampType(org.apache.flink.table.types.logical.TimestampType) VarCharType(org.apache.flink.table.types.logical.VarCharType) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) DateType(org.apache.flink.table.types.logical.DateType) BinaryFormatInfo(org.apache.inlong.sort.formats.common.BinaryFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) BinaryType(org.apache.flink.table.types.logical.BinaryType) BooleanType(org.apache.flink.table.types.logical.BooleanType) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) TinyIntType(org.apache.flink.table.types.logical.TinyIntType) NullFormatInfo(org.apache.inlong.sort.formats.common.NullFormatInfo) SmallIntType(org.apache.flink.table.types.logical.SmallIntType) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) DoubleType(org.apache.flink.table.types.logical.DoubleType) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) DecimalType(org.apache.flink.table.types.logical.DecimalType) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) NullType(org.apache.flink.table.types.logical.NullType) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) BasicFormatInfo(org.apache.inlong.sort.formats.common.BasicFormatInfo) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) BinaryFormatInfo(org.apache.inlong.sort.formats.common.BinaryFormatInfo) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) NullFormatInfo(org.apache.inlong.sort.formats.common.NullFormatInfo) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo)

Aggregations

ArrayFormatInfo (org.apache.inlong.sort.formats.common.ArrayFormatInfo)2 BooleanFormatInfo (org.apache.inlong.sort.formats.common.BooleanFormatInfo)2 ByteFormatInfo (org.apache.inlong.sort.formats.common.ByteFormatInfo)2 FormatInfo (org.apache.inlong.sort.formats.common.FormatInfo)2 IntFormatInfo (org.apache.inlong.sort.formats.common.IntFormatInfo)2 MapFormatInfo (org.apache.inlong.sort.formats.common.MapFormatInfo)2 RowFormatInfo (org.apache.inlong.sort.formats.common.RowFormatInfo)2 ShortFormatInfo (org.apache.inlong.sort.formats.common.ShortFormatInfo)2 StringFormatInfo (org.apache.inlong.sort.formats.common.StringFormatInfo)2 JsonNode (org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode)1 ArrayType (org.apache.flink.table.types.logical.ArrayType)1 BigIntType (org.apache.flink.table.types.logical.BigIntType)1 BinaryType (org.apache.flink.table.types.logical.BinaryType)1 BooleanType (org.apache.flink.table.types.logical.BooleanType)1 DateType (org.apache.flink.table.types.logical.DateType)1 DecimalType (org.apache.flink.table.types.logical.DecimalType)1 DoubleType (org.apache.flink.table.types.logical.DoubleType)1 FloatType (org.apache.flink.table.types.logical.FloatType)1 IntType (org.apache.flink.table.types.logical.IntType)1 LocalZonedTimestampType (org.apache.flink.table.types.logical.LocalZonedTimestampType)1