Examples with FieldInfo org.apache.inlong.sort.protocol.FieldInfo used on opensource projects

Example 1 with FieldInfo

use of org.apache.inlong.sort.protocol.FieldInfo in project incubator-inlong by apache.

the class SourceInfoUtils method createPulsarSourceInfo.

/**
 * Create source info for Pulsar
 */
private static SourceInfo createPulsarSourceInfo(PulsarClusterInfo pulsarCluster, ClusterBean clusterBean, InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, DeserializationInfo deserializationInfo, List<FieldInfo> fieldInfos) {
    String topicName = streamInfo.getMqResourceObj();
    InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupInfo.getMqExtInfo();
    String tenant = clusterBean.getDefaultTenant();
    if (StringUtils.isNotEmpty(pulsarInfo.getTenant())) {
        tenant = pulsarInfo.getTenant();
    }
    final String namespace = groupInfo.getMqResourceObj();
    // Full name of topic in Pulsar
    final String fullTopicName = "persistent://" + tenant + "/" + namespace + "/" + topicName;
    final String consumerGroup = clusterBean.getAppName() + "_" + topicName + "_consumer_group";
    FieldInfo[] fieldInfosArr = fieldInfos.toArray(new FieldInfo[0]);
    String type = pulsarCluster.getType();
    if (StringUtils.isNotEmpty(type) && Constant.MIDDLEWARE_TDMQ_PULSAR.equals(type)) {
        return new TDMQPulsarSourceInfo(pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, pulsarCluster.getToken(), deserializationInfo, fieldInfosArr);
    } else {
        return new PulsarSourceInfo(pulsarCluster.getAdminUrl(), pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, deserializationInfo, fieldInfosArr, pulsarCluster.getToken());
    }
}

Example 2 with FieldInfo

use of org.apache.inlong.sort.protocol.FieldInfo in project incubator-inlong by apache.

the class FieldMappingRuleTest method init.

@Override
public void init() {
    expectedObject = new FieldMappingRule(new FieldMappingRule.FieldMappingUnit[] { new FieldMappingRule.FieldMappingUnit(new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f2", DoubleFormatInfo.INSTANCE)) });
    expectedJson = "{\n" + "    \"type\":\"field_mapping\",\n" + "    \"field_mapping_units\":[\n" + "        {\n" + "            \"source_field\":{\n" + "                \"type\":\"base\",\n" + "                \"name\":\"f1\",\n" + "                \"format_info\":{\n" + "                    \"type\":\"string\"\n" + "                }\n" + "            },\n" + "            \"sink_field\":{\n" + "                \"type\":\"base\",\n" + "                \"name\":\"f2\",\n" + "                \"format_info\":{\n" + "                    \"type\":\"double\"\n" + "                }\n" + "            }\n" + "        }\n" + "    ]\n" + "}";
    equalObj1 = expectedObject;
    equalObj2 = new FieldMappingRule(new FieldMappingRule.FieldMappingUnit[] { new FieldMappingRule.FieldMappingUnit(new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f2", DoubleFormatInfo.INSTANCE)) });
    unequalObj = new FieldMappingRule(new FieldMappingRule.FieldMappingUnit[] { new FieldMappingRule.FieldMappingUnit(new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f3", DoubleFormatInfo.INSTANCE)) });
}

Example 3 with FieldInfo

use of org.apache.inlong.sort.protocol.FieldInfo in project incubator-inlong by apache.

the class HiveSinkITCase method prepareSinkSchema.

private HiveSinkInfo prepareSinkSchema() {
    final FieldInfo f1 = new FieldInfo(fieldName1, new TimestampFormatInfo("MILLIS"));
    final FieldInfo f2 = new FieldInfo(fieldName2, IntFormatInfo.INSTANCE);
    final FieldInfo f3 = new FieldInfo(fieldName3, StringFormatInfo.INSTANCE);
    final FieldInfo f4 = new FieldInfo(fieldName4, StringFormatInfo.INSTANCE);
    final HiveTimePartitionInfo timePartition = new HiveTimePartitionInfo(f1.getName(), timePartitionFormat);
    final HiveFieldPartitionInfo fieldPartition = new HiveFieldPartitionInfo(f2.getName());
    return new HiveSinkInfo(new FieldInfo[] { f1, f2, f3, f4 }, hiveMetastoreUrl, hiveDb, hiveTable, hiveUsername, hivePassword, dfsSchema + hdfsDataDir, new HivePartitionInfo[] { timePartition, fieldPartition }, new TextFileFormat("\t".charAt(0)));
}

Example 4 with FieldInfo

use of org.apache.inlong.sort.protocol.FieldInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordMatchSerializer.

@Test
public void testRecordMatchSerializer() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, "9527");
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    assertSame(record, transformer.matchRecordAndSerializerField(record, rowSerializers.get(1L)));
}

Example 5 with FieldInfo

use of org.apache.inlong.sort.protocol.FieldInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordNotMatchSerializer.

@Test
public void testRecordNotMatchSerializer() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
    final Row oneFieldRow = new Row(1);
    oneFieldRow.setField(0, 1024L);
    final Record oneFieldRecord = new Record(1L, System.currentTimeMillis(), oneFieldRow);
    assertEquals(2, transformer.matchRecordAndSerializerField(oneFieldRecord, rowSerializers.get(1L)).getRow().getArity());
    final Row threeFieldRow = new Row(3);
    threeFieldRow.setField(0, 1024L);
    threeFieldRow.setField(1, "9527");
    threeFieldRow.setField(2, 2048);
    final Record threeFieldRecord = new Record(1L, System.currentTimeMillis(), threeFieldRow);
    assertEquals(2, transformer.matchRecordAndSerializerField(threeFieldRecord, rowSerializers.get(1L)).getRow().getArity());
}