Search in sources :

Example 1 with LongFormatInfo

use of org.apache.inlong.sort.formats.common.LongFormatInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordMatchSerializer.

@Test
public void testRecordMatchSerializer() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, "9527");
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    assertSame(record, transformer.matchRecordAndSerializerField(record, rowSerializers.get(1L)));
}
Also used : RowSerializer(org.apache.flink.api.java.typeutils.runtime.RowSerializer) SerializedRecord(org.apache.inlong.sort.flink.SerializedRecord) Record(org.apache.inlong.sort.flink.Record) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Row(org.apache.flink.types.Row) EmptySourceInfo(org.apache.inlong.sort.util.TestingUtils.EmptySourceInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo) Test(org.junit.Test)

Example 2 with LongFormatInfo

use of org.apache.inlong.sort.formats.common.LongFormatInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordNotMatchSerializer.

@Test
public void testRecordNotMatchSerializer() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
    final Row oneFieldRow = new Row(1);
    oneFieldRow.setField(0, 1024L);
    final Record oneFieldRecord = new Record(1L, System.currentTimeMillis(), oneFieldRow);
    assertEquals(2, transformer.matchRecordAndSerializerField(oneFieldRecord, rowSerializers.get(1L)).getRow().getArity());
    final Row threeFieldRow = new Row(3);
    threeFieldRow.setField(0, 1024L);
    threeFieldRow.setField(1, "9527");
    threeFieldRow.setField(2, 2048);
    final Record threeFieldRecord = new Record(1L, System.currentTimeMillis(), threeFieldRow);
    assertEquals(2, transformer.matchRecordAndSerializerField(threeFieldRecord, rowSerializers.get(1L)).getRow().getArity());
}
Also used : RowSerializer(org.apache.flink.api.java.typeutils.runtime.RowSerializer) SerializedRecord(org.apache.inlong.sort.flink.SerializedRecord) Record(org.apache.inlong.sort.flink.Record) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Row(org.apache.flink.types.Row) EmptySourceInfo(org.apache.inlong.sort.util.TestingUtils.EmptySourceInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo) Test(org.junit.Test)

Example 3 with LongFormatInfo

use of org.apache.inlong.sort.formats.common.LongFormatInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testSerializerNotMatchRecord.

@Test
public void testSerializerNotMatchRecord() throws Exception {
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, "9527");
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
    assertEquals(1, transformer.matchRecordAndSerializerField(record, rowSerializers.get(1L)).getRow().getArity());
    final FieldInfo newField1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo newField2 = new FieldInfo("field2", new LongFormatInfo());
    final FieldInfo newField3 = new FieldInfo("field3", new LongFormatInfo());
    final TestingSinkInfo newSinkInfo = new TestingSinkInfo(new FieldInfo[] { newField1, newField2, newField3 });
    final DataFlowInfo newDataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), newSinkInfo);
    transformer.addDataFlow(newDataFlowInfo);
    Map<Long, RowSerializer> newRowSerializers = transformer.getRowSerializers();
    assertEquals(3, transformer.matchRecordAndSerializerField(record, newRowSerializers.get(1L)).getRow().getArity());
}
Also used : RowSerializer(org.apache.flink.api.java.typeutils.runtime.RowSerializer) SerializedRecord(org.apache.inlong.sort.flink.SerializedRecord) Record(org.apache.inlong.sort.flink.Record) Row(org.apache.flink.types.Row) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) EmptySourceInfo(org.apache.inlong.sort.util.TestingUtils.EmptySourceInfo) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo) Test(org.junit.Test)

Example 4 with LongFormatInfo

use of org.apache.inlong.sort.formats.common.LongFormatInfo in project incubator-inlong by apache.

the class FieldMappingTransformerTest method testTransform.

@Test
public void testTransform() throws Exception {
    final FieldInfo fieldInfo = new FieldInfo("id", new LongFormatInfo());
    final FieldInfo extraFieldInfo = new FieldInfo("not_important", new StringFormatInfo());
    final SourceInfo sourceInfo = new TestingSourceInfo(new FieldInfo[] { extraFieldInfo, fieldInfo });
    final SinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { extraFieldInfo, fieldInfo });
    final long dataFlowId = 1L;
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(dataFlowId, sourceInfo, sinkInfo);
    final FieldMappingTransformer transformer = new FieldMappingTransformer(new Configuration());
    transformer.addDataFlow(dataFlowInfo);
    // should be 4 fields (2 origin fields + time + attr)
    final Row sourceRow = new Row(2 + SOURCE_FIELD_SKIP_STEP);
    sourceRow.setField(0, System.currentTimeMillis());
    sourceRow.setField(1, "attr");
    sourceRow.setField(2, "not important");
    sourceRow.setField(3, 9527L);
    final Record sourceRecord = new Record(dataFlowId, System.currentTimeMillis(), sourceRow);
    final Record sinkRecord = transformer.transform(sourceRecord);
    assertEquals(dataFlowId, sinkRecord.getDataflowId());
    assertEquals(2, sinkRecord.getRow().getArity());
    assertEquals("not important", sinkRecord.getRow().getField(0));
    assertEquals(9527L, sinkRecord.getRow().getField(1));
}
Also used : TestingSourceInfo(org.apache.inlong.sort.util.TestingUtils.TestingSourceInfo) SourceInfo(org.apache.inlong.sort.protocol.source.SourceInfo) TestingSourceInfo(org.apache.inlong.sort.util.TestingUtils.TestingSourceInfo) Configuration(org.apache.inlong.sort.configuration.Configuration) SinkInfo(org.apache.inlong.sort.protocol.sink.SinkInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Record(org.apache.inlong.sort.flink.Record) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Row(org.apache.flink.types.Row) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) BuiltInFieldInfo(org.apache.inlong.sort.protocol.BuiltInFieldInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo) Test(org.junit.Test)

Example 5 with LongFormatInfo

use of org.apache.inlong.sort.formats.common.LongFormatInfo in project incubator-inlong by apache.

the class FieldMappingTransformerTest method testTransformWithDt.

@Test
public void testTransformWithDt() throws Exception {
    final FieldInfo fieldInfo = new FieldInfo("id", new LongFormatInfo());
    final FieldInfo dtFieldInfo = new BuiltInFieldInfo("dt", new TimestampFormatInfo(), BuiltInField.DATA_TIME);
    final SourceInfo sourceInfo = new TestingSourceInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final SinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final long dataFlowId = 1L;
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(dataFlowId, sourceInfo, sinkInfo);
    final FieldMappingTransformer transformer = new FieldMappingTransformer(new Configuration());
    transformer.addDataFlow(dataFlowInfo);
    // should be 3 fields (1 origin fields + time + attr)
    final Row sourceRow = new Row(1 + SOURCE_FIELD_SKIP_STEP);
    final long dt = System.currentTimeMillis();
    sourceRow.setField(0, dt);
    sourceRow.setField(1, "attr");
    sourceRow.setField(2, 9527L);
    final Record sourceRecord = new Record(dataFlowId, dt, sourceRow);
    final Record sinkRecord = transformer.transform(sourceRecord);
    assertEquals(dataFlowId, sinkRecord.getDataflowId());
    assertEquals(2, sinkRecord.getRow().getArity());
    assertEquals(9527L, sinkRecord.getRow().getField(0));
    assertEquals(new Timestamp(dt), sinkRecord.getRow().getField(1));
}
Also used : TestingSourceInfo(org.apache.inlong.sort.util.TestingUtils.TestingSourceInfo) SourceInfo(org.apache.inlong.sort.protocol.source.SourceInfo) TestingSourceInfo(org.apache.inlong.sort.util.TestingUtils.TestingSourceInfo) Configuration(org.apache.inlong.sort.configuration.Configuration) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) SinkInfo(org.apache.inlong.sort.protocol.sink.SinkInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Timestamp(java.sql.Timestamp) BuiltInFieldInfo(org.apache.inlong.sort.protocol.BuiltInFieldInfo) Record(org.apache.inlong.sort.flink.Record) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) TestingSinkInfo(org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo) Row(org.apache.flink.types.Row) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) BuiltInFieldInfo(org.apache.inlong.sort.protocol.BuiltInFieldInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo) Test(org.junit.Test)

Aggregations

LongFormatInfo (org.apache.inlong.sort.formats.common.LongFormatInfo)9 Record (org.apache.inlong.sort.flink.Record)8 DataFlowInfo (org.apache.inlong.sort.protocol.DataFlowInfo)8 FieldInfo (org.apache.inlong.sort.protocol.FieldInfo)8 Test (org.junit.Test)8 Row (org.apache.flink.types.Row)7 StringFormatInfo (org.apache.inlong.sort.formats.common.StringFormatInfo)7 TestingSinkInfo (org.apache.inlong.sort.util.TestingUtils.TestingSinkInfo)7 SerializedRecord (org.apache.inlong.sort.flink.SerializedRecord)5 EmptySourceInfo (org.apache.inlong.sort.util.TestingUtils.EmptySourceInfo)5 RowSerializer (org.apache.flink.api.java.typeutils.runtime.RowSerializer)3 Timestamp (java.sql.Timestamp)2 Configuration (org.apache.inlong.sort.configuration.Configuration)2 IntFormatInfo (org.apache.inlong.sort.formats.common.IntFormatInfo)2 TimestampFormatInfo (org.apache.inlong.sort.formats.common.TimestampFormatInfo)2 BuiltInFieldInfo (org.apache.inlong.sort.protocol.BuiltInFieldInfo)2 SinkInfo (org.apache.inlong.sort.protocol.sink.SinkInfo)2 SourceInfo (org.apache.inlong.sort.protocol.source.SourceInfo)2 TestingSourceInfo (org.apache.inlong.sort.util.TestingUtils.TestingSourceInfo)2 SimpleDateFormat (java.text.SimpleDateFormat)1