use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.
the class CreateSortConfigListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws Exception {
LOGGER.info("Create sort config for context={}", context);
ProcessForm form = context.getProcessForm();
if (form instanceof UpdateGroupProcessForm) {
UpdateGroupProcessForm updateGroupProcessForm = (UpdateGroupProcessForm) form;
OperateType operateType = updateGroupProcessForm.getOperateType();
if (operateType == OperateType.SUSPEND || operateType == OperateType.DELETE) {
return ListenerResult.success();
}
}
InlongGroupInfo groupInfo = this.getGroupInfo(form);
String groupId = groupInfo.getInlongGroupId();
if (StringUtils.isEmpty(groupId)) {
LOGGER.warn("GroupId is null for context={}", context);
return ListenerResult.success();
}
List<SinkResponse> sinkResponseList = streamSinkService.listSink(groupId, null);
if (CollectionUtils.isEmpty(sinkResponseList)) {
LOGGER.warn("Sink not found by groupId={}", groupId);
return ListenerResult.success();
}
Map<String, DataFlowInfo> dataFlowInfoMap = sinkResponseList.stream().map(sink -> {
DataFlowInfo flowInfo = commonOperateService.createDataFlow(groupInfo, sink);
return Pair.of(sink.getInlongStreamId(), flowInfo);
}).collect(Collectors.toMap(Pair::getKey, Pair::getValue));
String dataFlows = OBJECT_MAPPER.writeValueAsString(dataFlowInfoMap);
InlongGroupExtInfo extInfo = new InlongGroupExtInfo();
extInfo.setInlongGroupId(groupId);
extInfo.setKeyName(InlongGroupSettings.DATA_FLOW);
extInfo.setKeyValue(dataFlows);
if (groupInfo.getExtList() == null) {
groupInfo.setExtList(Lists.newArrayList());
}
upsertDataFlow(groupInfo, extInfo);
return ListenerResult.success();
}
use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.
the class MultiTenancyInLongMsgMixedDeserializerTest method testIsInLongMsgDataFlow.
@Test
public void testIsInLongMsgDataFlow() {
final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", ',', false), new FieldInfo[0]);
final EmptySinkInfo sinkInfo = new EmptySinkInfo();
final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
assertTrue(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(dataFlowInfo));
final DataFlowInfo nonInLongMsgDataFlow = new DataFlowInfo(2L, new EmptySourceInfo(), sinkInfo);
assertFalse(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(nonInLongMsgDataFlow));
}
use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.
the class RecordTransformerTest method testRecordMatchSerializer.
@Test
public void testRecordMatchSerializer() throws Exception {
final int bufferSize = 1024;
final RecordTransformer transformer = new RecordTransformer(bufferSize);
final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
transformer.addDataFlow(dataFlowInfo);
Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
final Row row = new Row(2);
row.setField(0, 1024L);
row.setField(1, "9527");
final Record record = new Record(1L, System.currentTimeMillis(), row);
assertSame(record, transformer.matchRecordAndSerializerField(record, rowSerializers.get(1L)));
}
use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.
the class RecordTransformerTest method testRecordNotMatchSerializer.
@Test
public void testRecordNotMatchSerializer() throws Exception {
final int bufferSize = 1024;
final RecordTransformer transformer = new RecordTransformer(bufferSize);
final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
transformer.addDataFlow(dataFlowInfo);
Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
final Row oneFieldRow = new Row(1);
oneFieldRow.setField(0, 1024L);
final Record oneFieldRecord = new Record(1L, System.currentTimeMillis(), oneFieldRow);
assertEquals(2, transformer.matchRecordAndSerializerField(oneFieldRecord, rowSerializers.get(1L)).getRow().getArity());
final Row threeFieldRow = new Row(3);
threeFieldRow.setField(0, 1024L);
threeFieldRow.setField(1, "9527");
threeFieldRow.setField(2, 2048);
final Record threeFieldRecord = new Record(1L, System.currentTimeMillis(), threeFieldRow);
assertEquals(2, transformer.matchRecordAndSerializerField(threeFieldRecord, rowSerializers.get(1L)).getRow().getArity());
}
use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.
the class RecordTransformerTest method testSerializerNotMatchRecord.
@Test
public void testSerializerNotMatchRecord() throws Exception {
final Row row = new Row(2);
row.setField(0, 1024L);
row.setField(1, "9527");
final Record record = new Record(1L, System.currentTimeMillis(), row);
final int bufferSize = 1024;
final RecordTransformer transformer = new RecordTransformer(bufferSize);
final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1 });
final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
transformer.addDataFlow(dataFlowInfo);
Map<Long, RowSerializer> rowSerializers = transformer.getRowSerializers();
assertEquals(1, transformer.matchRecordAndSerializerField(record, rowSerializers.get(1L)).getRow().getArity());
final FieldInfo newField1 = new FieldInfo("field1", new LongFormatInfo());
final FieldInfo newField2 = new FieldInfo("field2", new LongFormatInfo());
final FieldInfo newField3 = new FieldInfo("field3", new LongFormatInfo());
final TestingSinkInfo newSinkInfo = new TestingSinkInfo(new FieldInfo[] { newField1, newField2, newField3 });
final DataFlowInfo newDataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), newSinkInfo);
transformer.addDataFlow(newDataFlowInfo);
Map<Long, RowSerializer> newRowSerializers = transformer.getRowSerializers();
assertEquals(3, transformer.matchRecordAndSerializerField(record, newRowSerializers.get(1L)).getRow().getArity());
}
Aggregations