Examples with InLongMsgMixedSerializedRecord org.apache.inlong.sort.flink.InLongMsgMixedSerializedRecord used on opensource projects

Example 1 with InLongMsgMixedSerializedRecord

use of org.apache.inlong.sort.flink.InLongMsgMixedSerializedRecord in project incubator-inlong by apache.

the class InLongMsgMixedDeserializerTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    final InLongMsgMixedDeserializer mixedDeserializer = new InLongMsgMixedDeserializer();
    mixedDeserializer.updateDataFlow(dataFlowId1, tid, preDeserializer, deserializer);
    mixedDeserializer.updateDataFlow(dataFlowId2, tid, preDeserializer, deserializer);
    final Row row = new Row(3);
    row.setField(0, null);
    row.setField(1, new byte[0]);
    row.setField(2, tid);
    preDeserializer.records.add(row);
    mixedDeserializer.deserialize(new InLongMsgMixedSerializedRecord(), collector);
    assertEquals(2, collector.results.size());
    assertEquals(dataFlowId1, collector.results.get(0).getDataflowId());
    assertEquals(tid, collector.results.get(0).getRow().getField(2));
    assertEquals(dataFlowId2, collector.results.get(1).getDataflowId());
    assertEquals(tid, collector.results.get(1).getRow().getField(2));
}

Example 2 with InLongMsgMixedSerializedRecord

use of org.apache.inlong.sort.flink.InLongMsgMixedSerializedRecord in project incubator-inlong by apache.

the class DeserializationSchema method processElement.

@Override
public void processElement(SerializedRecord serializedRecord, Context context, Collector<SerializedRecord> collector) throws Exception {
    try {
        if (enableOutputMetrics && !config.getString(Constants.SOURCE_TYPE).equals(Constants.SOURCE_TYPE_TUBE)) {
            // If source is tube, we do not output metrics of package number
            final MetricData metricData = new MetricData(// since source could not have side-outputs, so it outputs metrics for source here
            MetricSource.SOURCE, MetricType.SUCCESSFUL, serializedRecord.getTimestampMillis(), serializedRecord.getDataFlowId(), "", 1L);
            context.output(METRIC_DATA_OUTPUT_TAG, metricData);
        }
        final CallbackCollector<Record> transformCollector = new CallbackCollector<>(sourceRecord -> {
            final Record sinkRecord = fieldMappingTransformer.transform(sourceRecord);
            if (enableOutputMetrics) {
                MetricData metricData = new MetricData(// TODO, outputs this metric in Sink Function
                MetricSource.SINK, MetricType.SUCCESSFUL, sinkRecord.getTimestampMillis(), sinkRecord.getDataflowId(), "", 1);
                context.output(METRIC_DATA_OUTPUT_TAG, metricData);
            }
            SerializedRecord serializedSinkRecord = recordTransformer.toSerializedRecord(sinkRecord);
            if (auditImp != null) {
                Pair<String, String> groupIdAndStreamId = inLongGroupIdAndStreamIdMap.getOrDefault(serializedRecord.getDataFlowId(), Pair.of("", ""));
                auditImp.add(Constants.METRIC_AUDIT_ID_FOR_INPUT, groupIdAndStreamId.getLeft(), groupIdAndStreamId.getRight(), sinkRecord.getTimestampMillis(), 1, serializedSinkRecord.getData().length);
            }
            collector.collect(serializedSinkRecord);
        });
        if (serializedRecord instanceof InLongMsgMixedSerializedRecord) {
            final InLongMsgMixedSerializedRecord inlongmsgRecord = (InLongMsgMixedSerializedRecord) serializedRecord;
            synchronized (schemaLock) {
                multiTenancyInLongMsgMixedDeserializer.deserialize(inlongmsgRecord, transformCollector);
            }
        } else {
            synchronized (schemaLock) {
                multiTenancyDeserializer.deserialize(serializedRecord, transformCollector);
            }
        }
    } catch (Exception e) {
        if (enableOutputMetrics && !config.getString(Constants.SOURCE_TYPE).equals(Constants.SOURCE_TYPE_TUBE)) {
            MetricData metricData = new MetricData(MetricSource.DESERIALIZATION, MetricType.ABANDONED, serializedRecord.getTimestampMillis(), serializedRecord.getDataFlowId(), (e.getMessage() == null || e.getMessage().isEmpty()) ? "Exception caught" : e.getMessage(), 1);
            context.output(METRIC_DATA_OUTPUT_TAG, metricData);
        }
        LOG.warn("Abandon data", e);
    }
}

Example 3 with InLongMsgMixedSerializedRecord

use of org.apache.inlong.sort.flink.InLongMsgMixedSerializedRecord in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    final MultiTenancyInLongMsgMixedDeserializer deserializer = new MultiTenancyInLongMsgMixedDeserializer();
    final FieldInfo stringField = new FieldInfo("not_important", new StringFormatInfo());
    final FieldInfo longField = new FieldInfo("id", new LongFormatInfo());
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", '|', false), new FieldInfo[] { stringField, longField });
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    deserializer.addDataFlow(dataFlowInfo);
    final InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    final String attrs = "m=0&" + InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID + "=tid&t=20210513";
    final String body1 = "tianqiwan|29";
    inLongMsg.addMsg(attrs, body1.getBytes());
    final TestingCollector<Record> collector = new TestingCollector<>();
    deserializer.deserialize(new InLongMsgMixedSerializedRecord("topic", 0, inLongMsg.buildArray()), collector);
    assertEquals(1, collector.results.size());
    assertEquals(1L, collector.results.get(0).getDataflowId());
    assertEquals(4, collector.results.get(0).getRow().getArity());
    final long time = new SimpleDateFormat("yyyyMMdd").parse("20210513").getTime();
    assertEquals(new Timestamp(time), collector.results.get(0).getRow().getField(0));
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("m", "0");
    attributes.put(InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID, "tid");
    attributes.put("t", "20210513");
    assertEquals(attributes, collector.results.get(0).getRow().getField(1));
    assertEquals("tianqiwan", collector.results.get(0).getRow().getField(2));
    assertEquals(29L, collector.results.get(0).getRow().getField(3));
}