Examples with SerializedRecord org.apache.inlong.sort.flink.SerializedRecord used on opensource projects

Example 6 with SerializedRecord

use of org.apache.inlong.sort.flink.SerializedRecord in project incubator-inlong by apache.

the class MultiTenancyDeserializer method generateDeserializer.

@VisibleForTesting
Deserializer<SerializedRecord, Record> generateDeserializer(FieldInfo[] fields, DeserializationInfo deserializationInfo) {
    final RowFormatInfo rowFormatInfo = CommonUtils.generateDeserializationRowFormatInfo(fields);
    final Deserializer<SerializedRecord, Record> deserializer;
    if (deserializationInfo instanceof InLongMsgCsvDeserializationInfo) {
        InLongMsgCsvDeserializationInfo inLongMsgCsvDeserializationInfo = (InLongMsgCsvDeserializationInfo) deserializationInfo;
        InLongMsgCsvFormatDeserializer inLongMsgCsvFormatDeserializer = new InLongMsgCsvFormatDeserializer(rowFormatInfo, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, TableFormatConstants.DEFAULT_CHARSET, inLongMsgCsvDeserializationInfo.getDelimiter(), null, null, null, inLongMsgCsvDeserializationInfo.isDeleteHeadDelimiter(), TableFormatConstants.DEFAULT_IGNORE_ERRORS);
        deserializer = new InLongMsgDeserializer(inLongMsgCsvFormatDeserializer);
    } else {
        // TODO, support more formats here
        throw new UnsupportedOperationException("Not supported yet " + deserializationInfo.getClass().getSimpleName());
    }
    return deserializer;
}

Example 7 with SerializedRecord

use of org.apache.inlong.sort.flink.SerializedRecord in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordAndSerializerFieldNotMatch.

@Test
public void testRecordAndSerializerFieldNotMatch() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, 2048);
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    try {
        transformer.toSerializedRecord(record);
        Assert.fail();
    } catch (Exception expected) {
    }
    final FieldInfo newField1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo newField2 = new FieldInfo("field2", new IntFormatInfo());
    final TestingSinkInfo newSinkInfo = new TestingSinkInfo(new FieldInfo[] { newField1, newField2 });
    final DataFlowInfo newDataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), newSinkInfo);
    transformer.updateDataFlow(newDataFlowInfo);
    SerializedRecord serializedRecord = transformer.toSerializedRecord(record);
    transformer.updateDataFlow(dataFlowInfo);
    try {
        transformer.toRecord(serializedRecord);
        Assert.fail();
    } catch (Exception expected) {
    }
}

Example 8 with SerializedRecord

use of org.apache.inlong.sort.flink.SerializedRecord in project incubator-inlong by apache.

the class MultiTopicPulsarSourceFunctionTest method producerMessageToPulsar.

protected void producerMessageToPulsar(String pulsarBrokerUrl) throws Exception {
    RowSerializer rowSerializer = CommonUtils.generateRowSerializer(new RowFormatInfo(new String[] { "f1", "f2" }, new FormatInfo[] { StringFormatInfo.INSTANCE, StringFormatInfo.INSTANCE }));
    DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(1024);
    try (PulsarClient client = PulsarClient.builder().serviceUrl(pulsarBrokerUrl).build();
        Producer<byte[]> producer = client.newProducer().topic(TEST_TOPIC).create()) {
        for (int cnt = 0; cnt < TOTAL_COUNT; cnt++) {
            Row row = Row.of(String.valueOf(cnt), String.valueOf(cnt));
            rowSerializer.serialize(row, dataOutputSerializer);
            ByteArrayOutputStream byt = new ByteArrayOutputStream();
            ObjectOutputStream objectOutputStream = new ObjectOutputStream(byt);
            objectOutputStream.writeObject(new SerializedRecord(1L, 0, dataOutputSerializer.getCopyOfBuffer()));
            producer.send(byt.toByteArray());
        }
    }
}

Example 9 with SerializedRecord

use of org.apache.inlong.sort.flink.SerializedRecord in project incubator-inlong by apache.

the class MultiTopicPulsarSourceFunction method processEvent.

private void processEvent(SourceEvent sourceEvent) throws Exception {
    SourceEventType sourceEventType = sourceEvent.getSourceEventType();
    PulsarSourceInfo pulsarSourceInfo = (PulsarSourceInfo) sourceEvent.getSourceInfo();
    Map<String, Object> properties = sourceEvent.getProperties();
    long dataFlowId = sourceEvent.getDataFlowId();
    switch(sourceEventType) {
        case ADDED:
            PulsarSourceFunction<SerializedRecord> pulsarSourceFunction = generateSourceFunction(dataFlowId, properties, pulsarSourceInfo);
            pulsarConsumer.addPulsarSource(dataFlowId, pulsarSourceFunction);
            break;
        case UPDATE:
            PulsarSourceFunction<SerializedRecord> updateSourceFunction = generateSourceFunction(dataFlowId, properties, pulsarSourceInfo);
            pulsarConsumer.updatePulsarSource(dataFlowId, updateSourceFunction);
            break;
        case REMOVED:
            pulsarConsumer.removePulsarSource(dataFlowId);
            break;
        default:
            LOG.error("Unknown source event type {}", sourceEvent.getSourceEventType());
            throw new RuntimeException("Unknown source event type " + sourceEvent.getSourceEventType());
    }
}

Example 10 with SerializedRecord

use of org.apache.inlong.sort.flink.SerializedRecord in project incubator-inlong by apache.

the class MultiTopicPulsarSourceFunction method generateSourceFunction.

public PulsarSourceFunction<SerializedRecord> generateSourceFunction(long dataFlowId, Map<String, Object> properties, PulsarSourceInfo pulsarSourceInfo) throws Exception {
    org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
    putMapToConfig(config, properties);
    PulsarSourceFunction<SerializedRecord> pulsarSourceFunction = new PulsarSourceFunction<>(pulsarSourceInfo.getAdminUrl(), pulsarSourceInfo.getServiceUrl(), pulsarSourceInfo.getTopic(), pulsarSourceInfo.getSubscriptionName(), pulsarSourceInfo.getAuthentication(), new SerializedRecordDeserializationSchema(dataFlowId), config);
    pulsarSourceFunction.setRuntimeContext(getRuntimeContext());
    pulsarSourceFunction.initializeState(new MultiTenantFunctionInitializationContext(dataFlowId, functionInitializationContext, getRuntimeContext().getExecutionConfig()));
    pulsarSourceFunction.open(config);
    return pulsarSourceFunction;
}