Examples with DataFlowInfo org.apache.inlong.sort.protocol.DataFlowInfo used on opensource projects

Example 11 with DataFlowInfo

use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.

the class FieldMappingTransformerTest method testTransformWithDt.

@Test
public void testTransformWithDt() throws Exception {
    final FieldInfo fieldInfo = new FieldInfo("id", new LongFormatInfo());
    final FieldInfo dtFieldInfo = new BuiltInFieldInfo("dt", new TimestampFormatInfo(), BuiltInField.DATA_TIME);
    final SourceInfo sourceInfo = new TestingSourceInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final SinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final long dataFlowId = 1L;
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(dataFlowId, sourceInfo, sinkInfo);
    final FieldMappingTransformer transformer = new FieldMappingTransformer(new Configuration());
    transformer.addDataFlow(dataFlowInfo);
    // should be 3 fields (1 origin fields + time + attr)
    final Row sourceRow = new Row(1 + SOURCE_FIELD_SKIP_STEP);
    final long dt = System.currentTimeMillis();
    sourceRow.setField(0, dt);
    sourceRow.setField(1, "attr");
    sourceRow.setField(2, 9527L);
    final Record sourceRecord = new Record(dataFlowId, dt, sourceRow);
    final Record sinkRecord = transformer.transform(sourceRecord);
    assertEquals(dataFlowId, sinkRecord.getDataflowId());
    assertEquals(2, sinkRecord.getRow().getArity());
    assertEquals(9527L, sinkRecord.getRow().getField(0));
    assertEquals(new Timestamp(dt), sinkRecord.getRow().getField(1));
}

Example 12 with DataFlowInfo

use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testTransformation.

@Test
public void testTransformation() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, "9527");
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    final Record transformed = transformer.toRecord(transformer.toSerializedRecord(record));
    assertEquals(record, transformed);
    // check the buffers
    assertEquals(0, transformer.getDataInputDeserializer().available());
    assertEquals(0, transformer.getDataOutputSerializer().length());
    assertEquals(bufferSize, transformer.getDataOutputSerializer().getSharedBuffer().length);
    transformer.removeDataFlow(dataFlowInfo);
    assertEquals(0, transformer.getRowSerializers().size());
}

Example 13 with DataFlowInfo

use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.

the class RecordTransformerTest method testRecordAndSerializerFieldNotMatch.

@Test
public void testRecordAndSerializerFieldNotMatch() throws Exception {
    final int bufferSize = 1024;
    final RecordTransformer transformer = new RecordTransformer(bufferSize);
    final FieldInfo field1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo field2 = new FieldInfo("field2", new StringFormatInfo());
    final TestingSinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { field1, field2 });
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), sinkInfo);
    transformer.addDataFlow(dataFlowInfo);
    final Row row = new Row(2);
    row.setField(0, 1024L);
    row.setField(1, 2048);
    final Record record = new Record(1L, System.currentTimeMillis(), row);
    try {
        transformer.toSerializedRecord(record);
        Assert.fail();
    } catch (Exception expected) {
    }
    final FieldInfo newField1 = new FieldInfo("field1", new LongFormatInfo());
    final FieldInfo newField2 = new FieldInfo("field2", new IntFormatInfo());
    final TestingSinkInfo newSinkInfo = new TestingSinkInfo(new FieldInfo[] { newField1, newField2 });
    final DataFlowInfo newDataFlowInfo = new DataFlowInfo(1L, new EmptySourceInfo(), newSinkInfo);
    transformer.updateDataFlow(newDataFlowInfo);
    SerializedRecord serializedRecord = transformer.toSerializedRecord(record);
    transformer.updateDataFlow(dataFlowInfo);
    try {
        transformer.toRecord(serializedRecord);
        Assert.fail();
    } catch (Exception expected) {
    }
}

Example 14 with DataFlowInfo

use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.

the class MetaManagerTest method testAddAndUpdateAndRemoveDataFlow.

@Test(timeout = 30000)
public void testAddAndUpdateAndRemoveDataFlow() throws Exception {
    final Configuration config = new Configuration();
    config.setString(Constants.CLUSTER_ID, cluster);
    config.setString(Constants.ZOOKEEPER_QUORUM, ZOOKEEPER.getConnectString());
    config.setString(Constants.ZOOKEEPER_ROOT, zkRoot);
    final MetaManager metaManager = MetaManager.getInstance(config);
    final TestDataFlowInfoListener testDataFlowInfoListener = new TestDataFlowInfoListener();
    metaManager.registerDataFlowInfoListener(testDataFlowInfoListener);
    // add dataFlow
    ZkTools.addDataFlowToCluster(cluster, 1, ZOOKEEPER.getConnectString(), zkRoot);
    ZkTools.updateDataFlowInfo(prepareDataFlowInfo(1), cluster, 1, ZOOKEEPER.getConnectString(), zkRoot);
    // update dataFlow
    ZkTools.updateDataFlowInfo(prepareDataFlowInfo(1), cluster, 1, ZOOKEEPER.getConnectString(), zkRoot);
    // remove dataFlow
    ZkTools.removeDataFlowFromCluster(cluster, 1, ZOOKEEPER.getConnectString(), zkRoot);
    while (true) {
        List<Object> operations = testDataFlowInfoListener.getOperations();
        if (operations.size() == 1) {
            assertTrue(operations.get(0) instanceof DataFlowInfo);
            assertEquals(1, ((DataFlowInfo) operations.get(0)).getId());
        } else if (operations.size() == 2) {
            assertTrue(operations.get(1) instanceof DataFlowInfo);
            assertEquals(1, ((DataFlowInfo) operations.get(0)).getId());
            break;
        } else if (operations.size() == 3) {
            assertTrue(operations.get(2) instanceof DataFlowInfo);
            assertEquals(1, ((DataFlowInfo) operations.get(0)).getId());
            break;
        } else {
            Thread.sleep(100);
        }
    }
}

Example 15 with DataFlowInfo

use of org.apache.inlong.sort.protocol.DataFlowInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    final MultiTenancyInLongMsgMixedDeserializer deserializer = new MultiTenancyInLongMsgMixedDeserializer();
    final FieldInfo stringField = new FieldInfo("not_important", new StringFormatInfo());
    final FieldInfo longField = new FieldInfo("id", new LongFormatInfo());
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", '|', false), new FieldInfo[] { stringField, longField });
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    deserializer.addDataFlow(dataFlowInfo);
    final InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    final String attrs = "m=0&" + InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID + "=tid&t=20210513";
    final String body1 = "tianqiwan|29";
    inLongMsg.addMsg(attrs, body1.getBytes());
    final TestingCollector<Record> collector = new TestingCollector<>();
    deserializer.deserialize(new InLongMsgMixedSerializedRecord("topic", 0, inLongMsg.buildArray()), collector);
    assertEquals(1, collector.results.size());
    assertEquals(1L, collector.results.get(0).getDataflowId());
    assertEquals(4, collector.results.get(0).getRow().getArity());
    final long time = new SimpleDateFormat("yyyyMMdd").parse("20210513").getTime();
    assertEquals(new Timestamp(time), collector.results.get(0).getRow().getField(0));
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("m", "0");
    attributes.put(InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID, "tid");
    attributes.put("t", "20210513");
    assertEquals(attributes, collector.results.get(0).getRow().getField(1));
    assertEquals("tianqiwan", collector.results.get(0).getRow().getField(2));
    assertEquals(29L, collector.results.get(0).getRow().getField(3));
}