Search in sources :

Example 6 with MQListenerFactory

use of com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory in project plugins by qlangtech.

the class CUDCDCTestSuit method startTest.

public void startTest(MQListenerFactory cdcFactory, String tabName) throws Exception {
    BasicDataXRdbmsReader dataxReader = createDataxReader(dataxName, tabName);
    // replay();
    List<SelectedTab> selectedTabs = dataxReader.getSelectedTabs();
    Optional<SelectedTab> firstSelectedTab = selectedTabs.stream().filter((t) -> tabName.equals(t.name)).findFirst();
    Assert.assertTrue("firstSelectedTab:" + tabName + " must be present", firstSelectedTab.isPresent());
    ISelectedTab tab = firstSelectedTab.get();
    this.cols = Lists.newArrayList();
    HdfsColMeta cMeta = null;
    for (ISelectedTab.ColMeta c : tab.getCols()) {
        cMeta = new HdfsColMeta(c.getName(), c.isNullable(), c.isPk(), c.getType());
        cols.add(cMeta);
    }
    IResultRows consumerHandle = getTestBasicFlinkSourceHandle(tabName);
    cdcFactory.setConsumerHandle(consumerHandle.getConsumerHandle());
    IMQListener<JobExecutionResult> imqListener = cdcFactory.create();
    this.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
    consumerHandle.cancel();
}
Also used : LineIterator(org.apache.commons.io.LineIterator) java.sql(java.sql) StringUtils(org.apache.commons.lang.StringUtils) java.util(java.util) TargetResName(com.qlangtech.tis.coredefine.module.action.TargetResName) Date(java.util.Date) SimpleDateFormat(java.text.SimpleDateFormat) DTO(com.qlangtech.tis.realtime.transfer.DTO) BigDecimal(java.math.BigDecimal) Lists(com.google.common.collect.Lists) ByteArrayInputStream(java.io.ByteArrayInputStream) TisUTF8(com.qlangtech.tis.manage.common.TisUTF8) MQListenerFactory(com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) ParseException(java.text.ParseException) TISSinkFactory(com.qlangtech.tis.plugin.incr.TISSinkFactory) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) BasicDataXRdbmsReader(com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader) RdbmsReaderContext(com.qlangtech.tis.plugin.datax.common.RdbmsReaderContext) com.qlangtech.tis.plugin.ds(com.qlangtech.tis.plugin.ds) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) HdfsColMeta(com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta) IOUtils(org.apache.commons.io.IOUtils) TestBasicFlinkSourceHandle(com.qlangtech.plugins.incr.flink.cdc.source.TestBasicFlinkSourceHandle) SelectedTab(com.qlangtech.tis.plugin.datax.SelectedTab) CloseableIterator(org.apache.flink.util.CloseableIterator) RowKind(org.apache.flink.types.RowKind) Row(org.apache.flink.types.Row) Assert(org.junit.Assert) IMQListener(com.qlangtech.tis.async.message.client.consumer.IMQListener) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) HdfsColMeta(com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta) SelectedTab(com.qlangtech.tis.plugin.datax.SelectedTab) BasicDataXRdbmsReader(com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader)

Example 7 with MQListenerFactory

use of com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory in project plugins by qlangtech.

the class TestPlugin method testSaveAndLoad.

public void testSaveAndLoad() throws IOException {
    // FileUtils.forceMkdir(tmpDir);
    try {
        // IncrComponent incrComponent = createIncrComponent();
        // TIS.get().saveComponent(collection, incrComponent);
        // incrComponent = TIS.get().loadIncrComponent(collection);
        IPluginStore<MQListenerFactory> pluginStore = TIS.getPluginStore(collection, MQListenerFactory.class);
        // incrComponent.getMqListenerFactory();
        List<MQListenerFactory> mqListenerFactory = pluginStore.getPlugins();
        assertEquals(1, mqListenerFactory.size());
        RocketMQListenerFactory rocketMQListenerFactory = (RocketMQListenerFactory) mqListenerFactory.get(0);
        // assertEquals(collection, incrComponent.getCollection());
        assertEquals(consumeId, rocketMQListenerFactory.consumeName);
        assertEquals(MQ_TOPIC, rocketMQListenerFactory.getMqTopic());
        assertEquals(NamesrvAddr, rocketMQListenerFactory.getNamesrvAddr());
        assertNotNull(rocketMQListenerFactory.getDeserialize());
        assertTrue(rocketMQListenerFactory.getDeserialize() instanceof DefaultJSONFormatDeserialize);
    } finally {
    // FileUtils.forceDelete(tmpDir);
    }
}
Also used : DefaultJSONFormatDeserialize(com.qlangtech.async.message.client.to.impl.DefaultJSONFormatDeserialize) RocketMQListenerFactory(com.qlangtech.async.message.client.consumer.RocketMQListenerFactory) MQListenerFactory(com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory) RocketMQListenerFactory(com.qlangtech.async.message.client.consumer.RocketMQListenerFactory)

Aggregations

MQListenerFactory (com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory)7 RocketMQListenerFactory (com.qlangtech.async.message.client.consumer.RocketMQListenerFactory)3 DefaultJSONFormatDeserialize (com.qlangtech.async.message.client.to.impl.DefaultJSONFormatDeserialize)2 IMQListener (com.qlangtech.tis.async.message.client.consumer.IMQListener)2 IDataxProcessor (com.qlangtech.tis.datax.IDataxProcessor)2 HdfsColMeta (com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta)1 JSONObject (com.alibaba.fastjson.JSONObject)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 TestBasicFlinkSourceHandle (com.qlangtech.plugins.incr.flink.cdc.source.TestBasicFlinkSourceHandle)1 IMQConsumerStatusFactory (com.qlangtech.tis.async.message.client.consumer.IMQConsumerStatusFactory)1 TargetResName (com.qlangtech.tis.coredefine.module.action.TargetResName)1 DataxReader (com.qlangtech.tis.datax.impl.DataxReader)1 PluginFormProperties (com.qlangtech.tis.extension.PluginFormProperties)1 PropertyType (com.qlangtech.tis.extension.impl.PropertyType)1 TisUTF8 (com.qlangtech.tis.manage.common.TisUTF8)1 SelectedTab (com.qlangtech.tis.plugin.datax.SelectedTab)1 BasicDataXRdbmsReader (com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader)1 RdbmsReaderContext (com.qlangtech.tis.plugin.datax.common.RdbmsReaderContext)1 com.qlangtech.tis.plugin.ds (com.qlangtech.tis.plugin.ds)1