Examples with DataXDorisWriter com.qlangtech.tis.plugin.datax.doris.DataXDorisWriter used on opensource projects

Example 1 with DataXDorisWriter

use of com.qlangtech.tis.plugin.datax.doris.DataXDorisWriter in project plugins by qlangtech.

the class TestStarRocksSinkFactory method testStartRocksWrite.

public void testStartRocksWrite() throws Exception {
    /**
     *         CREATE TABLE `totalpayinfo` (
     *         `id` varchar(32) NULL COMMENT "",
     *         `entity_id` varchar(10) NULL COMMENT "",
     *         `num` int(11) NULL COMMENT "",
     *         `create_time` bigint(20) NULL COMMENT "",
     *         `update_time` DATETIME   NULL,
     *         `update_date` DATE       NULL,
     *         `start_time`  DATETIME   NULL
     *         ) ENGINE=OLAP
     *         UNIQUE KEY(`id`)
     *         DISTRIBUTED BY HASH(`id`) BUCKETS 10
     *         PROPERTIES (
     *         "replication_num" = "1",
     *         "in_memory" = "false",
     *         "storage_format" = "DEFAULT"
     *         );
     */
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    String updateTime = "update_time";
    String updateDate = "update_date";
    String starTime = "start_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName).times(1);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(updateTime);
    cm.setType(new DataType(Types.TIMESTAMP));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(updateDate);
    cm.setType(new DataType(Types.DATE));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(starTime);
    cm.setType(new DataType(Types.TIMESTAMP));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).times(2);
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXDorisWriter dataXWriter = mock("dataXWriter", DataXDorisWriter.class);
    DorisSourceFactory sourceFactory = new DorisSourceFactory();
    sourceFactory.loadUrl = "[\"192.168.28.201:8030\"]";
    sourceFactory.userName = "root";
    sourceFactory.dbName = "tis";
    // sourceFactory.password = "";
    sourceFactory.port = 9030;
    sourceFactory.nodeDesc = "192.168.28.201";
    EasyMock.expect(dataXWriter.getDataSourceFactory()).andReturn(sourceFactory);
    dataXWriter.initWriterTable(tableName, Collections.singletonList("jdbc:mysql://192.168.28.201:9030/tis"));
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    StarRocksSinkFactory sinkFactory = new StarRocksSinkFactory();
    sinkFactory.columnSeparator = "x01";
    sinkFactory.rowDelimiter = "x02";
    sinkFactory.sinkSemantic = StarRocksSinkSemantic.AT_LEAST_ONCE.getName();
    sinkFactory.sinkBatchFlushInterval = 2000l;
    System.out.println("sinkFactory.sinkBatchFlushInterval:" + sinkFactory.sinkBatchFlushInterval);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFunction = sinkFactory.createSinkFunction(dataxProcessor);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DTO d = new DTO();
    d.setEventType(DTO.EventType.DELETE);
    d.setTableName(tableName);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "88888888887");
    after.put(colCreateTime, "20211113115959");
    after.put(updateTime, "2021-12-17T09:21:20Z");
    after.put(starTime, "2021-12-18 09:21:20");
    after.put(updateDate, "2021-12-9");
    d.setAfter(after);
    assertEquals(1, sinkFunction.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFunction.entrySet()) {
        env.fromElements(new DTO[] { d }).addSink(entry.getValue());
        break;
    }
    env.execute("testJob");
    Thread.sleep(14000);
    this.verifyAll();
}

Example 2 with DataXDorisWriter

use of com.qlangtech.tis.plugin.datax.doris.DataXDorisWriter in project plugins by qlangtech.

the class TestDataXStarRocksWriter method testRealDump.

public void testRealDump() throws Exception {
    String targetTableName = "customer_order_relation";
    String testDataXName = "mysql_doris";
    CreateDorisWriter createDorisWriter = new CreateDorisWriter().invoke();
    createDorisWriter.dsFactory.password = "";
    createDorisWriter.dsFactory.nodeDesc = "192.168.28.201";
    createDorisWriter.writer.autoCreateTable = true;
    DataxProcessor dataXProcessor = EasyMock.mock("dataXProcessor", DataxProcessor.class);
    File createDDLDir = new File(".");
    File createDDLFile = null;
    try {
        createDDLFile = new File(createDDLDir, targetTableName + IDataxProcessor.DATAX_CREATE_DDL_FILE_NAME_SUFFIX);
        FileUtils.write(createDDLFile, com.qlangtech.tis.extension.impl.IOUtils.loadResourceFromClasspath(DataXDorisWriter.class, "create_ddl_customer_order_relation.sql"), TisUTF8.get());
        EasyMock.expect(dataXProcessor.getDataxCreateDDLDir(null)).andReturn(createDDLDir);
        DataxWriter.dataxWriterGetter = (dataXName) -> {
            return createDorisWriter.writer;
        };
        DataxProcessor.processorGetter = (dataXName) -> {
            assertEquals(testDataXName, dataXName);
            return dataXProcessor;
        };
        EasyMock.replay(dataXProcessor);
        // DataXDorisWriter writer = new DataXDorisWriter();
        WriterTemplate.realExecuteDump("starrocks_writer_real_dump.json", createDorisWriter.writer);
        EasyMock.verify(dataXProcessor);
    } finally {
        FileUtils.forceDelete(createDDLFile);
    }
}