Examples with MySQLDataSourceFactory com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory used on opensource projects

Example 1 with MySQLDataSourceFactory

use of com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory in project plugins by qlangtech.

the class TestDataxMySQLReader method testGetSubTasks.

public void testGetSubTasks() {
    MySQLDataSourceFactory mysqlDs = new MySQLDataSourceFactory() {

        // @Override
        // protected Connection getConnection(String jdbcUrl, String username, String password) throws SQLException {
        // throw new UnsupportedOperationException();
        // }
        @Override
        public List<ColumnMetaData> getTableMetadata(String table) {
            switch(table) {
                case TestSelectedTabs.tabNameOrderDetail:
                    return TestSelectedTabs.tabColsMetaOrderDetail;
                case TestSelectedTabs.tabNameTotalpayinfo:
                    return TestSelectedTabs.tabColsMetaTotalpayinfo;
                default:
                    throw new IllegalArgumentException("table:" + table);
            }
        }
    };
    mysqlDs.dbName = dbName;
    mysqlDs.port = 3306;
    mysqlDs.encode = "utf8";
    mysqlDs.userName = userName;
    mysqlDs.password = password;
    mysqlDs.nodeDesc = "192.168.28.200[0-7]";
    Descriptor.ParseDescribable<DataSourceFactory> desc = new Descriptor.ParseDescribable<>(mysqlDs);
    Context context = EasyMock.createMock("context", Context.class);
    EasyMock.expect(context.hasErrors()).andReturn(false);
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    pluginContext.addDb(desc, dbName, context, true);
    EasyMock.replay(pluginContext, context);
    DataSourceFactoryPluginStore dbStore = TIS.getDataBasePluginStore(new PostedDSProp(dbName));
    assertTrue("save mysql db Config faild", dbStore.setPlugins(pluginContext, Optional.of(context), Collections.singletonList(desc)));
    DataxMySQLReader mySQLReader = new DataxMySQLReader() {

        @Override
        public MySQLDataSourceFactory getDataSourceFactory() {
            return mysqlDs;
        }
    };
    mySQLReader.dataXName = this.dataXName;
    List<SelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs();
    mySQLReader.setSelectedTabs(selectedTabs);
    List<SelectedTab> selectedTabs2 = mySQLReader.getSelectedTabs();
    assertEquals(2, selectedTabs2.size());
    for (SelectedTab tab : selectedTabs2) {
        tab.getCols().forEach((c) -> {
            assertNotNull("table:" + tab.getName() + "'s col " + c.getName() + " relevant type can not be null", c.getType());
        });
    }
    List<String> tabs = Lists.newArrayList();
    for (SelectedTab tab : selectedTabs) {
        for (int i = 0; i < 8; i++) {
            tabs.add(tab.name);
        }
    }
    int readerContextCount = 0;
    IDataxReaderContext readerContext = null;
    IGroupChildTaskIterator subTasks = mySQLReader.getSubTasks();
    while (subTasks.hasNext()) {
        readerContext = subTasks.next();
        assertEquals(tabs.get(readerContextCount), readerContext.getSourceEntityName());
        assertEquals(tabs.get(readerContextCount) + "_" + readerContextCount, readerContext.getTaskName());
        System.out.println(readerContext.getSourceEntityName() + " " + readerContext.getTaskName());
        assertNotNull(readerContext);
        readerContextCount++;
    }
    assertEquals(16, readerContextCount);
    Map<String, List<String>> groupedInfo = subTasks.getGroupedInfo();
    assertNotNull("groupedInfo can not be null", groupedInfo);
    List<String> subTabs = groupedInfo.get(TestSelectedTabs.tabNameOrderDetail);
    assertEquals(8, subTabs.size());
    subTabs = groupedInfo.get(TestSelectedTabs.tabNameTotalpayinfo);
    assertEquals(8, subTabs.size());
    EasyMock.verify(pluginContext, context);
}

Example 2 with MySQLDataSourceFactory

use of com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory in project plugins by qlangtech.

the class TestDataxMySQLWriter method testTempateGenerate.

public void testTempateGenerate() throws Exception {
    Optional<PluginExtraProps> extraProps = PluginExtraProps.load(DataxMySQLWriter.class);
    assertTrue("DataxMySQLWriter extraProps shall exist", extraProps.isPresent());
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    Context context = EasyMock.createMock("context", Context.class);
    EasyMock.expect(context.hasErrors()).andReturn(false);
    MySQLDataSourceFactory mysqlDs = new MySQLDataSourceFactory() {

        @Override
        public Connection getConnection(String jdbcUrl) throws SQLException {
            return null;
        }
    };
    mysqlDs.dbName = dbWriterName;
    mysqlDs.port = 3306;
    mysqlDs.encode = "utf8";
    mysqlDs.userName = "root";
    mysqlDs.password = "123456";
    mysqlDs.nodeDesc = "192.168.28.200";
    Descriptor.ParseDescribable<DataSourceFactory> desc = new Descriptor.ParseDescribable<>(mysqlDs);
    pluginContext.addDb(desc, dbWriterName, context, true);
    EasyMock.replay(pluginContext, context);
    DataSourceFactoryPluginStore dbStore = TIS.getDataBasePluginStore(new PostedDSProp(dbWriterName));
    assertTrue("save mysql db Config faild", dbStore.setPlugins(pluginContext, Optional.of(context), Collections.singletonList(desc)));
    DataxMySQLWriter mySQLWriter = new DataxMySQLWriter();
    mySQLWriter.dataXName = dataXName;
    mySQLWriter.writeMode = "replace";
    mySQLWriter.dbName = dbWriterName;
    mySQLWriter.template = DataxMySQLWriter.getDftTemplate();
    mySQLWriter.batchSize = 1001;
    mySQLWriter.preSql = "delete from test";
    mySQLWriter.postSql = "delete from test1";
    mySQLWriter.session = "set session sql_mode='ANSI'";
    validateConfigGenerate("mysql-datax-writer-assert.json", mySQLWriter);
    // System.out.println(mySQLWriter.getTemplate());
    // 将非必须输入的值去掉再测试一遍
    mySQLWriter.batchSize = null;
    mySQLWriter.preSql = null;
    mySQLWriter.postSql = null;
    mySQLWriter.session = null;
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
    mySQLWriter.preSql = " ";
    mySQLWriter.postSql = " ";
    mySQLWriter.session = " ";
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
}

Example 3 with MySQLDataSourceFactory

use of com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory in project plugins by qlangtech.

the class DataxMySQLWriter method getSubTask.

@Override
public IDataxContext getSubTask(Optional<IDataxProcessor.TableMap> tableMap) {
    if (!tableMap.isPresent()) {
        throw new IllegalArgumentException("param tableMap shall be present");
    }
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    IDataxProcessor.TableMap tm = tableMap.get();
    if (CollectionUtils.isEmpty(tm.getSourceCols())) {
        throw new IllegalStateException("tablemap " + tm + " source cols can not be null");
    }
    TISTable table = new TISTable();
    table.setTableName(tm.getTo());
    DataDumpers dataDumpers = dsFactory.getDataDumpers(table);
    if (dataDumpers.splitCount > 1) {
        // 写入库还支持多组路由的方式分发，只能向一个目标库中写入
        throw new IllegalStateException("dbSplit can not max than 1");
    }
    MySQLWriterContext context = new MySQLWriterContext(this.dataXName);
    if (dataDumpers.dumpers.hasNext()) {
        IDataSourceDumper next = dataDumpers.dumpers.next();
        context.jdbcUrl = next.getDbHost();
        context.password = dsFactory.password;
        context.username = dsFactory.userName;
        context.tabName = table.getTableName();
        context.cols = IDataxProcessor.TabCols.create(tm);
        context.dbName = this.dbName;
        context.writeMode = this.writeMode;
        context.preSql = this.preSql;
        context.postSql = this.postSql;
        context.session = session;
        context.batchSize = batchSize;
        return context;
    }
    throw new RuntimeException("dbName:" + dbName + " relevant DS is empty");
}

Example 4 with MySQLDataSourceFactory

use of com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory in project plugins by qlangtech.

the class DataXHbase11xsqlReader method getSubTasks.

@Override
public Iterator<IDataxReaderContext> getSubTasks() {
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    Memoizer<String, List<ColumnMetaData>> tabColsMap = new Memoizer<String, List<ColumnMetaData>>() {

        @Override
        public List<ColumnMetaData> compute(String tab) {
            return dsFactory.getTableMetadata(tab);
        }
    };
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new Iterator<IDataxReaderContext>() {

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            MySQLDataXReaderContext dataxContext = new MySQLDataXReaderContext(tab.getName() + "_" + taskIndex.getAndIncrement(), tab.getName());
            dataxContext.jdbcUrl = dumper.getDbHost();
            dataxContext.tabName = tab.getName();
            dataxContext.username = dsFactory.getUserName();
            dataxContext.password = dsFactory.getPassword();
            dataxContext.setWhere(tab.getWhere());
            List<ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
            if (tab.isAllCols()) {
                dataxContext.cols = tableMetadata.stream().map((t) -> t.getValue()).collect(Collectors.toList());
            } else {
                dataxContext.cols = tableMetadata.stream().filter((col) -> {
                    return tab.containCol(col.getKey());
                }).map((t) -> t.getValue()).collect(Collectors.toList());
            }
            return dataxContext;
        }
    };
}

Example 5 with MySQLDataSourceFactory

use of com.qlangtech.tis.plugin.ds.mysql.MySQLDataSourceFactory in project plugins by qlangtech.

the class DataXHbase11xsqlWriter method getSubTask.

@Override
public IDataxContext getSubTask(Optional<IDataxProcessor.TableMap> tableMap) {
    if (!tableMap.isPresent()) {
        throw new IllegalArgumentException("param tableMap shall be present");
    }
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    IDataxProcessor.TableMap tm = tableMap.get();
    if (CollectionUtils.isEmpty(tm.getSourceCols())) {
        throw new IllegalStateException("tablemap " + tm + " source cols can not be null");
    }
    TISTable table = new TISTable();
    table.setTableName(tm.getTo());
    DataDumpers dataDumpers = dsFactory.getDataDumpers(table);
    if (dataDumpers.splitCount > 1) {
        throw new IllegalStateException("dbSplit can not max than 1");
    }
    MySQLWriterContext context = new MySQLWriterContext();
    if (dataDumpers.dumpers.hasNext()) {
        IDataSourceDumper next = dataDumpers.dumpers.next();
        context.jdbcUrl = next.getDbHost();
        context.password = dsFactory.password;
        context.username = dsFactory.userName;
        context.tabName = table.getTableName();
        context.cols = tm.getSourceCols();
        context.dbName = this.dbName;
        context.writeMode = this.writeMode;
        context.preSql = this.preSql;
        context.postSql = this.postSql;
        context.session = session;
        context.batchSize = batchSize;
        return context;
    }
    throw new RuntimeException("dbName:" + dbName + " relevant DS is empty");
}