Examples with IGroupChildTaskIterator com.qlangtech.tis.datax.IGroupChildTaskIterator used on opensource projects

Example 1 with IGroupChildTaskIterator

use of com.qlangtech.tis.datax.IGroupChildTaskIterator in project plugins by qlangtech.

the class TestDataxMySQLReader method testGetSubTasks.

public void testGetSubTasks() {
    MySQLDataSourceFactory mysqlDs = new MySQLDataSourceFactory() {

        // @Override
        // protected Connection getConnection(String jdbcUrl, String username, String password) throws SQLException {
        // throw new UnsupportedOperationException();
        // }
        @Override
        public List<ColumnMetaData> getTableMetadata(String table) {
            switch(table) {
                case TestSelectedTabs.tabNameOrderDetail:
                    return TestSelectedTabs.tabColsMetaOrderDetail;
                case TestSelectedTabs.tabNameTotalpayinfo:
                    return TestSelectedTabs.tabColsMetaTotalpayinfo;
                default:
                    throw new IllegalArgumentException("table:" + table);
            }
        }
    };
    mysqlDs.dbName = dbName;
    mysqlDs.port = 3306;
    mysqlDs.encode = "utf8";
    mysqlDs.userName = userName;
    mysqlDs.password = password;
    mysqlDs.nodeDesc = "192.168.28.200[0-7]";
    Descriptor.ParseDescribable<DataSourceFactory> desc = new Descriptor.ParseDescribable<>(mysqlDs);
    Context context = EasyMock.createMock("context", Context.class);
    EasyMock.expect(context.hasErrors()).andReturn(false);
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    pluginContext.addDb(desc, dbName, context, true);
    EasyMock.replay(pluginContext, context);
    DataSourceFactoryPluginStore dbStore = TIS.getDataBasePluginStore(new PostedDSProp(dbName));
    assertTrue("save mysql db Config faild", dbStore.setPlugins(pluginContext, Optional.of(context), Collections.singletonList(desc)));
    DataxMySQLReader mySQLReader = new DataxMySQLReader() {

        @Override
        public MySQLDataSourceFactory getDataSourceFactory() {
            return mysqlDs;
        }
    };
    mySQLReader.dataXName = this.dataXName;
    List<SelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs();
    mySQLReader.setSelectedTabs(selectedTabs);
    List<SelectedTab> selectedTabs2 = mySQLReader.getSelectedTabs();
    assertEquals(2, selectedTabs2.size());
    for (SelectedTab tab : selectedTabs2) {
        tab.getCols().forEach((c) -> {
            assertNotNull("table:" + tab.getName() + "'s col " + c.getName() + " relevant type can not be null", c.getType());
        });
    }
    List<String> tabs = Lists.newArrayList();
    for (SelectedTab tab : selectedTabs) {
        for (int i = 0; i < 8; i++) {
            tabs.add(tab.name);
        }
    }
    int readerContextCount = 0;
    IDataxReaderContext readerContext = null;
    IGroupChildTaskIterator subTasks = mySQLReader.getSubTasks();
    while (subTasks.hasNext()) {
        readerContext = subTasks.next();
        assertEquals(tabs.get(readerContextCount), readerContext.getSourceEntityName());
        assertEquals(tabs.get(readerContextCount) + "_" + readerContextCount, readerContext.getTaskName());
        System.out.println(readerContext.getSourceEntityName() + " " + readerContext.getTaskName());
        assertNotNull(readerContext);
        readerContextCount++;
    }
    assertEquals(16, readerContextCount);
    Map<String, List<String>> groupedInfo = subTasks.getGroupedInfo();
    assertNotNull("groupedInfo can not be null", groupedInfo);
    List<String> subTabs = groupedInfo.get(TestSelectedTabs.tabNameOrderDetail);
    assertEquals(8, subTabs.size());
    subTabs = groupedInfo.get(TestSelectedTabs.tabNameTotalpayinfo);
    assertEquals(8, subTabs.size());
    EasyMock.verify(pluginContext, context);
}

Example 2 with IGroupChildTaskIterator

use of com.qlangtech.tis.datax.IGroupChildTaskIterator in project plugins by qlangtech.

the class BasicDataXRdbmsReader method getSubTasks.

// @Override
// public DBConfig getDbConfig() {
// return getBasicDataSource().getDbConfig();
// }
// 
// @Override
// public BasicDataSourceFactory getBasicDataSource() {
// return (BasicDataSourceFactory) getDataSourceFactory();
// }
@Override
public final IGroupChildTaskIterator getSubTasks() {
    Objects.requireNonNull(this.selectedTabs, "selectedTabs can not be null");
    DS dsFactory = this.getDataSourceFactory();
    Memoizer<String, Map<String, ColumnMetaData>> tabColsMap = getTabsMeta();
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    ConcurrentHashMap<String, List<String>> groupedInfo = new ConcurrentHashMap();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new IGroupChildTaskIterator() {

        @Override
        public Map<String, List<String>> getGroupedInfo() {
            return groupedInfo;
        }

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                int[] index = { 0 };
                tisTab.setReflectCols(tab.getCols().stream().map((c) -> {
                    return createColumnMetaData(index, c.getName());
                }).collect(Collectors.toList()));
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            String childTask = tab.getName() + "_" + taskIndex.getAndIncrement();
            List<String> childTasks = groupedInfo.computeIfAbsent(tab.getName(), (tabname) -> Lists.newArrayList());
            childTasks.add(childTask);
            RdbmsReaderContext dataxContext = createDataXReaderContext(childTask, tab, dumper);
            dataxContext.setWhere(tab.getWhere());
            if (isFilterUnexistCol()) {
                Map<String, ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
                dataxContext.setCols(tab.cols.stream().filter((c) -> tableMetadata.containsKey(c)).collect(Collectors.toList()));
            } else {
                dataxContext.setCols(tab.cols);
            }
            return dataxContext;
        }
    };
}