Examples with Memoizer com.qlangtech.tis.util.Memoizer used on opensource projects

Example 1 with Memoizer

use of com.qlangtech.tis.util.Memoizer in project plugins by qlangtech.

the class DataXHbase11xsqlReader method getSubTasks.

@Override
public Iterator<IDataxReaderContext> getSubTasks() {
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    Memoizer<String, List<ColumnMetaData>> tabColsMap = new Memoizer<String, List<ColumnMetaData>>() {

        @Override
        public List<ColumnMetaData> compute(String tab) {
            return dsFactory.getTableMetadata(tab);
        }
    };
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new Iterator<IDataxReaderContext>() {

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            MySQLDataXReaderContext dataxContext = new MySQLDataXReaderContext(tab.getName() + "_" + taskIndex.getAndIncrement(), tab.getName());
            dataxContext.jdbcUrl = dumper.getDbHost();
            dataxContext.tabName = tab.getName();
            dataxContext.username = dsFactory.getUserName();
            dataxContext.password = dsFactory.getPassword();
            dataxContext.setWhere(tab.getWhere());
            List<ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
            if (tab.isAllCols()) {
                dataxContext.cols = tableMetadata.stream().map((t) -> t.getValue()).collect(Collectors.toList());
            } else {
                dataxContext.cols = tableMetadata.stream().filter((col) -> {
                    return tab.containCol(col.getKey());
                }).map((t) -> t.getValue()).collect(Collectors.toList());
            }
            return dataxContext;
        }
    };
}

Example 2 with Memoizer

use of com.qlangtech.tis.util.Memoizer in project plugins by qlangtech.

the class DataXHbase11xReader method getSubTasks.

@Override
public Iterator<IDataxReaderContext> getSubTasks() {
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    Memoizer<String, List<ColumnMetaData>> tabColsMap = new Memoizer<String, List<ColumnMetaData>>() {

        @Override
        public List<ColumnMetaData> compute(String tab) {
            return dsFactory.getTableMetadata(tab);
        }
    };
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new Iterator<IDataxReaderContext>() {

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            MySQLDataXReaderContext dataxContext = new MySQLDataXReaderContext(tab.getName() + "_" + taskIndex.getAndIncrement(), tab.getName());
            dataxContext.jdbcUrl = dumper.getDbHost();
            dataxContext.tabName = tab.getName();
            dataxContext.username = dsFactory.getUserName();
            dataxContext.password = dsFactory.getPassword();
            dataxContext.setWhere(tab.getWhere());
            List<ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
            if (tab.isAllCols()) {
                dataxContext.cols = tableMetadata.stream().map((t) -> t.getValue()).collect(Collectors.toList());
            } else {
                dataxContext.cols = tableMetadata.stream().filter((col) -> {
                    return tab.containCol(col.getKey());
                }).map((t) -> t.getValue()).collect(Collectors.toList());
            }
            return dataxContext;
        }
    };
}

Example 3 with Memoizer

use of com.qlangtech.tis.util.Memoizer in project plugins by qlangtech.

the class DataXOdpsReader method getSubTasks.

@Override
public Iterator<IDataxReaderContext> getSubTasks() {
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    Memoizer<String, List<ColumnMetaData>> tabColsMap = new Memoizer<String, List<ColumnMetaData>>() {

        @Override
        public List<ColumnMetaData> compute(String tab) {
            return dsFactory.getTableMetadata(tab);
        }
    };
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new Iterator<IDataxReaderContext>() {

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            MySQLDataXReaderContext dataxContext = new MySQLDataXReaderContext(tab.getName() + "_" + taskIndex.getAndIncrement(), tab.getName());
            dataxContext.jdbcUrl = dumper.getDbHost();
            dataxContext.tabName = tab.getName();
            dataxContext.username = dsFactory.getUserName();
            dataxContext.password = dsFactory.getPassword();
            dataxContext.setWhere(tab.getWhere());
            List<ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
            if (tab.isAllCols()) {
                dataxContext.cols = tableMetadata.stream().map((t) -> t.getValue()).collect(Collectors.toList());
            } else {
                dataxContext.cols = tableMetadata.stream().filter((col) -> {
                    return tab.containCol(col.getKey());
                }).map((t) -> t.getValue()).collect(Collectors.toList());
            }
            return dataxContext;
        }
    };
}

Example 4 with Memoizer

use of com.qlangtech.tis.util.Memoizer in project plugins by qlangtech.

the class DataXHbase20xsqlReader method getSubTasks.

@Override
public Iterator<IDataxReaderContext> getSubTasks() {
    MySQLDataSourceFactory dsFactory = (MySQLDataSourceFactory) this.getDataSourceFactory();
    Memoizer<String, List<ColumnMetaData>> tabColsMap = new Memoizer<String, List<ColumnMetaData>>() {

        @Override
        public List<ColumnMetaData> compute(String tab) {
            return dsFactory.getTableMetadata(tab);
        }
    };
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new Iterator<IDataxReaderContext>() {

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            MySQLDataXReaderContext dataxContext = new MySQLDataXReaderContext(tab.getName() + "_" + taskIndex.getAndIncrement(), tab.getName());
            dataxContext.jdbcUrl = dumper.getDbHost();
            dataxContext.tabName = tab.getName();
            dataxContext.username = dsFactory.getUserName();
            dataxContext.password = dsFactory.getPassword();
            dataxContext.setWhere(tab.getWhere());
            List<ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
            if (tab.isAllCols()) {
                dataxContext.cols = tableMetadata.stream().map((t) -> t.getValue()).collect(Collectors.toList());
            } else {
                dataxContext.cols = tableMetadata.stream().filter((col) -> {
                    return tab.containCol(col.getKey());
                }).map((t) -> t.getValue()).collect(Collectors.toList());
            }
            return dataxContext;
        }
    };
}