Examples with DataSourceFactory com.qlangtech.tis.plugin.ds.DataSourceFactory used on opensource projects

Example 1 with DataSourceFactory

use of com.qlangtech.tis.plugin.ds.DataSourceFactory in project plugins by qlangtech.

the class TestDataxMySQLWriter method testTempateGenerate.

public void testTempateGenerate() throws Exception {
    Optional<PluginExtraProps> extraProps = PluginExtraProps.load(DataxMySQLWriter.class);
    assertTrue("DataxMySQLWriter extraProps shall exist", extraProps.isPresent());
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    Context context = EasyMock.createMock("context", Context.class);
    EasyMock.expect(context.hasErrors()).andReturn(false);
    MySQLDataSourceFactory mysqlDs = new MySQLDataSourceFactory() {

        @Override
        public Connection getConnection(String jdbcUrl) throws SQLException {
            return null;
        }
    };
    mysqlDs.dbName = dbWriterName;
    mysqlDs.port = 3306;
    mysqlDs.encode = "utf8";
    mysqlDs.userName = "root";
    mysqlDs.password = "123456";
    mysqlDs.nodeDesc = "192.168.28.200";
    Descriptor.ParseDescribable<DataSourceFactory> desc = new Descriptor.ParseDescribable<>(mysqlDs);
    pluginContext.addDb(desc, dbWriterName, context, true);
    EasyMock.replay(pluginContext, context);
    DataSourceFactoryPluginStore dbStore = TIS.getDataBasePluginStore(new PostedDSProp(dbWriterName));
    assertTrue("save mysql db Config faild", dbStore.setPlugins(pluginContext, Optional.of(context), Collections.singletonList(desc)));
    DataxMySQLWriter mySQLWriter = new DataxMySQLWriter();
    mySQLWriter.dataXName = dataXName;
    mySQLWriter.writeMode = "replace";
    mySQLWriter.dbName = dbWriterName;
    mySQLWriter.template = DataxMySQLWriter.getDftTemplate();
    mySQLWriter.batchSize = 1001;
    mySQLWriter.preSql = "delete from test";
    mySQLWriter.postSql = "delete from test1";
    mySQLWriter.session = "set session sql_mode='ANSI'";
    validateConfigGenerate("mysql-datax-writer-assert.json", mySQLWriter);
    // System.out.println(mySQLWriter.getTemplate());
    // 将非必须输入的值去掉再测试一遍
    mySQLWriter.batchSize = null;
    mySQLWriter.preSql = null;
    mySQLWriter.postSql = null;
    mySQLWriter.session = null;
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
    mySQLWriter.preSql = " ";
    mySQLWriter.postSql = " ";
    mySQLWriter.session = " ";
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
}

Example 2 with DataSourceFactory

use of com.qlangtech.tis.plugin.ds.DataSourceFactory in project plugins by qlangtech.

the class LocalTableDumpFactory method getDataSourceFactory.

private DataSourceFactory getDataSourceFactory(IDumpTable table) {
    if (dataSourceFactoryGetter == null) {
        dataSourceFactoryGetter = (tab) -> {
            IPluginStore<DataSourceFactory> dbPlugin = TIS.getDataBasePluginStore(new PostedDSProp(tab.getDbName(), DbScope.DETAILED));
            Objects.requireNonNull(dbPlugin, "dbPlugin can not be null");
            DataSourceFactory dsFactory = dbPlugin.getPlugin();
            if (dsFactory == null) {
                throw new IllegalStateException("table:" + table + " can not find relevant ds config,config file:" + dbPlugin.getTargetFile());
            }
            return dsFactory;
        };
    }
    return dataSourceFactoryGetter.get(table);
}

Example 3 with DataSourceFactory

use of com.qlangtech.tis.plugin.ds.DataSourceFactory in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method testSingleTableDump.

public void testSingleTableDump() throws Exception {
    LocalTableDumpFactory tableDumpFactory = new LocalTableDumpFactory();
    File dumpRoot = LocalTableDumpFactory.getLocalOfflineRootDir();
    tableDumpFactory.name = "test";
    DataSourceFactory mockEmployeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    tableDumpFactory.setDataSourceFactoryGetter((tab) -> {
        return mockEmployeesDataSource;
    });
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    // search4(.+?)_shard(\d+?)_replica_n(\d+?)
    String mockSolrCore = INDEX_COLLECTION + "_shard1_replica_n1";
    IJoinTaskContext execContext = this.mock("execContext", IJoinTaskContext.class);
    // EntityName targetTableName = EntityName.parse(DB_EMPLOYEES+"."); ctx.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME);
    EasyMock.expect(execContext.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME)).andReturn(getEmployeeTab()).anyTimes();
    replay();
    int round = 0;
    ArrayDeque<Date> createDates = Queues.newArrayDeque();
    // 一共测试5轮
    Date timestamp = null;
    while (round++ < 5) {
        timestamp = new Date();
        createDates.addLast(timestamp);
        TaskContext taskContext = MockTaskContextUtils.create(timestamp);
        taskContext.setCoordinator(zkCoordinator);
        /**
         * -----------------------------------------------------------
         * 开始执行数据导入流程
         *             -----------------------------------------------------------
         */
        startDump(tableDumpFactory, taskContext);
        /**
         * -----------------------------------------------------------
         * 开始执行索引构建流程
         *             -----------------------------------------------------------
         */
        startIndexBuild(mockSolrCore, execContext, zkCoordinator, MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(timestamp));
        Thread.sleep(1000);
    }
    int index = 0;
    File tableRoot = new File(dumpRoot, DB_EMPLOYEES + "/" + TABLE_EMPLOYEES + "/all");
    assertTrue(tableRoot.exists());
    String[] subTimeStampFiles = tableRoot.list();
    Set<String> timestamps = Sets.newHashSet();
    int maxHistorySave = ITableDumpConstant.MAX_PARTITION_SAVE + 1;
    while (index++ < maxHistorySave) {
        timestamps.add(MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(createDates.pollLast()));
    }
    assertEquals("maxHistorySave", maxHistorySave, subTimeStampFiles.length);
    for (String subFile : subTimeStampFiles) {
        assertTrue("shall contain file:" + new File(tableRoot, subFile), timestamps.contains(subFile));
    // TODO 继续校验文件夹中的内容是否正确
    }
    File indexBuildRoot = new File(dumpRoot, INDEX_COLLECTION + "/all/0/output");
    for (String indexBuildRootSub : indexBuildRoot.list()) {
        assertTrue("shall contain file:" + new File(indexBuildRoot, indexBuildRootSub), timestamps.contains(indexBuildRootSub));
    }
    verifyAll();
}

Example 4 with DataSourceFactory

use of com.qlangtech.tis.plugin.ds.DataSourceFactory in project tis by qlangtech.

the class PluginItems method save.

public List<Describable> save(Context context) {
    Objects.requireNonNull(this.pluginContext, "pluginContext can not be null");
    if (items == null) {
        throw new IllegalStateException("prop items can not be null");
    }
    Descriptor.ParseDescribable describable = null;
    AttrValMap attrValMap = null;
    List<Descriptor.ParseDescribable<?>> dlist = Lists.newArrayList();
    List<Describable> describableList = Lists.newArrayList();
    if (this.pluginMeta.isAppend()) {
        IPluginStore pluginStore = heteroEnum.getPluginStore(this.pluginContext, this.pluginMeta);
        if (pluginStore != null) {
            List<Describable> plugins = pluginStore.getPlugins();
            boolean firstSkip = false;
            for (Describable p : plugins) {
                if (!firstSkip) {
                    firstSkip = true;
                    Descriptor.ParseDescribable describablesWithMeta = PluginStore.getDescribablesWithMeta(pluginStore, p);
                    dlist.add(describablesWithMeta);
                } else {
                    dlist.add(new Descriptor.ParseDescribable(p));
                }
            }
        }
    }
    for (int i = 0; i < this.items.size(); i++) {
        attrValMap = this.items.get(i);
        /**
         *====================================================
         * 将客户端post数据包装
         *       ======================================================
         */
        describable = attrValMap.createDescribable(pluginContext);
        dlist.add(describable);
        describableList.add((Describable) describable.instance);
    }
    IPluginStoreSave<?> store = null;
    if (heteroEnum == HeteroEnum.APP_SOURCE) {
        for (Descriptor.ParseDescribable<?> d : dlist) {
            if (d.instance instanceof IdentityName) {
                store = IAppSource.getPluginStore(pluginContext, ((IdentityName) d.instance).identityValue());
                break;
            }
        }
        Objects.requireNonNull(store, "plugin type:" + heteroEnum.getIdentity() + " can not find relevant Store");
    } else if (this.pluginContext.isDataSourceAware()) {
        store = new IPluginStoreSave<DataSourceFactory>() {

            @Override
            public boolean setPlugins(IPluginContext pluginContext, Optional<Context> context, List<Descriptor.ParseDescribable<DataSourceFactory>> dlist, boolean update) {
                for (Descriptor.ParseDescribable<DataSourceFactory> plugin : dlist) {
                    PostedDSProp dbExtraProps = PostedDSProp.parse(pluginMeta);
                    if (DbScope.DETAILED == dbExtraProps.getDbType()) {
                        dbExtraProps.setDbname(plugin.instance.identityValue());
                    }
                    boolean success = TIS.getDataBasePluginStore(dbExtraProps).setPlugins(pluginContext, context, Collections.singletonList(plugin), dbExtraProps.isUpdate());
                    if (!success) {
                        return false;
                    }
                }
                return true;
            }
        };
    } else if (heteroEnum == HeteroEnum.DATAX_WRITER || heteroEnum == HeteroEnum.DATAX_READER) {
        // final String dataxName = pluginMeta.getExtraParam(DataxUtils.DATAX_NAME);
        // if (StringUtils.isEmpty(dataxName)) {
        // String saveDbName = pluginMeta.getExtraParam(DataxUtils.DATAX_DB_NAME);
        // if (StringUtils.isNotBlank(saveDbName)) {
        // store = DataxReader.getPluginStore(this.pluginContext, true, saveDbName);
        // } else {
        // throw new IllegalArgumentException("plugin extra param " + DataxUtils.DATAX_NAME + " can not be null");
        // }
        // } else {
        // KeyedPluginStore<?> keyStore = (heteroEnum == HeteroEnum.DATAX_READER)
        // ? DataxReader.getPluginStore(this.pluginContext, dataxName) : DataxWriter.getPluginStore(this.pluginContext, dataxName);
        // store = keyStore;
        // }
        // if ((heteroEnum == HeteroEnum.DATAX_READER)) {
        // for (Descriptor.ParseDescribable<?> dataXReader : dlist) {
        // DataSourceMeta sourceMeta = (DataSourceMeta) dataXReader.instance;
        // pluginContext.setBizResult(context, sourceMeta.getTablesInDB());
        // }
        // }
        store = HeteroEnum.getDataXReaderAndWriterStore(this.pluginContext, this.heteroEnum == HeteroEnum.DATAX_READER, this.pluginMeta);
        Optional<IPropertyType.SubFormFilter> subFormFilter = pluginMeta.getSubFormFilter();
        if (subFormFilter.isPresent()) {
            IPropertyType.SubFormFilter filter = subFormFilter.get();
            Optional<Descriptor> firstDesc = heteroEnum.descriptors().stream().filter((des) -> filter.match((Descriptor) des)).map((des) -> (Descriptor) des).findFirst();
            if (!firstDesc.isPresent()) {
                throw new IllegalStateException("can not find relevant descriptor:" + filter.uploadPluginMeta.toString());
            }
            // Describable p = keyStore.getPlugin();
            // Descriptor des = p.getDescriptor();
            PluginFormProperties pluginProps = firstDesc.get().getPluginFormPropertyTypes(subFormFilter);
            store = pluginProps.accept(new PluginFormProperties.IVisitor() {

                @Override
                public IPluginStoreSave<?> visit(SuFormProperties props) {
                    // 为了在更新插件时候不把plugin上的@SubForm标记的属性覆盖掉，需要先将老的plugin上的值覆盖到新http post过来的反序列化之后的plugin上
                    // IPluginContext pluginContext, String appname, SuFormProperties subfieldForm, Class<TT> clazz
                    Class<Describable> clazz = (Class<Describable>) heteroEnum.getExtensionPoint();
                    // DataxReader.SubFieldFormAppKey<Describable> key
                    // = new DataxReader.SubFieldFormAppKey<>(pluginContext, dataxName, props, clazz);
                    // 
                    DataxReader.SubFieldFormAppKey<Describable> key = HeteroEnum.createDataXReaderAndWriterRelevant(pluginContext, pluginMeta, new HeteroEnum.DataXReaderAndWriterRelevantCreator<DataxReader.SubFieldFormAppKey<Describable>>() {

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> dbRelevant(IPluginContext pluginContext, String saveDbName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, true, saveDbName, props, clazz);
                        }

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> appRelevant(IPluginContext pluginContext, String dataxName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, false, dataxName, props, clazz);
                        }
                    });
                    return KeyedPluginStore.getPluginStore(key);
                }
            });
        }
    } else if (heteroEnum == HeteroEnum.PARAMS_CONFIG) {
        store = new ParamsConfigPluginStore(this.pluginMeta);
    } else if (heteroEnum == HeteroEnum.DATAX_WORKER) {
        if (!this.pluginContext.isCollectionAware()) {
            throw new IllegalStateException("must be collection aware");
        }
        store = DataXJobWorker.getJobWorkerStore(new TargetResName(this.pluginContext.getCollectionName()));
    // } else if (this.pluginContext.isCollectionAware()) {
    // store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
    } else {
        if (heteroEnum.isAppNameAware()) {
            if (!this.pluginContext.isCollectionAware()) {
                throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be collection aware");
            }
            store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
        } else {
            store = TIS.getPluginStore(heteroEnum.getExtensionPoint());
        }
    // if (heteroEnum.isAppNameAware()) {
    // throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be app name aware");
    // }
    }
    // dlist
    if (!store.setPlugins(pluginContext, Optional.of(context), convert(dlist))) {
        return Collections.emptyList();
    }
    observable.notifyObservers(new PluginItemsSaveEvent(this.pluginContext, this.heteroEnum, describableList));
    return describableList;
}