Examples with DataSourceFactoryPluginStore com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore used on opensource projects

Example 1 with DataSourceFactoryPluginStore

use of com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore in project plugins by qlangtech.

the class TestDataxMySQLWriter method testTempateGenerate.

public void testTempateGenerate() throws Exception {
    Optional<PluginExtraProps> extraProps = PluginExtraProps.load(DataxMySQLWriter.class);
    assertTrue("DataxMySQLWriter extraProps shall exist", extraProps.isPresent());
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    Context context = EasyMock.createMock("context", Context.class);
    EasyMock.expect(context.hasErrors()).andReturn(false);
    MySQLDataSourceFactory mysqlDs = new MySQLDataSourceFactory() {

        @Override
        public Connection getConnection(String jdbcUrl) throws SQLException {
            return null;
        }
    };
    mysqlDs.dbName = dbWriterName;
    mysqlDs.port = 3306;
    mysqlDs.encode = "utf8";
    mysqlDs.userName = "root";
    mysqlDs.password = "123456";
    mysqlDs.nodeDesc = "192.168.28.200";
    Descriptor.ParseDescribable<DataSourceFactory> desc = new Descriptor.ParseDescribable<>(mysqlDs);
    pluginContext.addDb(desc, dbWriterName, context, true);
    EasyMock.replay(pluginContext, context);
    DataSourceFactoryPluginStore dbStore = TIS.getDataBasePluginStore(new PostedDSProp(dbWriterName));
    assertTrue("save mysql db Config faild", dbStore.setPlugins(pluginContext, Optional.of(context), Collections.singletonList(desc)));
    DataxMySQLWriter mySQLWriter = new DataxMySQLWriter();
    mySQLWriter.dataXName = dataXName;
    mySQLWriter.writeMode = "replace";
    mySQLWriter.dbName = dbWriterName;
    mySQLWriter.template = DataxMySQLWriter.getDftTemplate();
    mySQLWriter.batchSize = 1001;
    mySQLWriter.preSql = "delete from test";
    mySQLWriter.postSql = "delete from test1";
    mySQLWriter.session = "set session sql_mode='ANSI'";
    validateConfigGenerate("mysql-datax-writer-assert.json", mySQLWriter);
    // System.out.println(mySQLWriter.getTemplate());
    // 将非必须输入的值去掉再测试一遍
    mySQLWriter.batchSize = null;
    mySQLWriter.preSql = null;
    mySQLWriter.postSql = null;
    mySQLWriter.session = null;
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
    mySQLWriter.preSql = " ";
    mySQLWriter.postSql = " ";
    mySQLWriter.session = " ";
    validateConfigGenerate("mysql-datax-writer-assert-without-option-val.json", mySQLWriter);
}

Example 2 with DataSourceFactoryPluginStore

use of com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore in project tis by qlangtech.

the class ERRules method createDefaultErRule.

/**
 * 使用默认DumpNode创建ERRule并且持久化
 *
 * @param topology
 * @throws Exception
 */
public static void createDefaultErRule(SqlTaskNodeMeta.SqlDataFlowTopology topology) throws Exception {
    // 还没有定义erRule
    DependencyNode dumpNode = topology.getFirstDumpNode();
    DataSourceFactoryPluginStore dsStore = TIS.getDataBasePluginStore(new PostedDSProp(dumpNode.getDbName()));
    TISTable tab = dsStore.loadTableMeta(dumpNode.getName());
    // String topologyName, DependencyNode node, TargetColumnMeta targetColMetas
    Optional<ColumnMetaData> firstPK = tab.getReflectCols().stream().filter((col) -> col.isPk()).findFirst();
    if (!firstPK.isPresent()) {
        throw new IllegalStateException("table:" + dumpNode.parseEntityName() + " can not find relevant PK cols");
    }
    createErRule(topology.getName(), dumpNode, firstPK.get());
}

Example 3 with DataSourceFactoryPluginStore

use of com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore in project tis by qlangtech.

the class DBNode method registerDependencyDbsFacadeConfig.

/**
 * @param collection
 * @param timestamp
 * @return
 * @throws Exception
 */
public static void registerDependencyDbsFacadeConfig(String collection, long timestamp, DefaultListableBeanFactory factory) {
    try {
        Map<String, DataSourceFactoryPluginStore> dbConfigsMap = null;
        try (InputStream input = FileUtils.openInputStream(StreamContextConstant.getDbDependencyConfigMetaFile(collection, timestamp))) {
            // 这样可以去重
            dbConfigsMap = DBNode.load(input).stream().collect(Collectors.toMap((db) -> db.getDbName(), (db) -> TIS.getDataBasePluginStore(new PostedDSProp(db.getDbName(), DbScope.FACADE))));
            FacadeDataSource ds = null;
            for (Map.Entry<String, DataSourceFactoryPluginStore> entry : dbConfigsMap.entrySet()) {
                ds = entry.getValue().createFacadeDataSource();
                factory.registerSingleton(entry.getKey() + "Datasource", ds.dataSource);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 4 with DataSourceFactoryPluginStore

use of com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore in project tis by qlangtech.

the class TIS method deleteDB.

public static void deleteDB(String dbName, DbScope dbScope) {
    try {
        DataSourceFactoryPluginStore dsPluginStore = getDataBasePluginStore(new PostedDSProp(dbName, dbScope));
        dsPluginStore.deleteDB();
        databasePluginStore.clear(dsPluginStore.getDSKey());
    } catch (Exception e) {
        throw new RuntimeException(dbName, e);
    }
}

Example 5 with DataSourceFactoryPluginStore

use of com.qlangtech.tis.plugin.ds.DataSourceFactoryPluginStore in project tis by qlangtech.

the class GenerateDAOAndIncrScript method generateDAOScript.

// private void compileAndPackage(Context context, IControlMsgHandler msgHandler, Map<DBNode, List<String>> dbNameMap, File sourceRoot) throws Exception {
// /**
// * *********************************************************************************
// * 编译增量脚本
// * ***********************************************************************************
// */
// if (this.streamScriptCompile(sourceRoot, dbNameMap.keySet())) {
// msgHandler.addErrorMessage(context, "增量脚本编译失败");
// msgHandler.addFieldError(context, "incr_script_compile_error", "error");
// return;
// }
// /**
// * *********************************************************************************
// * 对scala代码进行 打包
// * ***********************************************************************************
// */
// JavaCompilerProcess.SourceGetterStrategy getterStrategy
// = new JavaCompilerProcess.SourceGetterStrategy(false, "/src/main/scala", ".scala") {
// 
// @Override
// public JavaFileObject.Kind getSourceKind() {
// // 没有scala的类型，暂且用other替换一下
// return JavaFileObject.Kind.OTHER;
// }
// 
// @Override
// public MyJavaFileObject processMyJavaFileObject(MyJavaFileObject fileObj) {
// try {
// try (InputStream input = FileUtils.openInputStream(fileObj.getSourceFile())) {
// IOUtils.copy(input, fileObj.openOutputStream());
// }
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// return fileObj;
// }
// };
// //
// JavaCompilerProcess.FileObjectsContext fileObjects = JavaCompilerProcess.getFileObjects(sourceRoot, getterStrategy);
// final JavaCompilerProcess.FileObjectsContext compiledCodeContext = new JavaCompilerProcess.FileObjectsContext();
// File streamScriptClassesDir = new File(sourceRoot, "classes");
// appendClassFile(streamScriptClassesDir, compiledCodeContext, null);
// // 取得spring配置文件相关resourece
// JavaCompilerProcess.FileObjectsContext xmlConfigs = indexStreamCodeGenerator.getSpringXmlConfigsObjectsContext();
// 
// JavaCompilerProcess.packageJar(
// sourceRoot, StreamContextConstant.getIncrStreamJarName(indexStreamCodeGenerator.collection)
// , fileObjects, compiledCodeContext, xmlConfigs);
// }
private void generateDAOScript(Context context, Map<Integer, Long> dependencyDbs) throws Exception {
    final Map<DBNode, List<String>> dbNameMap = Collections.unmodifiableMap(indexStreamCodeGenerator.getDbTables());
    if (dbNameMap.size() < 1) {
        throw new IllegalStateException("dbNameMap size can not small than 1");
    }
    if (dbNameMap.size() != dependencyDbs.size()) {
        throw new IllegalStateException("dbNameMap.size() " + dbNameMap.size() + " != dependencyDbs.size()" + dependencyDbs.size());
    }
    // long timestampp;// = Long.parseLong(ManageUtils.formatNowYyyyMMddHHmmss());
    DataSourceFactoryPluginStore dbPluginStore = null;
    final KoubeiProgressCallback koubeiProgressCallback = new KoubeiProgressCallback();
    List<IbatorContext> daoFacadeList = Lists.newArrayList();
    Long lastOptime = null;
    List<DataSourceFactoryPluginStore> leakFacadeDsPlugin = Lists.newArrayList();
    for (Map.Entry<DBNode, List<String>> /* dbname */
    entry : dbNameMap.entrySet()) {
        dbPluginStore = getFacadePluginStore(entry);
        if (dbPluginStore.getPlugin() == null) {
            leakFacadeDsPlugin.add(dbPluginStore);
        }
    }
    if (leakFacadeDsPlugin.size() > 0) {
        this.msgHandler.addErrorMessage(context, "数据库:" + leakFacadeDsPlugin.stream().map((p) -> "'" + p.getDSKey().keyVal + "'").collect(Collectors.joining(",")) + "还没有定义对应的Facade数据源");
        return;
    }
    for (Map.Entry<DBNode, List<String>> /* dbname */
    entry : dbNameMap.entrySet()) {
        lastOptime = dependencyDbs.get(entry.getKey().getDbId());
        if (lastOptime == null) {
            throw new IllegalStateException("db " + entry.getKey() + " is not find in dependency dbs:" + dbNameMap.keySet().stream().map((r) -> "[" + r.getDbId() + ":" + r.getDbName() + "]").collect(Collectors.joining(",")));
        }
        long timestamp = lastOptime;
        dbPluginStore = getFacadePluginStore(entry);
        FacadeDataSource facadeDataSource = dbPluginStore.createFacadeDataSource();
        IbatorProperties properties = new IbatorProperties(facadeDataSource, entry.getValue(), timestamp);
        entry.getKey().setTimestampVer(timestamp);
        if (entry.getValue().size() < 1) {
            throw new IllegalStateException("db:" + entry.getKey() + " relevant tablesList can not small than 1");
        }
        KoubeiIbatorRunner runner = new KoubeiIbatorRunner(properties) {

            @Override
            protected KoubeiProgressCallback getProgressCallback() {
                return koubeiProgressCallback;
            }
        };
        IbatorContext ibatorContext = runner.getIbatorContext();
        daoFacadeList.add(ibatorContext);
        try {
            if (!properties.isDaoScriptCreated()) {
                // 生成源代码
                runner.build();
                // dao script 脚本已经创建不需要再创建了
                // if (compilerAndPackage) {
                // 直接生成就行了，别管当前是不是要编译了
                File classpathDir = new File(Config.getDataDir(), "libs/tis-ibatis");
                // File classpathDir = new File("/Users/mozhenghua/Desktop/j2ee_solution/project/tis-ibatis/target/dependency");
                JavaCompilerProcess daoCompilerPackageProcess = new JavaCompilerProcess(facadeDataSource.dbMeta, properties.getDaoDir(), classpathDir);
                // 打包,生成jar包
                daoCompilerPackageProcess.compileAndBuildJar();
            // }
            }
        } catch (Exception e) {
            // 将文件夹清空
            FileUtils.forceDelete(properties.getDaoDir());
            throw new RuntimeException("dao path:" + properties.getDaoDir(), e);
        }
    }
    if (daoFacadeList.size() < 1) {
        throw new IllegalStateException("daoFacadeList can not small than 1");
    }
    daoFacadeList.stream().forEach((r) -> {
        FacadeContext fc = new FacadeContext();
        fc.setFacadeInstanceName(r.getFacadeInstanceName());
        fc.setFullFacadeClassName(r.getFacadeFullClassName());
        fc.setFacadeInterfaceName(r.getFacadeInterface());
        indexStreamCodeGenerator.getFacadeList().add(fc);
    });
// return dbNameMap;
}