Examples with IndexStreamCodeGenerator com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator used on opensource projects

Example 1 with IndexStreamCodeGenerator

use of com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator in project tis by qlangtech.

the class CoreAction method doCompileAndPackage.

/**
 * 编译打包
 *
 * @param context
 * @throws Exception
 */
@Func(value = PermissionConstant.PERMISSION_INCR_PROCESS_CONFIG_EDIT)
public void doCompileAndPackage(Context context) throws Exception {
    IBasicAppSource appSource = IAppSource.load(null, this.getCollectionName());
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(this);
    IndexIncrStatus incrStatus = new IndexIncrStatus();
    GenerateDAOAndIncrScript daoAndIncrScript = new GenerateDAOAndIncrScript(this, indexStreamCodeGenerator);
    // if (appSource.isExcludeFacadeDAOSupport()) {
    if (true) {
        daoAndIncrScript.generateIncrScript(context, incrStatus, true, Collections.emptyMap());
    } else {
        Map<Integer, Long> dependencyDbs = getDependencyDbsMap(this, indexStreamCodeGenerator);
        // 需要facadeDAO支持
        daoAndIncrScript.generate(context, incrStatus, true, dependencyDbs);
    }
}

Example 2 with IndexStreamCodeGenerator

use of com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator in project tis by qlangtech.

the class CoreAction method doDeployIncrSyncChannal.

/**
 * 部署实例
 * https://nightlies.apache.org/flink/flink-docs-release-1.14/docs/deployment/resource-providers/standalone/kubernetes/
 *
 * @param context
 * @throws Exception
 */
@Func(value = PermissionConstant.PERMISSION_INCR_PROCESS_MANAGE)
public void doDeployIncrSyncChannal(Context context) throws Exception {
    // 先进行打包编译
    StringBuffer logger = new StringBuffer("flin sync app:" + this.getCollectionName());
    try {
        TISK8sDelegate k8sClient = TISK8sDelegate.getK8SDelegate(this.getCollectionName());
        k8sClient.checkUseable();
        long start = System.currentTimeMillis();
        this.doCompileAndPackage(context);
        if (context.hasErrors()) {
            return;
        }
        logger.append("\n compile and package consume:" + (System.currentTimeMillis() - start) + "ms ");
        // 编译并且打包
        IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(this);
        // 将打包好的构建，发布到k8s集群中去
        // https://github.com/kubernetes-client/java
        start = System.currentTimeMillis();
        // 通过k8s发布
        k8sClient.deploy(null, indexStreamCodeGenerator.getIncrScriptTimestamp());
        logger.append("\n deploy to flink cluster consume:" + (System.currentTimeMillis() - start) + "ms ");
        IndexIncrStatus incrStatus = new IndexIncrStatus();
        this.setBizResult(context, incrStatus);
    } catch (Exception ex) {
        logger.append("an error occur:" + ex.getMessage());
        throw new TisException(ex.getMessage(), ex);
    } finally {
        log.info(logger.toString());
    }
}

Example 3 with IndexStreamCodeGenerator

use of com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator in project tis by qlangtech.

the class CoreAction method doIncrDelete.

/**
 * 删除增量通道
 *
 * @param context
 * @throws Exception
 */
@Func(value = PermissionConstant.PERMISSION_INCR_PROCESS_MANAGE)
public void doIncrDelete(Context context) throws Exception {
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(this);
    indexStreamCodeGenerator.deleteScript();
    TISK8sDelegate k8sDelegate = TISK8sDelegate.getK8SDelegate(this.getCollectionName());
    // 删除增量实例
    k8sDelegate.removeIncrProcess();
}

Example 4 with IndexStreamCodeGenerator

use of com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator in project tis by qlangtech.

the class CoreAction method generateDAOAndIncrScript.

/**
 * @param module
 * @param context
 * @param
 * @param validateGlobalIncrStreamFactory
 * @param compilerAndPackage
 * @param excludeFacadeDAOSupport         由于单表同步不需要dao支持，可以选择false即可
 * @return
 * @throws Exception
 */
public static IndexIncrStatus generateDAOAndIncrScript(BasicModule module, Context context, boolean validateGlobalIncrStreamFactory, boolean compilerAndPackage, boolean excludeFacadeDAOSupport) throws Exception {
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(module);
    // List<FacadeContext> facadeList = indexStreamCodeGenerator.getFacadeList();
    // PluginStore<IncrStreamFactory> store = TIS.getPluginStore(IncrStreamFactory.class);
    IndexIncrStatus incrStatus = doGetDataXReaderWriterDesc(module.getCollectionName());
    // if (validateGlobalIncrStreamFactory && store.getPlugin() == null) {
    // throw new IllegalStateException("global IncrStreamFactory config can not be null");
    // }
    // if (store.getPlugin() != null) {
    // 已经定义了全局插件
    IPluginStore<IncrStreamFactory> collectionBindIncrStreamFactoryStore = getIncrStreamFactoryStore(module);
    if (collectionBindIncrStreamFactoryStore.getPlugin() == null) {
    // 需要将全局插件属性拷贝到collection绑定的插件属性上来
    // Descriptor flinkStreamDesc = TIS.get().getDescriptor(IncrStreamFactory.FLINK_STREM);
    // if (flinkStreamDesc == null) {
    // throw new IllegalStateException(
    // "can not find findStream Factory in plugin repository, Descriptor ID:" + IncrStreamFactory.FLINK_STREM);
    // }
    // 
    // collectionBindIncrStreamFactoryStore.setPlugins(module, Optional.of(context)
    // , Collections.singletonList(flinkStreamDesc.newInstance(module, Collections.emptyMap(), Optional.empty())));
    // throw new IllegalStateException("collectionName:" + module.getCollectionName() + " relevant plugin can not be null " + IncrStreamFactory.class.getName());
    }
    // }
    // 这里永远是false应该
    incrStatus.setK8sPluginInitialized(false);
    // 判断是否已经成功创建，如果已经创建了，就退出
    if (indexStreamCodeGenerator.isIncrScriptDirCreated()) {
        incrStatus.setIncrScriptCreated(true);
        incrStatus.setIncrScriptMainFileContent(indexStreamCodeGenerator.readIncrScriptMainFileContent());
        log.info("incr script has create ignore it file path:{}", indexStreamCodeGenerator.getIncrScriptDirPath());
    // this.setBizResult(context, incrStatus);
    }
    GenerateDAOAndIncrScript generateDAOAndIncrScript = new GenerateDAOAndIncrScript(module, indexStreamCodeGenerator);
    if (excludeFacadeDAOSupport) {
        generateDAOAndIncrScript.generateIncrScript(context, incrStatus, compilerAndPackage, Collections.emptyMap());
    } else {
        // 需要facadeDAO支持
        generateDAOAndIncrScript.generate(context, incrStatus, compilerAndPackage, getDependencyDbsMap(module, indexStreamCodeGenerator));
    }
    incrStatus.setIncrScriptTimestamp(indexStreamCodeGenerator.getIncrScriptTimestamp());
    return incrStatus;
}

Example 5 with IndexStreamCodeGenerator

use of com.qlangtech.tis.compiler.streamcode.IndexStreamCodeGenerator in project tis by qlangtech.

the class CoreAction method getIndexIncrStatus.

public static IndexIncrStatus getIndexIncrStatus(BasicModule module, boolean getRcConfigInCache) throws Exception {
    IndexIncrStatus incrStatus = doGetDataXReaderWriterDesc(module.getCollectionName());
    // 是否可以取缓存中的deployment信息，在刚删除pod重启之后需要取全新的deployment信息不能缓存
    IPluginStore<IncrStreamFactory> store = getIncrStreamFactoryStore(module);
    if (store.getPlugin() == null) {
        incrStatus.setK8sPluginInitialized(false);
        return incrStatus;
    }
    incrStatus.setK8sPluginInitialized(true);
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(module);
    StreamCodeContext streamCodeContext = new StreamCodeContext(module.getCollectionName(), indexStreamCodeGenerator.incrScriptTimestamp);
    incrStatus.setIncrScriptCreated(streamCodeContext.isIncrScriptDirCreated());
    TISK8sDelegate k8s = TISK8sDelegate.getK8SDelegate(module.getCollectionName());
    IDeploymentDetail rcConfig = k8s.getRcConfig(getRcConfigInCache);
    incrStatus.setK8sReplicationControllerCreated(rcConfig != null);
    if (rcConfig != null) {
        rcConfig.accept(new IDeploymentDetail.IDeploymentDetailVisitor() {

            @Override
            public void visit(RcDeployment rcDeployment) {
                incrStatus.setRcDeployment(rcDeployment);
            }

            @Override
            public void visit(FlinkJobDeploymentDetails details) {
                incrStatus.setFlinkJobDetail(details);
            }
        });
        // JobType.RemoteCallResult<IndexJobRunningStatus> callResult
        // = JobType.QueryIndexJobRunningStatus.assembIncrControlWithResult(
        // getAssembleNodeAddress(module.getSolrZkClient()),
        // module.getCollectionName(), Collections.emptyList(), IndexJobRunningStatus.class);
        // if (callResult.success) {
        incrStatus.setIncrProcess(null);
    // }
    }
    return incrStatus;
}