Examples with IncrStreamFactory com.qlangtech.tis.plugin.incr.IncrStreamFactory used on opensource projects

Example 1 with IncrStreamFactory

use of com.qlangtech.tis.plugin.incr.IncrStreamFactory in project plugins by qlangtech.

the class TISFlinkCDCStart method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new IllegalArgumentException("args length must be 1,now is:" + args.length);
    }
    String dataxName = args[0];
    // -classpath /Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-flink-dependency/target/tis-flink-dependency/WEB-INF/lib/*:/Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-flink-cdc-plugin/target/tis-flink-cdc-plugin/WEB-INF/lib/*:/Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-elasticsearch7-sink-plugin/target/tis-elasticsearch7-sink-plugin/WEB-INF/lib/*:/Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-realtime-flink/target/tis-realtime-flink/WEB-INF/lib/*:/Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-realtime-flink-launch/target/tis-realtime-flink-launch.jar:/Users/mozhenghua/j2ee_solution/project/plugins/tis-incr/tis-realtime-flink-launch/target/dependency/*:/Users/mozhenghua/j2ee_solution/project/plugins/tis-datax/tis-datax-elasticsearch-plugin/target/tis-datax-elasticsearch-plugin/WEB-INF/lib/*:
    // CenterResource.setNotFetchFromCenterRepository();
    // Thread.currentThread().setContextClassLoader(TIS.get().pluginManager.uberClassLoader);
    // IPluginContext pluginContext = IPluginContext.namedContext(dataxName);
    // 
    // 
    // List<IncrStreamFactory> streamFactories = HeteroEnum.INCR_STREAM_CONFIG.getPlugins(pluginContext, null);
    // IRCController incrController = null;
    // for (IncrStreamFactory factory : streamFactories) {
    // incrController = factory.getIncrSync();
    // }
    // Objects.requireNonNull(incrController, "stream app:" + dataxName + " incrController can not not be null");
    IncrStreamFactory incrStreamFactory = HeteroEnum.getIncrStreamFactory(dataxName);
    BasicFlinkSourceHandle tableStreamHandle = createFlinkSourceHandle(dataxName);
    tableStreamHandle.setStreamFactory(incrStreamFactory);
    deploy(new TargetResName(dataxName), tableStreamHandle, null, -1);
}

Example 2 with IncrStreamFactory

use of com.qlangtech.tis.plugin.incr.IncrStreamFactory in project tis by qlangtech.

the class CollectionAction method doDeleteIndex.

/**
 * 删除索引实例
 *
 * @param context
 * @throws Exception
 */
public void doDeleteIndex(Context context) throws Exception {
    getIndexWithPost();
    // 删除
    Application app = this.getApplicationDAO().selectByName(this.indexName.getCollectionName());
    if (app == null) {
        throw new IllegalStateException("indexName:" + this.indexName.getCollectionName() + " relevant instance in db can not be empty");
    }
    // final WorkFlow workFlow = this.loadDF(app.getWorkFlowId());
    this.rescycleAppDB(app.getAppId());
    // this.getWorkflowDAOFacade().getWorkFlowDAO().deleteByPrimaryKey(workFlow.getId());
    WorkFlowBuildHistoryCriteria wfHistoryCriteria = new WorkFlowBuildHistoryCriteria();
    // wfHistoryCriteria.createCriteria().andWorkFlowIdEqualTo(workFlow.getId());
    this.getWorkflowDAOFacade().getWorkFlowBuildHistoryDAO().deleteByExample(wfHistoryCriteria);
    this.deleteCollectionInCloud(context, app.getProjectName());
    // 删除workflow数据库及本地存储文件
    // TopologyDir topologyDir = SqlTaskNodeMeta.getTopologyDir(workFlow.getName());
    // if (topologyDir.synchronizeSubRemoteRes().size() > 0) {
    // IndexStreamCodeGenerator indexStreamCodeGenerator
    // = CoreAction.getIndexStreamCodeGenerator(this, workFlow, false);
    // indexStreamCodeGenerator.deleteScript();
    // }
    // topologyDir.delete();
    IPluginStore<IncrStreamFactory> store = CoreAction.getIncrStreamFactoryStore(this);
    try {
        if (store.getPlugin() != null) {
            // 删除增量实例
            TISK8sDelegate k8sDelegate = TISK8sDelegate.getK8SDelegate(this.getCollectionName());
            k8sDelegate.removeIncrProcess();
        }
    } catch (Throwable e) {
        // 可能创建过程增量没有正常，不能导致删除失败
        logger.warn("k8sDelegate illegal", e);
    }
}

Example 3 with IncrStreamFactory

use of com.qlangtech.tis.plugin.incr.IncrStreamFactory in project tis by qlangtech.

the class CoreAction method generateDAOAndIncrScript.

/**
 * @param module
 * @param context
 * @param
 * @param validateGlobalIncrStreamFactory
 * @param compilerAndPackage
 * @param excludeFacadeDAOSupport         由于单表同步不需要dao支持，可以选择false即可
 * @return
 * @throws Exception
 */
public static IndexIncrStatus generateDAOAndIncrScript(BasicModule module, Context context, boolean validateGlobalIncrStreamFactory, boolean compilerAndPackage, boolean excludeFacadeDAOSupport) throws Exception {
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(module);
    // List<FacadeContext> facadeList = indexStreamCodeGenerator.getFacadeList();
    // PluginStore<IncrStreamFactory> store = TIS.getPluginStore(IncrStreamFactory.class);
    IndexIncrStatus incrStatus = doGetDataXReaderWriterDesc(module.getCollectionName());
    // if (validateGlobalIncrStreamFactory && store.getPlugin() == null) {
    // throw new IllegalStateException("global IncrStreamFactory config can not be null");
    // }
    // if (store.getPlugin() != null) {
    // 已经定义了全局插件
    IPluginStore<IncrStreamFactory> collectionBindIncrStreamFactoryStore = getIncrStreamFactoryStore(module);
    if (collectionBindIncrStreamFactoryStore.getPlugin() == null) {
    // 需要将全局插件属性拷贝到collection绑定的插件属性上来
    // Descriptor flinkStreamDesc = TIS.get().getDescriptor(IncrStreamFactory.FLINK_STREM);
    // if (flinkStreamDesc == null) {
    // throw new IllegalStateException(
    // "can not find findStream Factory in plugin repository, Descriptor ID:" + IncrStreamFactory.FLINK_STREM);
    // }
    // 
    // collectionBindIncrStreamFactoryStore.setPlugins(module, Optional.of(context)
    // , Collections.singletonList(flinkStreamDesc.newInstance(module, Collections.emptyMap(), Optional.empty())));
    // throw new IllegalStateException("collectionName:" + module.getCollectionName() + " relevant plugin can not be null " + IncrStreamFactory.class.getName());
    }
    // }
    // 这里永远是false应该
    incrStatus.setK8sPluginInitialized(false);
    // 判断是否已经成功创建，如果已经创建了，就退出
    if (indexStreamCodeGenerator.isIncrScriptDirCreated()) {
        incrStatus.setIncrScriptCreated(true);
        incrStatus.setIncrScriptMainFileContent(indexStreamCodeGenerator.readIncrScriptMainFileContent());
        log.info("incr script has create ignore it file path:{}", indexStreamCodeGenerator.getIncrScriptDirPath());
    // this.setBizResult(context, incrStatus);
    }
    GenerateDAOAndIncrScript generateDAOAndIncrScript = new GenerateDAOAndIncrScript(module, indexStreamCodeGenerator);
    if (excludeFacadeDAOSupport) {
        generateDAOAndIncrScript.generateIncrScript(context, incrStatus, compilerAndPackage, Collections.emptyMap());
    } else {
        // 需要facadeDAO支持
        generateDAOAndIncrScript.generate(context, incrStatus, compilerAndPackage, getDependencyDbsMap(module, indexStreamCodeGenerator));
    }
    incrStatus.setIncrScriptTimestamp(indexStreamCodeGenerator.getIncrScriptTimestamp());
    return incrStatus;
}

Example 4 with IncrStreamFactory

use of com.qlangtech.tis.plugin.incr.IncrStreamFactory in project tis by qlangtech.

the class CoreAction method doRelaunchIncrProcess.

/**
 * 重新启动增量执行进程
 *
 * @param context
 * @throws Exception
 */
public void doRelaunchIncrProcess(Context context) throws Exception {
    IPluginStore<IncrStreamFactory> incrStreamStore = getIncrStreamFactoryStore(this, true);
    IncrStreamFactory incrStream = incrStreamStore.getPlugin();
    IRCController incrSync = incrStream.getIncrSync();
    incrSync.relaunch(new TargetResName(this.getCollectionName()));
}

Example 5 with IncrStreamFactory

use of com.qlangtech.tis.plugin.incr.IncrStreamFactory in project tis by qlangtech.

the class CoreAction method getIndexIncrStatus.

public static IndexIncrStatus getIndexIncrStatus(BasicModule module, boolean getRcConfigInCache) throws Exception {
    IndexIncrStatus incrStatus = doGetDataXReaderWriterDesc(module.getCollectionName());
    // 是否可以取缓存中的deployment信息，在刚删除pod重启之后需要取全新的deployment信息不能缓存
    IPluginStore<IncrStreamFactory> store = getIncrStreamFactoryStore(module);
    if (store.getPlugin() == null) {
        incrStatus.setK8sPluginInitialized(false);
        return incrStatus;
    }
    incrStatus.setK8sPluginInitialized(true);
    IndexStreamCodeGenerator indexStreamCodeGenerator = getIndexStreamCodeGenerator(module);
    StreamCodeContext streamCodeContext = new StreamCodeContext(module.getCollectionName(), indexStreamCodeGenerator.incrScriptTimestamp);
    incrStatus.setIncrScriptCreated(streamCodeContext.isIncrScriptDirCreated());
    TISK8sDelegate k8s = TISK8sDelegate.getK8SDelegate(module.getCollectionName());
    IDeploymentDetail rcConfig = k8s.getRcConfig(getRcConfigInCache);
    incrStatus.setK8sReplicationControllerCreated(rcConfig != null);
    if (rcConfig != null) {
        rcConfig.accept(new IDeploymentDetail.IDeploymentDetailVisitor() {

            @Override
            public void visit(RcDeployment rcDeployment) {
                incrStatus.setRcDeployment(rcDeployment);
            }

            @Override
            public void visit(FlinkJobDeploymentDetails details) {
                incrStatus.setFlinkJobDetail(details);
            }
        });
        // JobType.RemoteCallResult<IndexJobRunningStatus> callResult
        // = JobType.QueryIndexJobRunningStatus.assembIncrControlWithResult(
        // getAssembleNodeAddress(module.getSolrZkClient()),
        // module.getCollectionName(), Collections.emptyList(), IndexJobRunningStatus.class);
        // if (callResult.success) {
        incrStatus.setIncrProcess(null);
    // }
    }
    return incrStatus;
}