Examples with ParamsConfig com.qlangtech.tis.config.ParamsConfig used on opensource projects

Example 1 with ParamsConfig

use of com.qlangtech.tis.config.ParamsConfig in project plugins by qlangtech.

the class Hadoop020RemoteJobTriggerFactory method getRemoteJobTrigger.

private IRemoteTaskTrigger getRemoteJobTrigger(String name, String startClassName, String appType, JobConfParams launcherParam) throws IOException, YarnException {
    ParamsConfig pConfig = (ParamsConfig) this.yarnConfig;
    YarnConfiguration yarnConfig = pConfig.createConfigInstance();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(yarnConfig);
    yarnClient.start();
    YarnClientApplication app = yarnClient.createApplication();
    ApplicationSubmissionContext submissionContext = app.getApplicationSubmissionContext();
    submissionContext.setApplicationType(appType);
    submissionContext.setMaxAppAttempts(2);
    submissionContext.setKeepContainersAcrossApplicationAttempts(false);
    final ApplicationId appid = submissionContext.getApplicationId();
    submissionContext.setApplicationName(name);
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    // 可以设置javaHome 留给以后扩展
    // "/usr/lib/java/jdk1.8.0_91";
    final String JAVA_HOME = "";
    String javaCommand = StringUtils.isEmpty(JAVA_HOME) ? "java" : (JAVA_HOME + "/bin/java ");
    final int memoryConsume = podSpec.getMaxHeapMemory();
    logger.info("index build podSpec MaxHeapMemory:{},MaxCPUCores:{}", memoryConsume, podSpec.getMaxCPUCores());
    final String command = javaCommand + getMemorySpec(memoryConsume) + getRemoteDebugParam() + " -cp " + getClasspath() + " " + startClassName + " " + launcherParam.paramSerialize() + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
    logger.info("\n=============================================\n" + command + "\n=============================================");
    amContainer.setCommands(Collections.singletonList(command));
    submissionContext.setAMContainerSpec(amContainer);
    // 使用4核10G的节点，原则上越大越好
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memoryConsume);
    capability.setVirtualCores(podSpec.getMaxCPUCores());
    // submissionContext.setNodeLabelExpression(nodeLabelExpression);
    submissionContext.setResource(capability);
    submissionContext.setQueue("default");
    Priority p = Records.newRecord(Priority.class);
    p.setPriority(2);
    submissionContext.setPriority(p);
    return new IRemoteTaskTrigger() {

        @Override
        public void run() {
            try {
                yarnClient.submitApplication(submissionContext);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public String getTaskName() {
            throw new UnsupportedOperationException();
        }

        @Override
        public RunningStatus getRunningStatus() {
            try {
                ApplicationReport appReport = yarnClient.getApplicationReport(appid);
                YarnApplicationState appState = appReport.getYarnApplicationState();
                FinalApplicationStatus finalStatus = appReport.getFinalApplicationStatus();
                if (appState != YarnApplicationState.RUNNING && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED && appState != YarnApplicationState.FINISHED) {
                    logger.info("waitting:" + name + " ,build task wait launch,current:" + appState);
                    // 等待任务开始
                    return new RunningStatus(0, false, false);
                }
                if (appState == YarnApplicationState.RUNNING) {
                    // 正在运行
                    return new RunningStatus(appReport.getProgress(), false, false);
                }
                if (appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED || finalStatus != FinalApplicationStatus.SUCCEEDED) {
                    logger.error("slice:" + name + " ,build result:" + appState + "\n finalStatus:" + finalStatus + "\ndiagnostics:" + appReport.getDiagnostics());
                    // 完成了，但是失败了
                    return new RunningStatus(appReport.getProgress(), true, false);
                } else {
                    logger.info("core:" + name + " app (" + appid + ") is " + appState);
                    return new RunningStatus(appReport.getProgress(), true, true);
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 2 with ParamsConfig

use of com.qlangtech.tis.config.ParamsConfig in project plugins by qlangtech.

the class TestDataXGlobalConfig method testDescriptJSONGenerate.

public void testDescriptJSONGenerate() {
    DataXGlobalConfig globalConfig = new DataXGlobalConfig();
    Descriptor<ParamsConfig> descriptor = globalConfig.getDescriptor();
    assertNotNull(descriptor);
    List<Descriptor<ParamsConfig>> singleton = Collections.singletonList(descriptor);
    DescriptorsJSON descriptorsJSON = new DescriptorsJSON(singleton);
    JSON.parseObject(IOUtils.loadResourceFromClasspath(TestDataXGlobalConfig.class, "dataXGlobalConfig-descriptor-assert.json"));
    // System.out.println(descriptorsJSON.getDescriptorsJSON().toJSONString());
    assertEquals(JSON.parseObject(IOUtils.loadResourceFromClasspath(TestDataXGlobalConfig.class, "dataXGlobalConfig-descriptor-assert.json")).toJSONString(), descriptorsJSON.getDescriptorsJSON().toJSONString());
}

Example 3 with ParamsConfig

use of com.qlangtech.tis.config.ParamsConfig in project tis by qlangtech.

the class TestParamsConfigPluginStore method testWriteAndGet.

public void testWriteAndGet() {
    UploadPluginMeta pluginMeta = UploadPluginMeta.parse(UploadPluginMeta.KEY_TARGET_PLUGIN_DESC + "_test1");
    ParamsConfigPluginStore paramsCfgPluginStore = new ParamsConfigPluginStore(pluginMeta);
    List<Descriptor.ParseDescribable<ParamsConfig>> dlist = Lists.newArrayList();
    Test1ParamsConfig cfg1 = new Test1ParamsConfig();
    cfg1.name = "id1";
    Descriptor.ParseDescribable<ParamsConfig> pluginDesc1 = new Descriptor.ParseDescribable<>(cfg1);
    pluginDesc1.extraPluginMetas.add(new XStream2.PluginMeta("test1meta", "1.0.0"));
    dlist.add(pluginDesc1);
    Test2ParamsConfig cfg2 = new Test2ParamsConfig();
    cfg2.name = "id1";
    Descriptor.ParseDescribable<ParamsConfig> pluginDesc2 = new Descriptor.ParseDescribable<>(cfg2);
    pluginDesc2.extraPluginMetas.add(new XStream2.PluginMeta("test2meta", "1.0.0"));
    dlist.add(pluginDesc2);
    paramsCfgPluginStore.setPlugins(null, Optional.empty(), dlist);
    List<ParamsConfig> plugins = paramsCfgPluginStore.getPlugins();
    assertEquals(3, plugins.size());
    cfg2 = new Test2ParamsConfig();
    cfg2.name = "id1";
    pluginDesc2 = new Descriptor.ParseDescribable<>(cfg2);
    pluginDesc2.extraPluginMetas.add(new XStream2.PluginMeta("test2meta", "1.0.0"));
    dlist.add(pluginDesc2);
    // 保存应该是要出错的，因为cfg2存在Id重复的问题
    paramsCfgPluginStore.setPlugins(null, Optional.empty(), dlist);
}