Examples with ClusterMode com.dtstack.taier.flink.base.enums.ClusterMode used on opensource projects

Example 1 with ClusterMode

use of com.dtstack.taier.flink.base.enums.ClusterMode in project Taier by DTStack.

the class PerJobClientFactory method appendJobConfigAndInitFs.

private Configuration appendJobConfigAndInitFs(JobClient jobClient, Configuration configuration) {
    Properties properties = jobClient.getConfProperties();
    if (properties != null) {
        for (Object key : properties.keySet()) {
            String keyStr = key.toString();
            if (!StringUtils.contains(keyStr, ".") && !StringUtils.equalsIgnoreCase(keyStr, ConfigConstrant.LOG_LEVEL_KEY)) {
                continue;
            }
            String value = properties.getProperty(keyStr);
            if (StringUtils.equalsIgnoreCase(keyStr, SecurityOptions.KERBEROS_LOGIN_CONTEXTS.key()) && StringUtils.isNotEmpty(value)) {
                value = StringUtils.replacePattern(value, "\\s*", "");
                String contexts = configuration.get(SecurityOptions.KERBEROS_LOGIN_CONTEXTS);
                contexts = StringUtils.replacePattern(contexts, "\\s*", "");
                contexts = StringUtils.isNotEmpty(contexts) ? String.format("%s,%s", value, contexts) : value;
                List<String> contextsTmp = Arrays.asList(StringUtils.split(contexts, ","));
                Set contextsSet = new HashSet(contextsTmp);
                value = StringUtils.join(contextsSet, ",");
            }
            configuration.setString(keyStr, value);
        }
    }
    String taskId = jobClient.getJobId();
    configuration.setString(ConfigConstrant.KEY_PROMGATEWAY_JOBNAME, taskId);
    ClusterMode clusterMode = ClusterMode.getClusteMode(flinkConfig.getClusterMode());
    Boolean isPerjob = ClusterMode.isPerjob(clusterMode);
    if (!flinkConfig.getFlinkHighAvailability() && !isPerjob) {
        setNoneHaModeConfig(configuration);
    } else {
        configuration.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.toString());
        configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, jobClient.getJobId());
    }
    configuration.setString(YarnConfigOptions.APPLICATION_NAME, jobClient.getJobName());
    if (StringUtils.isNotBlank(flinkConfig.getPluginLoadMode()) && ConfigConstrant.FLINK_PLUGIN_SHIPFILE_LOAD.equalsIgnoreCase(flinkConfig.getPluginLoadMode())) {
        configuration.setString(ConfigConstrant.FLINK_PLUGIN_LOAD_MODE, flinkConfig.getPluginLoadMode());
    }
    String classloaderCache = configuration.getString(ClassLoaderType.CLASSLOADER_DTSTACK_CACHE, ClassLoaderType.CLASSLOADER_DTSTACK_CACHE_TRUE);
    configuration.setString(ClassLoaderType.CLASSLOADER_DTSTACK_CACHE, classloaderCache);
    String append = configuration.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL);
    if (jobClient.getJobType() == EJobType.SQL || jobClient.getJobType() == EJobType.SYNC) {
        String dtstackAppend = "com.fasterxml.jackson.";
        if (StringUtils.isNotEmpty(append)) {
            dtstackAppend = dtstackAppend + ";" + append;
        }
        configuration.setString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, dtstackAppend);
    }
    try {
        FileSystem.initialize(configuration);
    } catch (Exception e) {
        LOG.error("", e);
        throw new PluginDefineException(e);
    }
    return configuration;
}

Example 2 with ClusterMode

use of com.dtstack.taier.flink.base.enums.ClusterMode in project Taier by DTStack.

the class AbstractClientFactory method createClientFactory.

public static IClientFactory createClientFactory(FlinkClientBuilder flinkClientBuilder) {
    FlinkConfig flinkConfig = flinkClientBuilder.getFlinkConfig();
    ClusterMode clusterMode = ClusterMode.getClusteMode(flinkConfig.getClusterMode());
    IClientFactory clientFactory;
    switch(clusterMode) {
        case PER_JOB:
            clientFactory = new PerJobClientFactory(flinkClientBuilder);
            break;
        case SESSION:
            clientFactory = new SessionClientFactory(flinkClientBuilder);
            break;
        case STANDALONE:
            clientFactory = new StandaloneClientFactory(flinkClientBuilder);
            break;
        default:
            throw new PluginDefineException("not support clusterMode: " + clusterMode);
    }
    return clientFactory;
}

Example 3 with ClusterMode

use of com.dtstack.taier.flink.base.enums.ClusterMode in project Taier by DTStack.

the class FlinkClient method submitJobWithJar.

private JobResult submitJobWithJar(JobClient jobClient, List<URL> classPaths, List<String> programArgList) {
    JobParam jobParam = new JobParam(jobClient);
    String jarPath = jobParam.getJarPath();
    if (jarPath == null) {
        logger.error("can not submit a job without jar path, please check it");
        return JobResult.createErrorResult("can not submit a job without jar path, please check it");
    }
    String args = jobParam.getClassArgs();
    if (StringUtils.isNotBlank(args)) {
        programArgList.addAll(DtStringUtil.splitIgnoreQuota(args, ' '));
    }
    // 如果jar包里面未指定mainclass,需要设置该参数
    String entryPointClass = jobParam.getMainClass();
    SavepointRestoreSettings spSettings = buildSavepointSetting(jobClient);
    String[] programArgs = programArgList.toArray(new String[programArgList.size()]);
    PackagedProgram packagedProgram = null;
    JobGraph jobGraph = null;
    Pair<String, String> runResult;
    try {
        ClusterMode clusterMode = ClusterMode.getClusteMode(flinkConfig.getClusterMode());
        if (ClusterMode.isPerjob(clusterMode)) {
            // perjob模式延后创建PackagedProgram
            ClusterSpecification clusterSpecification = FlinkConfUtil.createClusterSpecification(flinkClientBuilder.getFlinkConfiguration(), jobClient.getApplicationPriority(), jobClient.getConfProperties());
            clusterSpecification.setClasspaths(classPaths);
            clusterSpecification.setEntryPointClass(entryPointClass);
            clusterSpecification.setJarFile(new File(jarPath));
            clusterSpecification.setSpSetting(spSettings);
            clusterSpecification.setProgramArgs(programArgs);
            clusterSpecification.setCreateProgramDelay(true);
            clusterSpecification.setYarnConfiguration(hadoopConf.getYarnConfiguration());
            logger.info("--------taskId: {} run by PerJob mode-----", jobClient.getJobId());
            runResult = runJobByPerJob(clusterSpecification, jobClient);
            jobGraph = clusterSpecification.getJobGraph();
            packagedProgram = clusterSpecification.getProgram();
        } else {
            Integer runParallelism = FlinkUtil.getJobParallelism(jobClient.getConfProperties());
            packagedProgram = FlinkUtil.buildProgram(jarPath, classPaths, jobClient.getJobType(), entryPointClass, programArgs, spSettings, flinkClientBuilder.getFlinkConfiguration(), filesystemManager);
            jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, flinkClientBuilder.getFlinkConfiguration(), runParallelism, false);
            // 只有当程序本身没有指定并行度的时候该参数才生效
            clearClassPathShipfileLoadMode(packagedProgram);
            logger.info("--------taskId: {} run by Session mode-----", jobClient.getJobId());
            runResult = runJobBySession(jobGraph);
        }
        JobResult jobResult = JobResult.createSuccessResult(runResult.getSecond(), runResult.getFirst());
        jobResult.setExtraData(JobResultConstant.JOB_GRAPH, JobGraphBuildUtil.buildLatencyMarker(jobGraph));
        long checkpointInterval = jobGraph.getCheckpointingSettings().getCheckpointCoordinatorConfiguration().getCheckpointInterval();
        if (checkpointInterval >= Long.MAX_VALUE) {
            checkpointInterval = 0;
        }
        jobResult.setExtraData(JobResultConstant.FLINK_CHECKPOINT, String.valueOf(checkpointInterval));
        return jobResult;
    } catch (Throwable e) {
        return JobResult.createErrorResult(e);
    } finally {
        if (packagedProgram != null) {
            packagedProgram.deleteExtractedLibraries();
        }
    }
}

Example 4 with ClusterMode

use of com.dtstack.taier.flink.base.enums.ClusterMode in project Taier by DTStack.

the class FlinkClient method judgeSlots.

@Override
public JudgeResult judgeSlots(JobClient jobClient) {
    ClusterMode clusterMode = ClusterMode.getClusteMode(flinkConfig.getClusterMode());
    boolean isPerJob = ClusterMode.isPerjob(clusterMode);
    JudgeResult judgeResult = JudgeResult.notOk(null);
    try {
        if (ClusterMode.isStandalone(clusterMode)) {
            return judgeSessionSlot(jobClient, true);
        }
        judgeResult = KerberosUtils.login(flinkConfig, () -> {
            FlinkPerJobResourceInfo perJobResourceInfo = FlinkPerJobResourceInfo.FlinkPerJobResourceInfoBuilder().withYarnClient(flinkClientBuilder.getYarnClient()).withQueueName(flinkConfig.getQueue()).withYarnAccepterTaskNumber(flinkConfig.getYarnAccepterTaskNumber()).build();
            return perJobResourceInfo.judgeSlots(jobClient);
        }, hadoopConf.getYarnConfiguration());
        if (judgeResult.available() && !isPerJob) {
            judgeResult = judgeSessionSlot(jobClient, false);
        }
    } catch (Exception e) {
        logger.error("taskId:{} judgeSlots error: ", jobClient.getJobId(), e);
        judgeResult = JudgeResult.exception("judgeSlots error:" + ExceptionUtil.getErrorMessage(e));
    }
    logger.info("taskId:{}, judgeResult: {}, reason: {}", jobClient.getJobId(), judgeResult.getResult(), judgeResult.getReason());
    return judgeResult;
}

Example 5 with ClusterMode

use of com.dtstack.taier.flink.base.enums.ClusterMode in project Taier by DTStack.

the class SyncPluginInfo method createSyncPluginArgs.

public List<String> createSyncPluginArgs(JobClient jobClient, FlinkClient flinkClient) {
    String args = jobClient.getClassArgs();
    List<String> programArgList = Lists.newArrayList();
    if (StringUtils.isNotBlank(args)) {
        programArgList.addAll(Arrays.asList(args.split("\\s+")));
    }
    ClusterMode clusterMode = ClusterMode.getClusteMode(flinkConfig.getClusterMode());
    programArgList.add("-monitor");
    if (StringUtils.isNotEmpty(monitorAddress)) {
        programArgList.add(monitorAddress);
    } else {
        programArgList.add(flinkClient.getReqUrl(clusterMode));
    }
    programArgList.add("-pluginLoadMode");
    programArgList.add(pluginLoadMode);
    programArgList.add("-mode");
    if (ClusterMode.isPerjob(clusterMode)) {
        programArgList.add("yarnPer");
    } else {
        programArgList.add("yarn");
    }
    // 数据源keytab
    StringBuilder fileName = new StringBuilder(32);
    String taskWorkspace = FlinkUtil.getTaskWorkspace(jobClient.getJobId());
    String taskKeytabDirPath = taskWorkspace + ConfigConstrant.SP + "resource";
    File taskKeytabDir = new File(taskKeytabDirPath);
    File[] taskKeytabFiles = taskKeytabDir.listFiles();
    if (taskKeytabFiles != null && taskKeytabFiles.length > 0) {
        for (File file : taskKeytabFiles) {
            fileName.append(file.getAbsoluteFile()).append(",");
        }
    }
    if (StringUtils.isNotBlank(fileName)) {
        programArgList.add("-shipfile");
        programArgList.add(fileName.toString());
    }
    return programArgList;
}