Examples with PluginDefineException com.dtstack.taier.pluginapi.exception.PluginDefineException used on opensource projects

Example 46 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SessionClientFactory method acquireAppIdAndSetClusterId.

private ApplicationId acquireAppIdAndSetClusterId(Configuration configuration) {
    try {
        Set<String> set = new HashSet<>();
        set.add("Apache Flink");
        EnumSet<YarnApplicationState> enumSet = EnumSet.noneOf(YarnApplicationState.class);
        enumSet.add(YarnApplicationState.RUNNING);
        enumSet.add(YarnApplicationState.ACCEPTED);
        YarnClient yarnClient = flinkClientBuilder.getYarnClient();
        if (null == yarnClient) {
            throw new PluginDefineException("getYarnClient error, Yarn Client is null!");
        }
        List<ApplicationReport> reportList = yarnClient.getApplications(set, enumSet);
        int maxMemory = -1;
        int maxCores = -1;
        ApplicationId applicationId = null;
        for (ApplicationReport report : reportList) {
            LOG.info("filter flink session application current reportName:{} queue:{} status:{}", report.getName(), report.getQueue(), report.getYarnApplicationState());
            if (!report.getYarnApplicationState().equals(YarnApplicationState.RUNNING)) {
                continue;
            }
            if (!report.getName().startsWith(flinkConfig.getFlinkSessionName())) {
                continue;
            }
            if (flinkConfig.getSessionStartAuto() && !report.getName().endsWith(sessionAppNameSuffix)) {
                continue;
            }
            if (!report.getQueue().endsWith(flinkConfig.getQueue())) {
                continue;
            }
            int thisMemory = report.getApplicationResourceUsageReport().getNeededResources().getMemory();
            int thisCores = report.getApplicationResourceUsageReport().getNeededResources().getVirtualCores();
            LOG.info("current flink session memory {},Cores{}", thisMemory, thisCores);
            if (thisMemory > maxMemory || thisMemory == maxMemory && thisCores > maxCores) {
                maxMemory = thisMemory;
                maxCores = thisCores;
                applicationId = report.getApplicationId();
                // flinkClusterId不为空 且 yarnsession不是由engine来管控时，需要设置clusterId（兼容手动启动yarnsession的情况）
                if (StringUtils.isBlank(configuration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID)) || report.getName().endsWith(sessionAppNameSuffix)) {
                    configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, applicationId.toString());
                }
            }
        }
        return applicationId;
    } catch (Exception e) {
        LOG.error("", e);
        throw new PluginDefineException(e);
    }
}

Example 47 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SessionClientFactory method createYarnSessionClusterDescriptor.

public YarnClusterDescriptor createYarnSessionClusterDescriptor() throws MalformedURLException {
    Configuration newConf = new Configuration(flinkConfiguration);
    String flinkJarPath = flinkConfig.getFlinkJarPath();
    String pluginLoadMode = flinkConfig.getPluginLoadMode();
    YarnConfiguration yarnConf = flinkClientBuilder.getYarnConf();
    FileUtil.checkFileExist(flinkJarPath);
    if (!flinkConfig.getFlinkHighAvailability()) {
        setNoneHaModeConfig(newConf);
    } else {
        // 由engine管控的yarnsession clusterId不进行设置，默认使用appId作为clusterId
        newConf.removeConfig(HighAvailabilityOptions.HA_CLUSTER_ID);
    }
    List<File> keytabFiles = null;
    if (flinkConfig.isOpenKerberos()) {
        keytabFiles = getKeytabFilesAndSetSecurityConfig(newConf);
    }
    newConf = setHdfsFlinkJarPath(flinkConfig, newConf);
    YarnClusterDescriptor clusterDescriptor = getClusterDescriptor(newConf, yarnConf);
    if (StringUtils.isNotBlank(pluginLoadMode) && ConfigConstrant.FLINK_PLUGIN_SHIPFILE_LOAD.equalsIgnoreCase(pluginLoadMode)) {
        newConf.setString(ConfigConstrant.FLINK_PLUGIN_LOAD_MODE, flinkConfig.getPluginLoadMode());
        String flinkPluginRoot = flinkConfig.getFlinkPluginRoot();
        if (StringUtils.isNotBlank(flinkPluginRoot)) {
            String syncPluginDir = flinkPluginRoot + ConfigConstrant.SP + ConfigConstrant.SYNCPLUGIN_DIR;
            File syncFile = new File(syncPluginDir);
            if (!syncFile.exists()) {
                throw new PluginDefineException("syncPlugin path is null");
            }
            List<File> pluginPaths = Arrays.stream(syncFile.listFiles()).filter(file -> !file.getName().endsWith("zip")).collect(Collectors.toList());
            clusterDescriptor.addShipFiles(pluginPaths);
        }
    }
    if (CollectionUtils.isNotEmpty(keytabFiles)) {
        clusterDescriptor.addShipFiles(keytabFiles);
    }
    List<URL> classpaths = getFlinkJarFile(flinkJarPath, clusterDescriptor);
    clusterDescriptor.setProvidedUserJarFiles(classpaths);
    return clusterDescriptor;
}

Example 48 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClient method getPerJobStatus.

/**
 * per-job模式其实获取的任务状态是yarn-application状态
 * @param applicationId
 * @return
 */
public TaskStatus getPerJobStatus(String applicationId) {
    try {
        return KerberosUtils.login(flinkConfig, () -> {
            ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
            try {
                ApplicationReport report = flinkClientBuilder.getYarnClient().getApplicationReport(appId);
                YarnApplicationState applicationState = report.getYarnApplicationState();
                switch(applicationState) {
                    case KILLED:
                        return TaskStatus.KILLED;
                    case NEW:
                    case NEW_SAVING:
                        return TaskStatus.CREATED;
                    case SUBMITTED:
                        // FIXME 特殊逻辑,认为已提交到计算引擎的状态为等待资源状态
                        return TaskStatus.WAITCOMPUTE;
                    case ACCEPTED:
                        return TaskStatus.SCHEDULED;
                    case RUNNING:
                        return TaskStatus.RUNNING;
                    case FINISHED:
                        // state 为finished状态下需要兼顾判断finalStatus.
                        FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
                        if (finalApplicationStatus == FinalApplicationStatus.FAILED) {
                            return TaskStatus.FAILED;
                        } else if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
                            return TaskStatus.FINISHED;
                        } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
                            return TaskStatus.KILLED;
                        } else if (finalApplicationStatus == FinalApplicationStatus.UNDEFINED) {
                            return TaskStatus.FAILED;
                        } else {
                            return TaskStatus.RUNNING;
                        }
                    case FAILED:
                        return TaskStatus.FAILED;
                    default:
                        throw new PluginDefineException("Unsupported application state");
                }
            } catch (YarnException | IOException e) {
                logger.error("appId: {}, getPerJobStatus with yarnClient error: ", applicationId, e);
                return TaskStatus.NOTFOUND;
            }
        }, hadoopConf.getYarnConfiguration());
    } catch (Exception e) {
        logger.error("appId: {}, getPerJobStatus with yarnClient error: ", applicationId, e);
        // 防止因为kerberos 认证不过出现notfound最后变为failed
        return TaskStatus.RUNNING;
    }
}

Example 49 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClient method grammarCheck.

@Override
public CheckResult grammarCheck(JobClient jobClient) {
    CheckResult checkResult = CheckResult.success();
    String taskId = jobClient.getJobId();
    try {
        // 1. before download jar
        beforeSubmitFunc(jobClient);
        // 2. flink sql args
        String taskWorkspace = FlinkUtil.getTaskWorkspace(jobClient.getJobId());
        List<String> args = sqlPluginInfo.buildExeArgs(jobClient);
        List<String> attachJarLists = cacheFile.get(taskWorkspace);
        List<URL> attachJarUrls = Lists.newArrayList();
        if (!CollectionUtils.isEmpty(attachJarLists)) {
            args.add("-addjar");
            String attachJarStr = PublicUtil.objToString(attachJarLists);
            args.add(URLEncoder.encode(attachJarStr, Charsets.UTF_8.name()));
            attachJarUrls = attachJarLists.stream().map(k -> {
                try {
                    return new File(k).toURL();
                } catch (MalformedURLException e) {
                    throw new PluginDefineException(e);
                }
            }).collect(Collectors.toList());
        }
        JarFileInfo coreJarInfo = sqlPluginInfo.createCoreJarInfo();
        jobClient.setCoreJarInfo(coreJarInfo);
        // 3. build jobGraph
        String[] programArgs = args.toArray(new String[args.size()]);
        Configuration flinkConfig = flinkClientBuilder.getFlinkConfiguration();
        PackagedProgram program = PackagedProgram.newBuilder().setJarFile(new File(coreJarInfo.getJarPath())).setUserClassPaths(attachJarUrls).setConfiguration(flinkConfig).setArguments(programArgs).build();
        PackagedProgramUtils.createJobGraph(program, flinkConfig, 1, false);
        logger.info("TaskId: {}, GrammarCheck success!", taskId);
    } catch (Exception e) {
        logger.error("TaskId: {}, GrammarCheck error: ", taskId, e);
        checkResult = CheckResult.exception(ExceptionUtil.getErrorMessage(e));
    } finally {
        try {
            afterSubmitFunc(jobClient);
        } catch (Exception e) {
        }
    }
    return checkResult;
}

Example 50 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClientBuilder method initFlinkGlobalConfiguration.

public void initFlinkGlobalConfiguration(Properties extProp) {
    Configuration config = new Configuration();
    config.setString("akka.client.timeout", ConfigConstrant.AKKA_CLIENT_TIMEOUT);
    config.setString("akka.ask.timeout", ConfigConstrant.AKKA_ASK_TIMEOUT);
    config.setString("akka.tcp.timeout", ConfigConstrant.AKKA_TCP_TIMEOUT);
    // yarn queue
    config.setString(YarnConfigOptions.APPLICATION_QUEUE, flinkConfig.getQueue());
    config.setBytes(ConfigConstrant.HADOOP_CONF_BYTES_KEY, HadoopConfTool.serializeHadoopConf(hadoopConf));
    config.setBytes(ConfigConstrant.YARN_CONF_BYTES_KEY, HadoopConfTool.serializeHadoopConf(yarnConf));
    if (extProp != null) {
        for (Object key : extProp.keySet()) {
            String newKey = (String) key;
            String value = extProp.getProperty(newKey);
            if (StringUtils.isEmpty(value)) {
                continue;
            }
            if (!FlinkConfig.getEngineFlinkConfigs().contains(key.toString())) {
                config.setString(newKey, value);
            }
        }
    }
    config.setBoolean(ConfigConstrant.OPEN_KERBEROS_KEY, flinkConfig.isOpenKerberos());
    try {
        FileSystem.initialize(config);
    } catch (Exception e) {
        LOG.error("", e);
        throw new PluginDefineException(e);
    }
    flinkConfiguration = config;
}