Examples with PluginDefineException com.dtstack.taier.pluginapi.exception.PluginDefineException used on opensource projects

Example 26 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClient method beforeSubmitFunc.

@Override
public void beforeSubmitFunc(JobClient jobClient) {
    logger.info("Job[{}] submit before", jobClient.getJobId());
    String sql = jobClient.getSql();
    List<String> sqlArr = DtStringUtil.splitIgnoreQuota(sql, ';');
    if (sqlArr.size() == 0) {
        return;
    }
    List<String> sqlList = Lists.newArrayList(sqlArr);
    Iterator<String> sqlItera = sqlList.iterator();
    List<String> fileList = Lists.newArrayList();
    String taskWorkspace = FlinkUtil.getTaskWorkspace(jobClient.getJobId());
    while (sqlItera.hasNext()) {
        String tmpSql = sqlItera.next();
        // handle add jar statements and comment statements on the same line
        tmpSql = PrepareOperator.handleSql(tmpSql);
        if (PrepareOperator.verificResource(tmpSql)) {
            sqlItera.remove();
            String localResourceDir = taskWorkspace + ConfigConstrant.SP + "resource";
            if (!new File(localResourceDir).exists()) {
                new File(localResourceDir).mkdirs();
            }
            File resourceFile = PrepareOperator.getResourceFile(tmpSql);
            String resourceFileName = PrepareOperator.getResourceFileName(tmpSql);
            String remoteFile = resourceFile.getAbsolutePath();
            String localFile = localResourceDir + ConfigConstrant.SP + resourceFileName;
            // download file and close
            File downloadFile = filesystemManager.downloadFile(remoteFile, localFile);
            logger.info("Download Resource File :" + downloadFile.getAbsolutePath());
        } else if (PrepareOperator.verificJar(tmpSql)) {
            sqlItera.remove();
            JarFileInfo jarFileInfo = PrepareOperator.parseJarFile(tmpSql);
            String addFilePath = jarFileInfo.getJarPath();
            String tmpJarDir = taskWorkspace + ConfigConstrant.SP + "jar";
            if (!new File(tmpJarDir).exists()) {
                new File(tmpJarDir).mkdirs();
            }
            File jarFile = null;
            try {
                jarFile = FlinkUtil.downloadJar(addFilePath, tmpJarDir, filesystemManager, false);
            } catch (Exception e) {
                throw new PluginDefineException(e);
            }
            fileList.add(jarFile.getAbsolutePath());
            // 更改路径为本地路径
            jarFileInfo.setJarPath(jarFile.getAbsolutePath());
            if (jobClient.getJobType() == EJobType.SQL) {
                jobClient.addAttachJarInfo(jarFileInfo);
            } else {
                // 非sql任务只允许提交一个附件包
                jobClient.setCoreJarInfo(jarFileInfo);
                break;
            }
        }
    }
    cacheFile.put(taskWorkspace, fileList);
    String newSql = String.join(";", sqlList);
    jobClient.setSql(newSql);
}

Example 27 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClientBuilder method buildYarnClient.

/**
 * 创建YarnClient 增加KerberosUtils 逻辑
 * @return
 */
private YarnClient buildYarnClient() {
    try {
        return KerberosUtils.login(flinkConfig, () -> {
            LOG.info("buildYarnClient, init YarnClient!");
            YarnClient yarnClient1 = YarnClient.createYarnClient();
            yarnClient1.init(yarnConf);
            yarnClient1.start();
            return yarnClient1;
        }, yarnConf);
    } catch (Exception e) {
        LOG.error("buildYarnClient initSecurity happens error", e);
        throw new PluginDefineException(e);
    }
}

Example 28 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class HadoopConfTool method deserializeYanrConf.

public static Configuration deserializeYanrConf(byte[] bytes) {
    Configuration hadoopConf = new Configuration(false);
    YarnConfiguration yanrConf = new YarnConfiguration(hadoopConf);
    try (ByteArrayInputStream in = new ByteArrayInputStream(bytes);
        DataInputStream datain = new DataInputStream(in)) {
        yanrConf.readFields(datain);
        return yanrConf;
    } catch (IOException e) {
        LOG.error("Deserialize yanrConf happens error: {}", e.getMessage());
        throw new PluginDefineException(e);
    }
}

Example 29 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class KerberosUtils method getPrincipal.

public static String getPrincipal(String filePath) {
    Keytab keytab = null;
    try {
        keytab = Keytab.loadKeytab(new File(filePath));
    } catch (IOException e) {
        logger.error("Principal {} parse error e: {}!", filePath, e.getMessage());
        throw new PluginDefineException("keytab文件解析异常", e);
    }
    List<PrincipalName> principals = keytab.getPrincipals();
    String principal = "";
    if (principals.size() != 0) {
        principal = principals.get(0).getName();
    } else {
        logger.error("Principal must not be null!");
    }
    logger.info("filePath:{} principal:{}", filePath, principal);
    return principal;
}

Example 30 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class FlinkClient method submitSqlJobForStream.

/**
 * 1: 不再对操作顺序做限制
 * 2：不再限制输入源数量
 * 3：不再限制输出源数量
 * @param jobClient
 * @return
 * @throws IOException
 * @throws ClassNotFoundException
 */
private JobResult submitSqlJobForStream(JobClient jobClient) {
    try {
        String taskWorkspace = String.format("%s/%s_%s", ConfigConstrant.TMP_DIR, jobClient.getJobId(), Thread.currentThread().getId());
        // 构建args
        List<String> args = sqlPluginInfo.buildExeArgs(jobClient);
        List<String> attachJarLists = cacheFile.get(taskWorkspace);
        List<URL> attachJarUrls = Lists.newArrayList();
        if (!CollectionUtils.isEmpty(attachJarLists)) {
            args.add("-addjar");
            String attachJarStr = PublicUtil.objToString(attachJarLists);
            args.add(URLEncoder.encode(attachJarStr, Charsets.UTF_8.name()));
            attachJarUrls = attachJarLists.stream().map(k -> {
                try {
                    return new File(k).toURL();
                } catch (MalformedURLException e) {
                    throw new PluginDefineException(e);
                }
            }).collect(Collectors.toList());
        }
        JarFileInfo coreJarInfo = sqlPluginInfo.createCoreJarInfo();
        jobClient.setCoreJarInfo(coreJarInfo);
        return submitJobWithJar(jobClient, attachJarUrls, args);
    } catch (Exception e) {
        return JobResult.createErrorResult(e);
    }
}