Examples with PluginDefineException com.dtstack.taier.pluginapi.exception.PluginDefineException used on opensource projects

Example 41 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class PoolHttpClient method getRequest.

private static String getRequest(String url, Header[] headers) throws IOException {
    String respBody = null;
    HttpGet httpGet = null;
    CloseableHttpResponse response = null;
    int statusCode = 0;
    try {
        httpGet = new HttpGet(url);
        if (headers != null && headers.length > 0) {
            httpGet.setHeaders(headers);
        }
        response = httpClient.execute(httpGet);
        statusCode = response.getStatusLine().getStatusCode();
        if (statusCode == HttpStatus.SC_OK) {
            HttpEntity entity = response.getEntity();
            respBody = EntityUtils.toString(entity, charset);
        } else if (statusCode == HttpStatus.SC_UNAUTHORIZED) {
            throw new PluginDefineException("登陆状态失效" + statusCode);
        } else {
            LOGGER.warn("request url:{} fail:{}", url, response.getStatusLine().getStatusCode());
            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) {
                throw new PluginDefineException("status code " + HttpStatus.SC_NOT_FOUND);
            } else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
                throw new PluginDefineException("status code " + HttpStatus.SC_INTERNAL_SERVER_ERROR);
            }
        }
    } catch (IOException e) {
        LOGGER.error("url:{}--->http request error:", url, e);
        throw e;
    } finally {
        if (HttpStatus.SC_OK != statusCode && null != httpGet) {
            httpGet.abort();
        }
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                LOGGER.error("", e);
            }
        }
    }
    return respBody;
}

Example 42 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class PerJobClientFactory method deleteTaskIfExist.

/**
 * kill application which name and queue are the same as this jobClient
 * when run in stream-computing mode
 */
public void deleteTaskIfExist(JobClient jobClient) {
    if (ComputeType.BATCH.equals(jobClient.getComputeType())) {
        return;
    }
    try {
        String taskName = jobClient.getJobName();
        String queueName = flinkConfig.getQueue();
        YarnClient yarnClient = flinkClientBuilder.getYarnClient();
        EnumSet<YarnApplicationState> enumSet = EnumSet.noneOf(YarnApplicationState.class);
        enumSet.add(YarnApplicationState.ACCEPTED);
        enumSet.add(YarnApplicationState.RUNNING);
        List<ApplicationReport> existApps = yarnClient.getApplications(enumSet).stream().filter(report -> report.getQueue().endsWith(queueName)).filter(report -> report.getName().equals(taskName)).collect(Collectors.toList());
        for (ApplicationReport report : existApps) {
            ApplicationId appId = report.getApplicationId();
            LOG.info("try to kill application " + appId.toString() + " which name is " + report.getName());
            yarnClient.killApplication(appId);
        }
    } catch (Exception e) {
        LOG.error("Delete task error ", e);
        throw new PluginDefineException("Delete task error");
    }
}

Example 43 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class PerJobClientFactory method getClusterClient.

@Override
public ClusterClient getClusterClient(JobIdentifier jobIdentifier) {
    String applicationId = jobIdentifier.getApplicationId();
    String jobId = jobIdentifier.getJobId();
    ClusterClient clusterClient = null;
    try {
        clusterClient = KerberosUtils.login(flinkConfig, () -> {
            try {
                return perJobClientCache.get(applicationId, () -> {
                    ParamAction action = new ParamAction();
                    action.setJobId(jobId);
                    action.setName("jobId-" + jobId);
                    action.setTaskType(EJobType.SQL.getType());
                    action.setComputeType(ComputeType.STREAM.getType());
                    action.setTenantId(-1L);
                    String taskParams = "flinkTaskRunMode=per_job";
                    action.setTaskParams(taskParams);
                    JobClient jobClient = new JobClient(action);
                    try (YarnClusterDescriptor perJobYarnClusterDescriptor = this.createPerJobClusterDescriptor(jobClient)) {
                        return perJobYarnClusterDescriptor.retrieve(ConverterUtils.toApplicationId(applicationId)).getClusterClient();
                    }
                });
            } catch (ExecutionException e) {
                throw new PluginDefineException(e);
            }
        }, flinkClientBuilder.getYarnConf());
    } catch (Exception e) {
        LOG.error("job[{}] get perJobClient exception:{}", jobId, e);
        throw new PluginDefineException(e);
    }
    return clusterClient;
}

Example 44 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SessionClientFactory method initZkClient.

private void initZkClient() {
    String zkAddress = flinkConfiguration.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);
    if (StringUtils.isBlank(zkAddress)) {
        throw new PluginDefineException("zkAddress is error");
    }
    this.zkClient = CuratorFrameworkFactory.builder().connectString(zkAddress).retryPolicy(new ExponentialBackoffRetry(1000, 3)).connectionTimeoutMs(flinkConfig.getZkConnectionTimeout()).sessionTimeoutMs(flinkConfig.getZkSessionTimeout()).build();
    this.zkClient.start();
    try {
        if (null == this.leaderLatch) {
            this.leaderLatch = getLeaderLatch();
            this.leaderLatch.addListener(new LeaderLatchListener() {

                @Override
                public void isLeader() {
                    isLeader.set(true);
                    LOG.info(">>>My monitor role is Leader.");
                }

                @Override
                public void notLeader() {
                    isLeader.set(false);
                    LOG.info(">>>My monitor role is Follower.");
                }
            });
            this.leaderLatch.start();
            // 这里需要sleep一下，避免leader还未选举完就走到下一步 默认5S
            Thread.sleep(flinkConfig.getMonitorElectionWaitTime());
        }
    } catch (Exception e) {
        LOG.error("join leader election failed.", e);
    }
    LOG.warn("connector zk success...");
}

Example 45 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SessionClientFactory method getKeytabFilesAndSetSecurityConfig.

private List<File> getKeytabFilesAndSetSecurityConfig(Configuration config) {
    Map<String, File> keytabs = new HashMap<>();
    String remoteDir = flinkConfig.getRemoteDir();
    // 任务提交keytab
    String clusterKeytabDirPath = ConfigConstant.LOCAL_KEYTAB_DIR_PARENT + remoteDir;
    File clusterKeytabDir = new File(clusterKeytabDirPath);
    File[] clusterKeytabFiles = clusterKeytabDir.listFiles();
    if (clusterKeytabFiles == null || clusterKeytabFiles.length == 0) {
        throw new PluginDefineException("not find keytab file from " + clusterKeytabDirPath);
    }
    for (File file : clusterKeytabFiles) {
        String fileName = file.getName();
        String keytabPath = file.getAbsolutePath();
        String keytabFileName = flinkConfig.getPrincipalFile();
        if (StringUtils.equals(fileName, keytabFileName)) {
            String principal = flinkConfig.getPrincipal();
            if (StringUtils.isEmpty(principal)) {
                principal = KerberosUtils.getPrincipal(keytabPath);
            }
            config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
            config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, principal);
            continue;
        }
        keytabs.put(file.getName(), file);
    }
    return keytabs.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList());
}