Examples with CheckResult com.dtstack.taier.pluginapi.pojo.CheckResult used on opensource projects

Example 1 with CheckResult

use of com.dtstack.taier.pluginapi.pojo.CheckResult in project Taier by DTStack.

the class FlinkClientTest method testGrammarCheck.

/*@Test
    public void testGetJobStatus() throws Exception {
        String jobId = "40c01cd0c53928fff6a55e8d8b8b022c";
        String appId = "application_1594003499276_1278";
        String taskId = "taskId";
        JobIdentifier jobIdentifier = JobIdentifier.createInstance(jobId, appId, taskId);

        ApplicationReportPBImpl report = YarnMockUtil.mockApplicationReport(null);
        when(yarnClient.getApplicationReport(any())).thenReturn(report);
        when(flinkClientBuilder.getYarnClient()).thenReturn(yarnClient);

        RdosTaskStatus jobStatus = flinkClient.getJobStatus(jobIdentifier);
        Assert.assertNotNull(jobStatus);

        PowerMockito.mockStatic(PoolHttpClient.class);
        when(PoolHttpClient.get(any())).thenReturn("{\"state\":\"RUNNING\"}");

        ClusterClient clusterClient = YarnMockUtil.mockClusterClient();
        when(flinkClusterClientManager.getClusterClient(null)).thenReturn(clusterClient);
        jobIdentifier.setApplicationId(null);
        RdosTaskStatus jobStatus2 = flinkClient.getJobStatus(jobIdentifier);
        Assert.assertNotNull(jobStatus2);
    }*/
@Test
public void testGrammarCheck() throws Exception {
    MemberModifier.field(FlinkClient.class, "cacheFile").set(flinkClient, Maps.newConcurrentMap());
    String sqlPluginRootDir = temporaryFolder.newFolder("sqlPluginDir").getAbsolutePath();
    temporaryFolder.newFolder("sqlPluginDir", "sqlplugin");
    temporaryFolder.newFile("sqlPluginDir/sqlplugin/core-test.jar").getAbsolutePath();
    FlinkConfig flinkConfig = new FlinkConfig();
    flinkConfig.setFlinkPluginRoot(sqlPluginRootDir);
    SqlPluginInfo sqlPluginInfo = SqlPluginInfo.create(flinkConfig);
    MemberModifier.field(FlinkClient.class, "sqlPluginInfo").set(flinkClient, sqlPluginInfo);
    PowerMockito.mockStatic(PackagedProgram.class);
    PackagedProgram.Builder builder = PowerMockito.mock(PackagedProgram.Builder.class);
    when(PackagedProgram.newBuilder()).thenReturn(builder);
    PackagedProgram packagedProgram = PowerMockito.mock(PackagedProgram.class);
    when(builder.setJarFile(any(File.class))).thenReturn(builder);
    when(builder.setUserClassPaths(any(List.class))).thenReturn(builder);
    when(builder.setConfiguration(any(Configuration.class))).thenReturn(builder);
    when(builder.setArguments(any())).thenReturn(builder);
    when(builder.build()).thenReturn(packagedProgram);
    Configuration configuration = new Configuration();
    when(flinkClientBuilder.getFlinkConfiguration()).thenReturn(configuration);
    PowerMockito.mockStatic(PackagedProgramUtils.class);
    when(PackagedProgramUtils.createJobGraph(any(PackagedProgram.class), any(Configuration.class), any(int.class), any(boolean.class))).thenReturn(PowerMockito.mock(JobGraph.class));
    String absolutePath = temporaryFolder.newFile("core-flinksql.jar").getAbsolutePath();
    String sqlText = "CREATE TABLE MyTable ( id int, name varchar )  WITH (topicIsPattern = 'false', updateMode = 'append', bootstrapServers = 'dtstack01:9092', timezone = 'Asia/Shanghai', parallelism = '1', topic = 'grammar', type = 'kafka11', enableKeyPartitions = 'false', offsetReset = 'latest');  CREATE TABLE MyResult ( id INT, name VARCHAR )  WITH (type = 'console');  INSERT INTO MyResult SELECT a.id, a.name FROM MyTable a;";
    JobClient jobClient = YarnMockUtil.mockJobClient("perJob", sqlText, absolutePath);
    CheckResult checkResult = flinkClient.grammarCheck(jobClient);
    Assert.assertTrue(checkResult.isResult() == true);
}

Example 2 with CheckResult

use of com.dtstack.taier.pluginapi.pojo.CheckResult in project Taier by DTStack.

the class FlinkClient method grammarCheck.

@Override
public CheckResult grammarCheck(JobClient jobClient) {
    CheckResult checkResult = CheckResult.success();
    String taskId = jobClient.getJobId();
    try {
        // 1. before download jar
        beforeSubmitFunc(jobClient);
        // 2. flink sql args
        String taskWorkspace = FlinkUtil.getTaskWorkspace(jobClient.getJobId());
        List<String> args = sqlPluginInfo.buildExeArgs(jobClient);
        List<String> attachJarLists = cacheFile.get(taskWorkspace);
        List<URL> attachJarUrls = Lists.newArrayList();
        if (!CollectionUtils.isEmpty(attachJarLists)) {
            args.add("-addjar");
            String attachJarStr = PublicUtil.objToString(attachJarLists);
            args.add(URLEncoder.encode(attachJarStr, Charsets.UTF_8.name()));
            attachJarUrls = attachJarLists.stream().map(k -> {
                try {
                    return new File(k).toURL();
                } catch (MalformedURLException e) {
                    throw new PluginDefineException(e);
                }
            }).collect(Collectors.toList());
        }
        JarFileInfo coreJarInfo = sqlPluginInfo.createCoreJarInfo();
        jobClient.setCoreJarInfo(coreJarInfo);
        // 3. build jobGraph
        String[] programArgs = args.toArray(new String[args.size()]);
        Configuration flinkConfig = flinkClientBuilder.getFlinkConfiguration();
        PackagedProgram program = PackagedProgram.newBuilder().setJarFile(new File(coreJarInfo.getJarPath())).setUserClassPaths(attachJarUrls).setConfiguration(flinkConfig).setArguments(programArgs).build();
        PackagedProgramUtils.createJobGraph(program, flinkConfig, 1, false);
        logger.info("TaskId: {}, GrammarCheck success!", taskId);
    } catch (Exception e) {
        logger.error("TaskId: {}, GrammarCheck error: ", taskId, e);
        checkResult = CheckResult.exception(ExceptionUtil.getErrorMessage(e));
    } finally {
        try {
            afterSubmitFunc(jobClient);
        } catch (Exception e) {
        }
    }
    return checkResult;
}