Examples with PackagedProgram org.apache.flink.client.program.PackagedProgram used on opensource projects

Example 1 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class JarListHandler method handleJsonRequest.

@Override
public String handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, ActorGateway jobManager) throws Exception {
    try {
        StringWriter writer = new StringWriter();
        JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
        gen.writeStartObject();
        gen.writeStringField("address", queryParams.get(RuntimeMonitorHandler.WEB_MONITOR_ADDRESS_KEY));
        gen.writeArrayFieldStart("files");
        File[] list = jarDir.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".jar");
            }
        });
        for (File f : list) {
            // separate the uuid and the name parts.
            String id = f.getName();
            int startIndex = id.indexOf("_");
            if (startIndex < 0) {
                continue;
            }
            String name = id.substring(startIndex + 1);
            if (name.length() < 5 || !name.endsWith(".jar")) {
                continue;
            }
            gen.writeStartObject();
            gen.writeStringField("id", id);
            gen.writeStringField("name", name);
            gen.writeNumberField("uploaded", f.lastModified());
            gen.writeArrayFieldStart("entry");
            String[] classes = new String[0];
            try {
                JarFile jar = new JarFile(f);
                Manifest manifest = jar.getManifest();
                String assemblerClass = null;
                if (manifest != null) {
                    assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS);
                    if (assemblerClass == null) {
                        assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS);
                    }
                }
                if (assemblerClass != null) {
                    classes = assemblerClass.split(",");
                }
            } catch (IOException ignored) {
            // we simply show no entries here
            }
            // show every entry class that can be loaded later on.
            for (String clazz : classes) {
                clazz = clazz.trim();
                PackagedProgram program = null;
                try {
                    program = new PackagedProgram(f, clazz, new String[0]);
                } catch (Exception ignored) {
                // ignore jar files which throw an error upon creating a PackagedProgram
                }
                if (program != null) {
                    gen.writeStartObject();
                    gen.writeStringField("name", clazz);
                    String desc = program.getDescription();
                    gen.writeStringField("description", desc == null ? "No description provided" : desc);
                    gen.writeEndObject();
                }
            }
            gen.writeEndArray();
            gen.writeEndObject();
        }
        gen.writeEndArray();
        gen.writeEndObject();
        gen.close();
        return writer.toString();
    } catch (Exception e) {
        throw new RuntimeException("Failed to fetch jar list: " + e.getMessage(), e);
    }
}

Example 2 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontend method info.

/**
	 * Executes the info action.
	 * 
	 * @param args Command line arguments for the info action.
	 */
protected int info(String[] args) {
    LOG.info("Running 'info' command.");
    // Parse command line options
    InfoOptions options;
    try {
        options = CliFrontendParser.parseInfoCommand(args);
    } catch (CliArgsException e) {
        return handleArgException(e);
    } catch (Throwable t) {
        return handleError(t);
    }
    // evaluate help flag
    if (options.isPrintHelp()) {
        CliFrontendParser.printHelpForInfo();
        return 0;
    }
    if (options.getJarFilePath() == null) {
        return handleArgException(new CliArgsException("The program JAR file was not specified."));
    }
    // -------- build the packaged program -------------
    PackagedProgram program;
    try {
        LOG.info("Building program from JAR file");
        program = buildProgram(options);
    } catch (Throwable t) {
        return handleError(t);
    }
    try {
        int parallelism = options.getParallelism();
        LOG.info("Creating program plan dump");
        Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
        FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);
        String jsonPlan = null;
        if (flinkPlan instanceof OptimizedPlan) {
            jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
        } else if (flinkPlan instanceof StreamingPlan) {
            jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
        }
        if (jsonPlan != null) {
            System.out.println("----------------------- Execution Plan -----------------------");
            System.out.println(jsonPlan);
            System.out.println("--------------------------------------------------------------");
        } else {
            System.out.println("JSON plan could not be generated.");
        }
        String description = program.getDescription();
        if (description != null) {
            System.out.println();
            System.out.println(description);
        } else {
            System.out.println();
            System.out.println("No description provided.");
        }
        return 0;
    } catch (Throwable t) {
        return handleError(t);
    } finally {
        program.deleteExtractedLibraries();
    }
}

Example 3 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testVariantWithExplicitJarAndNoArgumentsOption.

@Test
public void testVariantWithExplicitJarAndNoArgumentsOption() {
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-j", getTestJarPath(), "--debug", "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = new String[] { "--debug", "true", "arg1", "arg2" };
        RunOptions options = CliFrontendParser.parseRunCommand(arguments);
        assertEquals(getTestJarPath(), options.getJarFilePath());
        assertArrayEquals(classpath, options.getClasspaths().toArray());
        assertArrayEquals(reducedArguments, options.getProgramArgs());
        CliFrontend frontend = new CliFrontend(CliFrontendTestUtils.getConfigDir());
        PackagedProgram prog = frontend.buildProgram(options);
        Assert.assertArrayEquals(reducedArguments, prog.getArguments());
        Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testPlanWithExternalClass.

/**
	 * Ensure that we will never have the following error.
	 *
	 * <pre>
	 * 	org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
	 *		at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398)
	 *		at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
	 *		at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140)
	 *		at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125)
	 *		at org.apache.flink.client.CliFrontend.info(CliFrontend.java:439)
	 *		at org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931)
	 *		at org.apache.flink.client.CliFrontend.main(CliFrontend.java:951)
	 *	Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat
	 *		at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102)
	 *		at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54)
	 *		at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322)
	 *		at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380)
	 *		at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	 *		at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	 *		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	 *		at java.lang.reflect.Method.invoke(Method.java:622)
	 *		at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
	 * </pre>
	 *
	 * The test works as follows:
	 *
	 * <ul>
	 *   <li> Use the CliFrontend to invoke a jar file that loads a class which is only available
	 * 	      in the jarfile itself (via a custom classloader)
	 *   <li> Change the Usercode classloader of the PackagedProgram to a special classloader for this test
	 *   <li> the classloader will accept the special class (and return a String.class)
	 * </ul>
	 */
@Test
public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException {
    // create a final object reference, to be able to change its val later
    final boolean[] callme = { false };
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-c", TEST_JAR_CLASSLOADERTEST_CLASS, getTestJarPath(), "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = { "true", "arg1", "arg2" };
        RunOptions options = CliFrontendParser.parseRunCommand(arguments);
        assertEquals(getTestJarPath(), options.getJarFilePath());
        assertArrayEquals(classpath, options.getClasspaths().toArray());
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, options.getEntryPointClassName());
        assertArrayEquals(reducedArguments, options.getProgramArgs());
        CliFrontend frontend = new CliFrontend(CliFrontendTestUtils.getConfigDir());
        PackagedProgram prog = spy(frontend.buildProgram(options));
        ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {

            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
                if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) {
                    callme[0] = true;
                    // Intentionally return the wrong class.
                    return String.class;
                } else {
                    return super.loadClass(name);
                }
            }
        };
        when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
        assertArrayEquals(reducedArguments, prog.getArguments());
        Configuration c = new Configuration();
        Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);
        // we expect this to fail with a "ClassNotFoundException"
        ClusterClient.getOptimizedPlanAsJson(compiler, prog, 666);
        fail("Should have failed with a ClassNotFoundException");
    } catch (ProgramInvocationException e) {
        if (!(e.getCause() instanceof ClassNotFoundException)) {
            e.printStackTrace();
            fail("Program didn't throw ClassNotFoundException");
        }
        assertTrue("Classloader was not called", callme[0]);
    } catch (Exception e) {
        e.printStackTrace();
        fail("Program failed with the wrong exception: " + e.getClass().getName());
    }
}

Example 5 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class AvroExternalJarProgramITCase method testExternalProgram.

@Test
public void testExternalProgram() {
    LocalFlinkMiniCluster testMiniCluster = null;
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
        testMiniCluster = new LocalFlinkMiniCluster(config, false);
        testMiniCluster.start();
        String jarFile = JAR_FILE;
        String testData = getClass().getResource(TEST_DATA_FILE).toString();
        PackagedProgram program = new PackagedProgram(new File(jarFile), new String[] { testData });
        config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
        config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, testMiniCluster.getLeaderRPCPort());
        ClusterClient client = new StandaloneClusterClient(config);
        client.setPrintStatusDuringExecution(false);
        client.run(program, 4);
    } catch (Throwable t) {
        System.err.println(t.getMessage());
        t.printStackTrace();
        Assert.fail("Error during the packaged program execution: " + t.getMessage());
    } finally {
        if (testMiniCluster != null) {
            try {
                testMiniCluster.stop();
            } catch (Throwable t) {
            // ignore
            }
        }
    }
}