Examples with PackagedProgram org.apache.flink.client.program.PackagedProgram used on opensource projects

Example 26 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class JarListHandler method handleRequest.

@Override
protected CompletableFuture<JarListInfo> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
    final String localAddress;
    Preconditions.checkState(localAddressFuture.isDone());
    try {
        localAddress = localAddressFuture.get();
    } catch (Exception e) {
        return FutureUtils.completedExceptionally(e);
    }
    return CompletableFuture.supplyAsync(() -> {
        try {
            final File[] list = getJarFiles();
            final List<JarListInfo.JarFileInfo> jarFileList = new ArrayList<>(list.length);
            for (File f : list) {
                // separate the uuid and the name parts.
                String id = f.getName();
                int startIndex = id.indexOf("_");
                if (startIndex < 0) {
                    continue;
                }
                String name = id.substring(startIndex + 1);
                if (name.length() < 5 || !name.endsWith(".jar")) {
                    continue;
                }
                List<JarListInfo.JarEntryInfo> jarEntryList = new ArrayList<>();
                String[] classes = new String[0];
                try (JarFile jar = new JarFile(f)) {
                    Manifest manifest = jar.getManifest();
                    String assemblerClass = null;
                    if (manifest != null) {
                        assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS);
                        if (assemblerClass == null) {
                            assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS);
                        }
                    }
                    if (assemblerClass != null) {
                        classes = assemblerClass.split(",");
                    }
                } catch (IOException ignored) {
                // we simply show no entries here
                }
                // show every entry class that can be loaded later on.
                for (String clazz : classes) {
                    clazz = clazz.trim();
                    try (PackagedProgram program = PackagedProgram.newBuilder().setJarFile(f).setEntryPointClassName(clazz).setConfiguration(configuration).build()) {
                        JarListInfo.JarEntryInfo jarEntryInfo = new JarListInfo.JarEntryInfo(clazz, program.getDescription());
                        jarEntryList.add(jarEntryInfo);
                    } catch (Exception ignored) {
                    // ignore jar files which throw an error upon creating a
                    // PackagedProgram
                    }
                }
                jarFileList.add(new JarListInfo.JarFileInfo(id, name, f.lastModified(), jarEntryList));
            }
            return new JarListInfo(localAddress, jarFileList);
        } catch (Exception e) {
            throw new CompletionException(new FlinkException("Failed to fetch jar list.", e));
        }
    }, executor);
}

Example 27 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontend method getPackagedProgram.

private PackagedProgram getPackagedProgram(ProgramOptions programOptions, Configuration effectiveConfiguration) throws ProgramInvocationException, CliArgsException {
    PackagedProgram program;
    try {
        LOG.info("Building program from JAR file");
        program = buildProgram(programOptions, effectiveConfiguration);
    } catch (FileNotFoundException e) {
        throw new CliArgsException("Could not build the program from JAR file: " + e.getMessage(), e);
    }
    return program;
}

Example 28 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testPlanWithExternalClass.

/**
 * Ensure that we will never have the following error.
 *
 * <pre>
 * 	org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398)
 * 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125)
 * 	at org.apache.flink.client.cli.CliFrontend.info(CliFrontend.java:439)
 * 	at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:931)
 * 	at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:951)
 * Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102)
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54)
 * 	at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322)
 * 	at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
 * 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 * 	at java.lang.reflect.Method.invoke(Method.java:622)
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
 * </pre>
 *
 * <p>The test works as follows:
 *
 * <ul>
 *   <li>Use the CliFrontend to invoke a jar file that loads a class which is only available in
 *       the jarfile itself (via a custom classloader)
 *   <li>Change the Usercode classloader of the PackagedProgram to a special classloader for
 *       this test
 *   <li>the classloader will accept the special class (and return a String.class)
 * </ul>
 */
@Test
public void testPlanWithExternalClass() throws Exception {
    final boolean[] callme = { false };
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-c", TEST_JAR_CLASSLOADERTEST_CLASS, getTestJarPath(), "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = { "true", "arg1", "arg2" };
        CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
        ProgramOptions programOptions = ProgramOptions.create(commandLine);
        assertEquals(getTestJarPath(), programOptions.getJarFilePath());
        assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, programOptions.getEntryPointClassName());
        assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
        PackagedProgram prog = spy(frontend.buildProgram(programOptions));
        ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {

            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
                if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) {
                    callme[0] = true;
                    // Intentionally return the wrong class.
                    return String.class;
                } else {
                    return super.loadClass(name);
                }
            }
        };
        when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
        assertArrayEquals(reducedArguments, prog.getArguments());
        Configuration c = new Configuration();
        Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);
        // we expect this to fail with a "ClassNotFoundException"
        Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(prog, c, 666, true);
        FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);
        fail("Should have failed with a ClassNotFoundException");
    } catch (ProgramInvocationException e) {
        if (!(e.getCause() instanceof ClassNotFoundException)) {
            e.printStackTrace();
            fail("Program didn't throw ClassNotFoundException");
        }
        assertTrue("Classloader was not called", callme[0]);
    }
}

Example 29 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testVariantWithExplicitJarAndArgumentsOption.

@Test
public void testVariantWithExplicitJarAndArgumentsOption() throws Exception {
    String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-j", getTestJarPath(), "-a", "--debug", "true", "arg1", "arg2" };
    URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
    String[] reducedArguments = new String[] { "--debug", "true", "arg1", "arg2" };
    CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
    ProgramOptions programOptions = ProgramOptions.create(commandLine);
    assertEquals(getTestJarPath(), programOptions.getJarFilePath());
    assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
    assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
    PackagedProgram prog = frontend.buildProgram(programOptions);
    Assert.assertArrayEquals(reducedArguments, prog.getArguments());
    Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
}

Example 30 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testVariantWithExplicitJarAndNoArgumentsOption.

@Test
public void testVariantWithExplicitJarAndNoArgumentsOption() throws Exception {
    String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-j", getTestJarPath(), "--debug", "true", "arg1", "arg2" };
    URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
    String[] reducedArguments = new String[] { "--debug", "true", "arg1", "arg2" };
    CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
    ProgramOptions programOptions = ProgramOptions.create(commandLine);
    assertEquals(getTestJarPath(), programOptions.getJarFilePath());
    assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
    assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
    PackagedProgram prog = frontend.buildProgram(programOptions);
    Assert.assertArrayEquals(reducedArguments, prog.getArguments());
    Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
}