Examples with PackagedProgram org.apache.flink.client.program.PackagedProgram used on opensource projects

Example 21 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontend method run.

// --------------------------------------------------------------------------------------------
//  Execute Actions
// --------------------------------------------------------------------------------------------
/**
	 * Executions the run action.
	 * 
	 * @param args Command line arguments for the run action.
	 */
protected int run(String[] args) {
    LOG.info("Running 'run' command.");
    RunOptions options;
    try {
        options = CliFrontendParser.parseRunCommand(args);
    } catch (CliArgsException e) {
        return handleArgException(e);
    } catch (Throwable t) {
        return handleError(t);
    }
    // evaluate help flag
    if (options.isPrintHelp()) {
        CliFrontendParser.printHelpForRun();
        return 0;
    }
    if (options.getJarFilePath() == null) {
        return handleArgException(new CliArgsException("The program JAR file was not specified."));
    }
    PackagedProgram program;
    try {
        LOG.info("Building program from JAR file");
        program = buildProgram(options);
    } catch (FileNotFoundException e) {
        return handleArgException(e);
    } catch (Throwable t) {
        return handleError(t);
    }
    ClusterClient client = null;
    try {
        client = createClient(options, program);
        client.setPrintStatusDuringExecution(options.getStdoutLogging());
        client.setDetached(options.getDetachedMode());
        LOG.debug("Client slots is set to {}", client.getMaxSlots());
        LOG.debug(options.getSavepointRestoreSettings().toString());
        int userParallelism = options.getParallelism();
        LOG.debug("User parallelism is set to {}", userParallelism);
        if (client.getMaxSlots() != -1 && userParallelism == -1) {
            logAndSysout("Using the parallelism provided by the remote cluster (" + client.getMaxSlots() + "). " + "To use another parallelism, set it at the ./bin/flink client.");
            userParallelism = client.getMaxSlots();
        }
        return executeProgram(program, client, userParallelism);
    } catch (Throwable t) {
        return handleError(t);
    } finally {
        if (client != null) {
            client.shutdown();
        }
        if (program != null) {
            program.deleteExtractedLibraries();
        }
    }
}

Example 22 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testValidVariantWithNoJarAndNoArgumentsOption.

@Test
public void testValidVariantWithNoJarAndNoArgumentsOption() {
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", getTestJarPath(), "--debug", "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = { "--debug", "true", "arg1", "arg2" };
        RunOptions options = CliFrontendParser.parseRunCommand(arguments);
        assertEquals(getTestJarPath(), options.getJarFilePath());
        assertArrayEquals(classpath, options.getClasspaths().toArray());
        assertArrayEquals(reducedArguments, options.getProgramArgs());
        CliFrontend frontend = new CliFrontend(CliFrontendTestUtils.getConfigDir());
        PackagedProgram prog = frontend.buildProgram(options);
        Assert.assertArrayEquals(reducedArguments, prog.getArguments());
        Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 23 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testVariantWithExplicitJarAndArgumentsOption.

@Test
public void testVariantWithExplicitJarAndArgumentsOption() {
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-j", getTestJarPath(), "-a", "--debug", "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = new String[] { "--debug", "true", "arg1", "arg2" };
        RunOptions options = CliFrontendParser.parseRunCommand(arguments);
        assertEquals(getTestJarPath(), options.getJarFilePath());
        assertArrayEquals(classpath, options.getClasspaths().toArray());
        assertArrayEquals(reducedArguments, options.getProgramArgs());
        CliFrontend frontend = new CliFrontend(CliFrontendTestUtils.getConfigDir());
        PackagedProgram prog = frontend.buildProgram(options);
        Assert.assertArrayEquals(reducedArguments, prog.getArguments());
        Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 24 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class PythonProgramOptionsITCase method testConfigurePythonExecution.

/**
 * It requires setting a job jar to build a {@link PackagedProgram}, and the dummy job jar used
 * in this test case is available only after the packaging phase completed, so we make it as an
 * ITCase.
 */
@Test
public void testConfigurePythonExecution() throws Exception {
    final String[] args = { "--python", "xxx.py", "--pyModule", "xxx", "--pyFiles", "/absolute/a.py,relative/b.py,relative/c.py", "--pyRequirements", "d.txt#e_dir", "--pyExecutable", "/usr/bin/python", "--pyArchives", "g.zip,h.zip#data,h.zip#data2", "userarg1", "userarg2" };
    final File[] dummyJobJar = { null };
    Files.walkFileTree(FileSystems.getDefault().getPath(System.getProperty("user.dir") + "/artifacts"), new SimpleFileVisitor<Path>() {

        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
            FileVisitResult result = super.visitFile(file, attrs);
            if (file.getFileName().toString().startsWith("dummy")) {
                dummyJobJar[0] = file.toFile();
            }
            return result;
        }
    });
    PackagedProgram packagedProgram = PackagedProgram.newBuilder().setArguments(args).setJarFile(dummyJobJar[0]).build();
    Configuration configuration = new Configuration();
    ProgramOptionsUtils.configurePythonExecution(configuration, packagedProgram);
    assertEquals("/absolute/a.py,relative/b.py,relative/c.py", configuration.get(PythonOptions.PYTHON_FILES));
    assertEquals("d.txt#e_dir", configuration.get(PYTHON_REQUIREMENTS));
    assertEquals("g.zip,h.zip#data,h.zip#data2", configuration.get(PythonOptions.PYTHON_ARCHIVES));
    assertEquals("/usr/bin/python", configuration.get(PYTHON_EXECUTABLE));
    assertArrayEquals(new String[] { "--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2" }, packagedProgram.getArguments());
}

Example 25 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class JarRunHandler method handleRequest.

@Override
@VisibleForTesting
public CompletableFuture<JarRunResponseBody> handleRequest(@Nonnull final HandlerRequest<JarRunRequestBody> request, @Nonnull final DispatcherGateway gateway) throws RestHandlerException {
    final Configuration effectiveConfiguration = new Configuration(configuration);
    effectiveConfiguration.set(DeploymentOptions.ATTACHED, false);
    effectiveConfiguration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);
    final JarHandlerContext context = JarHandlerContext.fromRequest(request, jarDir, log);
    context.applyToConfiguration(effectiveConfiguration);
    SavepointRestoreSettings.toConfiguration(getSavepointRestoreSettings(request), effectiveConfiguration);
    final PackagedProgram program = context.toPackagedProgram(effectiveConfiguration);
    return CompletableFuture.supplyAsync(() -> applicationRunner.run(gateway, program, effectiveConfiguration), executor).handle((jobIds, throwable) -> {
        program.close();
        if (throwable != null) {
            throw new CompletionException(new RestHandlerException("Could not execute application.", HttpResponseStatus.BAD_REQUEST, throwable));
        } else if (jobIds.isEmpty()) {
            throw new CompletionException(new RestHandlerException("No jobs included in application.", HttpResponseStatus.BAD_REQUEST));
        }
        return new JarRunResponseBody(jobIds.get(0));
    });
}