Examples with PackagedProgram org.apache.flink.client.program.PackagedProgram used on opensource projects

Example 6 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class ClassLoaderITCase method testJobsWithCustomClassLoader.

@Test
public void testJobsWithCustomClassLoader() {
    try {
        int port = testCluster.getLeaderRPCPort();
        PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE), new String[] { INPUT_SPLITS_PROG_JAR_FILE, // classpath
        "", "localhost", String.valueOf(port), // parallelism
        "4" });
        inputSplitTestProg.invokeInteractiveModeForExecution();
        PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE), new String[] { STREAMING_INPUT_SPLITS_PROG_JAR_FILE, "localhost", String.valueOf(port), // parallelism
        "4" });
        streamingInputSplitTestProg.invokeInteractiveModeForExecution();
        String classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL().toString();
        PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE), new String[] { "", // classpath
        classpath, "localhost", String.valueOf(port), // parallelism
        "4" });
        inputSplitTestProg2.invokeInteractiveModeForExecution();
        // regular streaming job
        PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE), new String[] { STREAMING_PROG_JAR_FILE, "localhost", String.valueOf(port) });
        streamingProg.invokeInteractiveModeForExecution();
        // the test also ensures that user specific exceptions are serializable between JobManager <--> JobClient.
        try {
            PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE), new String[] { STREAMING_CHECKPOINTED_PROG_JAR_FILE, "localhost", String.valueOf(port) });
            streamingCheckpointedProg.invokeInteractiveModeForExecution();
        } catch (Exception e) {
            // we can not access the SuccessException here when executing the tests with maven, because its not available in the jar.
            assertEquals("Program should terminate with a 'SuccessException'", "org.apache.flink.test.classloading.jar.CheckpointedStreamingProgram.SuccessException", e.getCause().getCause().getClass().getCanonicalName());
        }
        PackagedProgram kMeansProg = new PackagedProgram(new File(KMEANS_JAR_PATH), new String[] { KMEANS_JAR_PATH, "localhost", String.valueOf(port), // parallelism
        "4", KMeansData.DATAPOINTS, KMeansData.INITIAL_CENTERS, "25" });
        kMeansProg.invokeInteractiveModeForExecution();
        // test FLINK-3633
        final PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH), new String[] { USERCODETYPE_JAR_PATH, "localhost", String.valueOf(port) });
        userCodeTypeProg.invokeInteractiveModeForExecution();
        File checkpointDir = FOLDER.newFolder();
        File outputDir = FOLDER.newFolder();
        final PackagedProgram program = new PackagedProgram(new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH), new String[] { CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH, "localhost", String.valueOf(port), checkpointDir.toURI().toString(), outputDir.toURI().toString() });
        program.invokeInteractiveModeForExecution();
    } catch (Exception e) {
        if (!(e.getCause().getCause() instanceof SuccessException)) {
            fail(e.getMessage());
        }
    }
}

Example 7 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class AvroExternalJarProgramITCase method testExternalProgram.

@Test
public void testExternalProgram() throws Exception {
    String jarFile = JAR_FILE;
    try {
        JarUtils.checkJarFile(new File(jarFile).getAbsoluteFile().toURI().toURL());
    } catch (IOException e) {
        jarFile = "target/".concat(jarFile);
    }
    TestEnvironment.setAsContext(MINI_CLUSTER, PARALLELISM, Collections.singleton(new Path(jarFile)), Collections.emptyList());
    String testData = getClass().getResource(TEST_DATA_FILE).toString();
    PackagedProgram program = PackagedProgram.newBuilder().setJarFile(new File(jarFile)).setArguments(new String[] { testData }).build();
    program.invokeInteractiveModeForExecution();
}

Example 8 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class StandaloneApplicationClusterEntryPoint method main.

public static void main(String[] args) {
    // startup checks and logging
    EnvironmentInformation.logEnvironmentInfo(LOG, StandaloneApplicationClusterEntryPoint.class.getSimpleName(), args);
    SignalHandler.register(LOG);
    JvmShutdownSafeguard.installAsShutdownHook(LOG);
    final StandaloneApplicationClusterConfiguration clusterConfiguration = ClusterEntrypointUtils.parseParametersOrExit(args, new StandaloneApplicationClusterConfigurationParserFactory(), StandaloneApplicationClusterEntryPoint.class);
    Configuration configuration = loadConfigurationFromClusterConfig(clusterConfiguration);
    PackagedProgram program = null;
    try {
        program = getPackagedProgram(clusterConfiguration, configuration);
    } catch (Exception e) {
        LOG.error("Could not create application program.", e);
        System.exit(1);
    }
    try {
        configureExecution(configuration, program);
    } catch (Exception e) {
        LOG.error("Could not apply application configuration.", e);
        System.exit(1);
    }
    StandaloneApplicationClusterEntryPoint entrypoint = new StandaloneApplicationClusterEntryPoint(configuration, program);
    ClusterEntrypoint.runClusterEntrypoint(entrypoint);
}

Example 9 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class CliFrontendPackageProgramTest method testValidVariantWithNoJarAndNoArgumentsOption.

@Test
public void testValidVariantWithNoJarAndNoArgumentsOption() throws Exception {
    String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", getTestJarPath(), "--debug", "true", "arg1", "arg2" };
    URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
    String[] reducedArguments = { "--debug", "true", "arg1", "arg2" };
    CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
    ProgramOptions programOptions = ProgramOptions.create(commandLine);
    assertEquals(getTestJarPath(), programOptions.getJarFilePath());
    assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
    assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
    PackagedProgram prog = frontend.buildProgram(programOptions);
    Assert.assertArrayEquals(reducedArguments, prog.getArguments());
    Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
}

Example 10 with PackagedProgram

use of org.apache.flink.client.program.PackagedProgram in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testClusterIsShutdownInAttachedModeWhenJobCancelled.

@Test
public void testClusterIsShutdownInAttachedModeWhenJobCancelled() throws Exception {
    final CompletableFuture<ApplicationStatus> clusterShutdown = new CompletableFuture<>();
    final TestingDispatcherGateway dispatcherGateway = canceledJobGatewayBuilder().setClusterShutdownFunction(status -> {
        clusterShutdown.complete(status);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    final PackagedProgram program = getProgram(2);
    final Configuration configuration = getConfiguration();
    configuration.set(DeploymentOptions.ATTACHED, true);
    final ApplicationDispatcherBootstrap bootstrap = new ApplicationDispatcherBootstrap(program, Collections.emptyList(), configuration, dispatcherGateway, scheduledExecutor, e -> {
    });
    final CompletableFuture<Void> applicationFuture = bootstrap.getApplicationCompletionFuture();
    assertException(applicationFuture, UnsuccessfulExecutionException.class);
    assertEquals(clusterShutdown.get(), ApplicationStatus.CANCELED);
}