Examples with Pipeline org.apache.flink.api.dag.Pipeline used on opensource projects

Example 1 with Pipeline

use of org.apache.flink.api.dag.Pipeline in project flink by apache.

the class AbstractSessionClusterExecutor method execute.

@Override
public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration, @Nonnull final ClassLoader userCodeClassloader) throws Exception {
    final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);
    try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
        final ClusterID clusterID = clusterClientFactory.getClusterId(configuration);
        checkState(clusterID != null);
        final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor.retrieve(clusterID);
        ClusterClient<ClusterID> clusterClient = clusterClientProvider.getClusterClient();
        return clusterClient.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction(jobId -> {
            ClientUtils.waitUntilJobInitializationFinished(() -> clusterClient.getJobStatus(jobId).get(), () -> clusterClient.requestJobResult(jobId).get(), userCodeClassloader);
            return jobId;
        })).thenApplyAsync(jobID -> (JobClient) new ClusterClientJobClientAdapter<>(clusterClientProvider, jobID, userCodeClassloader)).whenCompleteAsync((ignored1, ignored2) -> clusterClient.close());
    }
}

Example 2 with Pipeline

use of org.apache.flink.api.dag.Pipeline in project flink by apache.

the class PackagedProgramUtilsPipelineTest method testUserClassloaderForConfiguration.

@Test
public void testUserClassloaderForConfiguration() throws Exception {
    String userSerializerClassName = "UserSerializer";
    List<URL> userUrls = getClassUrls(userSerializerClassName);
    PackagedProgram packagedProgram = PackagedProgram.newBuilder().setUserClassPaths(userUrls).setEntryPointClassName(testParameter.entryClass().getName()).build();
    Configuration config = new Configuration();
    config.set(PipelineOptions.KRYO_DEFAULT_SERIALIZERS, Collections.singletonList(String.format("class:%s,serializer:%s", PackagedProgramUtilsPipelineTest.class.getName(), userSerializerClassName)));
    Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(packagedProgram, config, 1, /* parallelism */
    false);
    ExecutionConfig executionConfig = testParameter.extractExecutionConfig(pipeline);
    assertThat(executionConfig.getDefaultKryoSerializerClasses().get(PackagedProgramUtilsPipelineTest.class).getName(), is(userSerializerClassName));
}

Example 3 with Pipeline

use of org.apache.flink.api.dag.Pipeline in project flink by apache.

the class PackagedProgramUtilsPipelineTest method testConfigurationForwarding.

/**
 * This tests whether configuration forwarding from a {@link Configuration} to the environment
 * works.
 */
@Test
public void testConfigurationForwarding() throws Exception {
    // we want to test forwarding with this config, ensure that the default is what we expect.
    assertThat(ExecutionEnvironment.getExecutionEnvironment().getConfig().isAutoTypeRegistrationDisabled(), is(false));
    PackagedProgram packagedProgram = PackagedProgram.newBuilder().setEntryPointClassName(testParameter.entryClass().getName()).build();
    Configuration config = new Configuration();
    config.set(PipelineOptions.AUTO_TYPE_REGISTRATION, false);
    Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(packagedProgram, config, 1, /* parallelism */
    false);
    ExecutionConfig executionConfig = testParameter.extractExecutionConfig(pipeline);
    // we want to test forwarding with this config, ensure that the default is what we expect.
    assertThat(executionConfig.isAutoTypeRegistrationDisabled(), is(true));
}

Example 4 with Pipeline

use of org.apache.flink.api.dag.Pipeline in project flink by apache.

the class TableEnvironmentImpl method executeQueryOperation.

private TableResultInternal executeQueryOperation(QueryOperation operation) {
    CollectModifyOperation sinkOperation = new CollectModifyOperation(operation);
    List<Transformation<?>> transformations = translate(Collections.singletonList(sinkOperation));
    final String defaultJobName = "collect";
    Pipeline pipeline = execEnv.createPipeline(transformations, tableConfig.getConfiguration(), defaultJobName);
    try {
        JobClient jobClient = execEnv.executeAsync(pipeline);
        ResultProvider resultProvider = sinkOperation.getSelectResultProvider();
        resultProvider.setJobClient(jobClient);
        return TableResultImpl.builder().jobClient(jobClient).resultKind(ResultKind.SUCCESS_WITH_CONTENT).schema(operation.getResolvedSchema()).resultProvider(resultProvider).setPrintStyle(PrintStyle.tableauWithTypeInferredColumnWidths(// sinkOperation.getConsumedDataType() handles legacy types
        DataTypeUtils.expandCompositeTypeToSchema(sinkOperation.getConsumedDataType()), resultProvider.getRowDataStringConverter(), PrintStyle.DEFAULT_MAX_COLUMN_WIDTH, false, isStreamingMode)).build();
    } catch (Exception e) {
        throw new TableException("Failed to execute sql", e);
    }
}

Example 5 with Pipeline

use of org.apache.flink.api.dag.Pipeline in project flink by apache.

the class CliFrontendPackageProgramTest method testPlanWithExternalClass.

/**
 * Ensure that we will never have the following error.
 *
 * <pre>
 * 	org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398)
 * 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125)
 * 	at org.apache.flink.client.cli.CliFrontend.info(CliFrontend.java:439)
 * 	at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:931)
 * 	at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:951)
 * Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102)
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54)
 * 	at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322)
 * 	at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
 * 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 * 	at java.lang.reflect.Method.invoke(Method.java:622)
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
 * </pre>
 *
 * <p>The test works as follows:
 *
 * <ul>
 *   <li>Use the CliFrontend to invoke a jar file that loads a class which is only available in
 *       the jarfile itself (via a custom classloader)
 *   <li>Change the Usercode classloader of the PackagedProgram to a special classloader for
 *       this test
 *   <li>the classloader will accept the special class (and return a String.class)
 * </ul>
 */
@Test
public void testPlanWithExternalClass() throws Exception {
    final boolean[] callme = { false };
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-c", TEST_JAR_CLASSLOADERTEST_CLASS, getTestJarPath(), "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = { "true", "arg1", "arg2" };
        CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
        ProgramOptions programOptions = ProgramOptions.create(commandLine);
        assertEquals(getTestJarPath(), programOptions.getJarFilePath());
        assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, programOptions.getEntryPointClassName());
        assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
        PackagedProgram prog = spy(frontend.buildProgram(programOptions));
        ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {

            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
                if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) {
                    callme[0] = true;
                    // Intentionally return the wrong class.
                    return String.class;
                } else {
                    return super.loadClass(name);
                }
            }
        };
        when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
        assertArrayEquals(reducedArguments, prog.getArguments());
        Configuration c = new Configuration();
        Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);
        // we expect this to fail with a "ClassNotFoundException"
        Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(prog, c, 666, true);
        FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);
        fail("Should have failed with a ClassNotFoundException");
    } catch (ProgramInvocationException e) {
        if (!(e.getCause() instanceof ClassNotFoundException)) {
            e.printStackTrace();
            fail("Program didn't throw ClassNotFoundException");
        }
        assertTrue("Classloader was not called", callme[0]);
    }
}