Examples with DefaultCostEstimator org.apache.flink.optimizer.costs.DefaultCostEstimator used on opensource projects

Example 11 with DefaultCostEstimator

use of org.apache.flink.optimizer.costs.DefaultCostEstimator in project flink by apache.

the class ExecutionPlanCreationTest method testGetExecutionPlan.

@Test
public void testGetExecutionPlan() {
    try {
        PackagedProgram prg = PackagedProgram.newBuilder().setEntryPointClassName(TestOptimizerPlan.class.getName()).setArguments("/dev/random", "/tmp").build();
        InetAddress mockAddress = InetAddress.getLocalHost();
        InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);
        Configuration config = new Configuration();
        config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName());
        config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort());
        Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
        Plan plan = (Plan) PackagedProgramUtils.getPipelineFromProgram(prg, config, -1, true);
        OptimizedPlan op = optimizer.compile(plan);
        assertNotNull(op);
        PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
        assertNotNull(dumper.getOptimizerPlanAsJSON(op));
        // test HTML escaping
        PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
        dumper2.setEncodeForHTML(true);
        String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);
        assertEquals(-1, htmlEscaped.indexOf('\\'));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with DefaultCostEstimator

use of org.apache.flink.optimizer.costs.DefaultCostEstimator in project flink by apache.

the class CliFrontendPackageProgramTest method testPlanWithExternalClass.

/**
 * Ensure that we will never have the following error.
 *
 * <pre>
 * 	org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398)
 * 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140)
 * 	at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125)
 * 	at org.apache.flink.client.cli.CliFrontend.info(CliFrontend.java:439)
 * 	at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:931)
 * 	at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:951)
 * Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102)
 * 	at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54)
 * 	at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322)
 * 	at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 * 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
 * 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 * 	at java.lang.reflect.Method.invoke(Method.java:622)
 * 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
 * </pre>
 *
 * <p>The test works as follows:
 *
 * <ul>
 *   <li>Use the CliFrontend to invoke a jar file that loads a class which is only available in
 *       the jarfile itself (via a custom classloader)
 *   <li>Change the Usercode classloader of the PackagedProgram to a special classloader for
 *       this test
 *   <li>the classloader will accept the special class (and return a String.class)
 * </ul>
 */
@Test
public void testPlanWithExternalClass() throws Exception {
    final boolean[] callme = { false };
    try {
        String[] arguments = { "--classpath", "file:///tmp/foo", "--classpath", "file:///tmp/bar", "-c", TEST_JAR_CLASSLOADERTEST_CLASS, getTestJarPath(), "true", "arg1", "arg2" };
        URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
        String[] reducedArguments = { "true", "arg1", "arg2" };
        CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
        ProgramOptions programOptions = ProgramOptions.create(commandLine);
        assertEquals(getTestJarPath(), programOptions.getJarFilePath());
        assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, programOptions.getEntryPointClassName());
        assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
        PackagedProgram prog = spy(frontend.buildProgram(programOptions));
        ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {

            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
                if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) {
                    callme[0] = true;
                    // Intentionally return the wrong class.
                    return String.class;
                } else {
                    return super.loadClass(name);
                }
            }
        };
        when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
        assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
        assertArrayEquals(reducedArguments, prog.getArguments());
        Configuration c = new Configuration();
        Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);
        // we expect this to fail with a "ClassNotFoundException"
        Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(prog, c, 666, true);
        FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);
        fail("Should have failed with a ClassNotFoundException");
    } catch (ProgramInvocationException e) {
        if (!(e.getCause() instanceof ClassNotFoundException)) {
            e.printStackTrace();
            fail("Program didn't throw ClassNotFoundException");
        }
        assertTrue("Classloader was not called", callme[0]);
    }
}