Examples with Optimizer org.apache.flink.optimizer.Optimizer used on opensource projects

Example 11 with Optimizer

use of org.apache.flink.optimizer.Optimizer in project flink by apache.

the class LocalExecutor method getOptimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 *
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), this.configuration);
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

Example 12 with Optimizer

use of org.apache.flink.optimizer.Optimizer in project flink by apache.

the class LocalExecutor method optimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 * 
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

Example 13 with Optimizer

use of org.apache.flink.optimizer.Optimizer in project flink by apache.

the class ExecutionPlanCreationTest method testGetExecutionPlan.

@Test
public void testGetExecutionPlan() {
    try {
        PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
        assertNotNull(prg.getPreviewPlan());
        InetAddress mockAddress = InetAddress.getLocalHost();
        InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);
        Configuration config = new Configuration();
        config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, mockJmAddress.getHostName());
        config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, mockJmAddress.getPort());
        Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
        OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1);
        assertNotNull(op);
        PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
        assertNotNull(dumper.getOptimizerPlanAsJSON(op));
        // test HTML escaping
        PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
        dumper2.setEncodeForHTML(true);
        String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);
        assertEquals(-1, htmlEscaped.indexOf('\\'));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with Optimizer

use of org.apache.flink.optimizer.Optimizer in project flink by apache.

the class TestUtils method verifyParallelism.

/**
	 * Verify operator parallelism.
	 *
	 * @param env the Flink execution environment.
	 * @param expectedParallelism expected operator parallelism
	 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
    env.setParallelism(2 * expectedParallelism);
    Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
    OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());
    List<PlanNode> queue = new ArrayList<>();
    queue.addAll(optimizedPlan.getDataSinks());
    while (queue.size() > 0) {
        PlanNode node = queue.remove(queue.size() - 1);
        // Data sources may have parallelism of 1, so simply check that the node
        // parallelism has not been increased by setting the default parallelism
        assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);
        for (Channel channel : node.getInputs()) {
            queue.add(channel.getSource());
        }
    }
}

Example 15 with Optimizer

use of org.apache.flink.optimizer.Optimizer in project flink by apache.

the class JarActionHandler method getJobGraphAndClassLoader.

protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
    // generate the graph
    JobGraph graph = null;
    PackagedProgram program = new PackagedProgram(new File(jarDir, config.getJarFile()), config.getEntryClass(), config.getProgramArgs());
    ClassLoader classLoader = program.getUserCodeClassLoader();
    Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
    FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());
    if (plan instanceof StreamingPlan) {
        graph = ((StreamingPlan) plan).getJobGraph();
    } else if (plan instanceof OptimizedPlan) {
        graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
    }
    if (graph == null) {
        throw new CompilerException("A valid job graph couldn't be generated for the jar.");
    }
    // Set the savepoint settings
    graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());
    for (URL jar : program.getAllLibraries()) {
        try {
            graph.addJar(new Path(jar.toURI()));
        } catch (URISyntaxException e) {
            throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
        }
    }
    return Tuple2.of(graph, classLoader);
}