Examples with DataStatistics org.apache.flink.optimizer.DataStatistics used on opensource projects

Example 6 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class LocalExecutor method executePlan.

/**
	 * Executes the given program on a local runtime and waits for the job to finish.
	 * 
	 * <p>If the executor has not been started before, this starts the executor and shuts it down
	 * after the job finished. If the job runs in session mode, the executor is kept alive until
	 * no more references to the executor exist.</p>
	 * 
	 * @param plan The plan of the program to execute.
	 * @return The net runtime of the program, in milliseconds.
	 * 
	 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
	 *                   caused an exception.
	 */
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
    if (plan == null) {
        throw new IllegalArgumentException("The plan may not be null.");
    }
    synchronized (this.lock) {
        // check if we start a session dedicated for this execution
        final boolean shutDownAtEnd;
        if (flink == null) {
            shutDownAtEnd = true;
            // configure the number of local slots equal to the parallelism of the local plan
            if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
                int maxParallelism = plan.getMaximumParallelism();
                if (maxParallelism > 0) {
                    this.taskManagerNumSlots = maxParallelism;
                }
            }
            // start the cluster for us
            start();
        } else {
            // we use the existing session
            shutDownAtEnd = false;
        }
        try {
            Configuration configuration = this.flink.configuration();
            Optimizer pc = new Optimizer(new DataStatistics(), configuration);
            OptimizedPlan op = pc.compile(plan);
            JobGraphGenerator jgg = new JobGraphGenerator(configuration);
            JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());
            boolean sysoutPrint = isPrintingStatusDuringExecution();
            return flink.submitJobAndWait(jobGraph, sysoutPrint);
        } finally {
            if (shutDownAtEnd) {
                stop();
            }
        }
    }
}

Example 7 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class RemoteExecutor method getOptimizerPlanAsJSON.

@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
    OptimizedPlan optPlan = opt.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan);
}

Example 8 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class ClientTest method testGetExecutionPlan.

@Test
public void testGetExecutionPlan() {
    try {
        jobManagerSystem.actorOf(Props.create(FailureReturningActor.class), JobManager.JOB_MANAGER_NAME());
        PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
        assertNotNull(prg.getPreviewPlan());
        Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
        OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1);
        assertNotNull(op);
        PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
        assertNotNull(dumper.getOptimizerPlanAsJSON(op));
        // test HTML escaping
        PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
        dumper2.setEncodeForHTML(true);
        String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);
        assertEquals(-1, htmlEscaped.indexOf('\\'));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class LocalExecutor method getOptimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 *
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), this.configuration);
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

Example 10 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class LocalExecutor method optimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 * 
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}