Examples with DataStatistics org.apache.flink.optimizer.DataStatistics used on opensource projects

Example 11 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class ExecutionPlanCreationTest method testGetExecutionPlan.

@Test
public void testGetExecutionPlan() {
    try {
        PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
        assertNotNull(prg.getPreviewPlan());
        InetAddress mockAddress = InetAddress.getLocalHost();
        InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);
        Configuration config = new Configuration();
        config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, mockJmAddress.getHostName());
        config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, mockJmAddress.getPort());
        Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
        OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1);
        assertNotNull(op);
        PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
        assertNotNull(dumper.getOptimizerPlanAsJSON(op));
        // test HTML escaping
        PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
        dumper2.setEncodeForHTML(true);
        String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);
        assertEquals(-1, htmlEscaped.indexOf('\\'));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class JarActionHandler method getJobGraphAndClassLoader.

protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
    // generate the graph
    JobGraph graph = null;
    PackagedProgram program = new PackagedProgram(new File(jarDir, config.getJarFile()), config.getEntryClass(), config.getProgramArgs());
    ClassLoader classLoader = program.getUserCodeClassLoader();
    Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
    FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());
    if (plan instanceof StreamingPlan) {
        graph = ((StreamingPlan) plan).getJobGraph();
    } else if (plan instanceof OptimizedPlan) {
        graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
    }
    if (graph == null) {
        throw new CompilerException("A valid job graph couldn't be generated for the jar.");
    }
    // Set the savepoint settings
    graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());
    for (URL jar : program.getAllLibraries()) {
        try {
            graph.addJar(new Path(jar.toURI()));
        } catch (URISyntaxException e) {
            throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
        }
    }
    return Tuple2.of(graph, classLoader);
}

Example 13 with DataStatistics

use of org.apache.flink.optimizer.DataStatistics in project flink by apache.

the class TestEnvironment method compileProgram.

private OptimizedPlan compileProgram(String jobName) {
    Plan p = createProgramPlan(jobName);
    Optimizer pc = new Optimizer(new DataStatistics(), this.executor.configuration());
    return pc.compile(p);
}