Examples with Result org.apache.flink.graph.library.similarity.JaccardIndex.Result used on opensource projects

Example 1 with Result

use of org.apache.flink.graph.library.similarity.JaccardIndex.Result in project flink by apache.

the class JaccardIndex method main.

public static void main(String[] args) throws Exception {
    // Set up the execution environment
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableObjectReuse();
    ParameterTool parameters = ParameterTool.fromArgs(args);
    env.getConfig().setGlobalJobParameters(parameters);
    int little_parallelism = parameters.getInt("little_parallelism", PARALLELISM_DEFAULT);
    DataSet ji;
    switch(parameters.get("input", "")) {
        case "csv":
            {
                String lineDelimiter = StringEscapeUtils.unescapeJava(parameters.get("input_line_delimiter", CsvOutputFormat.DEFAULT_LINE_DELIMITER));
                String fieldDelimiter = StringEscapeUtils.unescapeJava(parameters.get("input_field_delimiter", CsvOutputFormat.DEFAULT_FIELD_DELIMITER));
                GraphCsvReader reader = Graph.fromCsvReader(parameters.getRequired("input_filename"), env).ignoreCommentsEdges("#").lineDelimiterEdges(lineDelimiter).fieldDelimiterEdges(fieldDelimiter);
                switch(parameters.get("type", "")) {
                    case "integer":
                        {
                            Graph<LongValue, NullValue, NullValue> graph = reader.keyType(LongValue.class);
                            if (parameters.getBoolean("simplify", false)) {
                                graph = graph.run(new org.apache.flink.graph.asm.simple.undirected.Simplify<LongValue, NullValue, NullValue>(false).setParallelism(little_parallelism));
                            }
                            ji = graph.run(new org.apache.flink.graph.library.similarity.JaccardIndex<LongValue, NullValue, NullValue>().setLittleParallelism(little_parallelism));
                        }
                        break;
                    case "string":
                        {
                            Graph<StringValue, NullValue, NullValue> graph = reader.keyType(StringValue.class);
                            if (parameters.getBoolean("simplify", false)) {
                                graph = graph.run(new org.apache.flink.graph.asm.simple.undirected.Simplify<StringValue, NullValue, NullValue>(false).setParallelism(little_parallelism));
                            }
                            ji = graph.run(new org.apache.flink.graph.library.similarity.JaccardIndex<StringValue, NullValue, NullValue>().setLittleParallelism(little_parallelism));
                        }
                        break;
                    default:
                        throw new ProgramParametrizationException(getUsage("invalid CSV type"));
                }
            }
            break;
        case "rmat":
            {
                int scale = parameters.getInt("scale", DEFAULT_SCALE);
                int edgeFactor = parameters.getInt("edge_factor", DEFAULT_EDGE_FACTOR);
                RandomGenerableFactory<JDKRandomGenerator> rnd = new JDKRandomGeneratorFactory();
                long vertexCount = 1L << scale;
                long edgeCount = vertexCount * edgeFactor;
                Graph<LongValue, NullValue, NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount).setParallelism(little_parallelism).generate();
                boolean clipAndFlip = parameters.getBoolean("clip_and_flip", DEFAULT_CLIP_AND_FLIP);
                if (scale > 32) {
                    ji = graph.run(new Simplify<LongValue, NullValue, NullValue>(clipAndFlip).setParallelism(little_parallelism)).run(new org.apache.flink.graph.library.similarity.JaccardIndex<LongValue, NullValue, NullValue>().setLittleParallelism(little_parallelism));
                } else {
                    ji = graph.run(new TranslateGraphIds<LongValue, IntValue, NullValue, NullValue>(new LongValueToUnsignedIntValue()).setParallelism(little_parallelism)).run(new Simplify<IntValue, NullValue, NullValue>(clipAndFlip).setParallelism(little_parallelism)).run(new org.apache.flink.graph.library.similarity.JaccardIndex<IntValue, NullValue, NullValue>().setLittleParallelism(little_parallelism));
                }
            }
            break;
        default:
            throw new ProgramParametrizationException(getUsage("invalid input type"));
    }
    switch(parameters.get("output", "")) {
        case "print":
            System.out.println();
            for (Object e : ji.collect()) {
                Result result = (Result) e;
                System.out.println(result.toPrintableString());
            }
            break;
        case "hash":
            System.out.println();
            System.out.println(DataSetUtils.checksumHashCode(ji));
            break;
        case "csv":
            String filename = parameters.getRequired("output_filename");
            String lineDelimiter = StringEscapeUtils.unescapeJava(parameters.get("output_line_delimiter", CsvOutputFormat.DEFAULT_LINE_DELIMITER));
            String fieldDelimiter = StringEscapeUtils.unescapeJava(parameters.get("output_field_delimiter", CsvOutputFormat.DEFAULT_FIELD_DELIMITER));
            ji.writeAsCsv(filename, lineDelimiter, fieldDelimiter);
            env.execute("Jaccard Index");
            break;
        default:
            throw new ProgramParametrizationException(getUsage("invalid output type"));
    }
    JobExecutionResult result = env.getLastJobExecutionResult();
    NumberFormat nf = NumberFormat.getInstance();
    System.out.println();
    System.out.println("Execution runtime: " + nf.format(result.getNetRuntime()) + " ms");
}

Example 2 with Result

use of org.apache.flink.graph.library.similarity.JaccardIndex.Result in project flink by apache.

the class JaccardIndexTest method testRMatGraph.

@Test
public void testRMatGraph() throws Exception {
    long vertexCount = 1 << 8;
    long edgeCount = 8 * vertexCount;
    RandomGenerableFactory<JDKRandomGenerator> rnd = new JDKRandomGeneratorFactory();
    Graph<LongValue, NullValue, NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount).generate().run(new Simplify<LongValue, NullValue, NullValue>(false));
    DataSet<Result<LongValue>> ji = graph.run(new JaccardIndex<LongValue, NullValue, NullValue>().setGroupSize(4));
    Checksum checksum = new ChecksumHashCode<Result<LongValue>>().run(ji).execute();
    assertEquals(13954, checksum.getCount());
    assertEquals(0x00001b1a1f7a9d0bL, checksum.getChecksum());
}

Example 3 with Result

use of org.apache.flink.graph.library.similarity.JaccardIndex.Result in project flink by apache.

the class JaccardIndexTest method testWithRMatGraph.

@Test
public void testWithRMatGraph() throws Exception {
    DataSet<Result<LongValue>> ji = undirectedRMatGraph(8, 8).run(new JaccardIndex<LongValue, NullValue, NullValue>().setGroupSize(4));
    Checksum checksum = new ChecksumHashCode<Result<LongValue>>().run(ji).execute();
    assertEquals(13954, checksum.getCount());
    assertEquals(0x00001b1a1f7a9d0bL, checksum.getChecksum());
}