Examples with SparkConf org.apache.spark.SparkConf used on opensource projects

Example 56 with SparkConf

use of org.apache.spark.SparkConf in project learning-spark by databricks.

the class BasicAvgWithKryo method main.

public static void main(String[] args) throws Exception {
    String master;
    if (args.length > 0) {
        master = args[0];
    } else {
        master = "local";
    }
    SparkConf conf = new SparkConf().setMaster(master).setAppName("basicavgwithkyro");
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    conf.set("spark.kryo.registrator", AvgRegistrator.class.getName());
    JavaSparkContext sc = new JavaSparkContext(conf);
    JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4));
    Function2<AvgCount, Integer, AvgCount> addAndCount = new Function2<AvgCount, Integer, AvgCount>() {

        @Override
        public AvgCount call(AvgCount a, Integer x) {
            a.total_ += x;
            a.num_ += 1;
            return a;
        }
    };
    Function2<AvgCount, AvgCount, AvgCount> combine = new Function2<AvgCount, AvgCount, AvgCount>() {

        @Override
        public AvgCount call(AvgCount a, AvgCount b) {
            a.total_ += b.total_;
            a.num_ += b.num_;
            return a;
        }
    };
    AvgCount initial = new AvgCount(0, 0);
    AvgCount result = rdd.aggregate(initial, addAndCount, combine);
    System.out.println(result.avg());
}

Example 57 with SparkConf

use of org.apache.spark.SparkConf in project deeplearning4j by deeplearning4j.

the class JavaQueueStream method main.

public static void main(String[] args) throws Exception {
    SparkConf sparkConf = new SparkConf().setMaster("local[*]");
    // Create the context
    JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));
    // Create the queue through which RDDs can be pushed to
    // a QueueInputDStream
    Queue<JavaRDD<Integer>> rddQueue = new LinkedList<>();
    // Create and push some RDDs into the queue
    List<Integer> list = Lists.newArrayList();
    for (int i = 0; i < 1000; i++) {
        list.add(i);
    }
    for (int i = 0; i < 30; i++) {
        rddQueue.add(ssc.sparkContext().parallelize(list));
    }
    // Create the QueueInputDStream and use it do some processing
    JavaDStream<Integer> inputStream = ssc.queueStream(rddQueue);
    JavaPairDStream<Integer, Integer> mappedStream = inputStream.mapToPair(new PairFunction<Integer, Integer, Integer>() {

        @Override
        public Tuple2<Integer, Integer> call(Integer i) {
            return new Tuple2<>(i % 10, 1);
        }
    });
    JavaPairDStream<Integer, Integer> reducedStream = mappedStream.reduceByKey(new Function2<Integer, Integer, Integer>() {

        @Override
        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });
    reducedStream.print();
    ssc.start();
    ssc.awaitTermination();
}

Example 58 with SparkConf

use of org.apache.spark.SparkConf in project deeplearning4j by deeplearning4j.

the class SparkWord2VecTest method setUp.

@Before
public void setUp() throws Exception {
    if (sentences == null) {
        sentences = new ArrayList<>();
        sentences.add("one two thee four");
        sentences.add("some once again");
        sentences.add("one another sentence");
    }
    SparkConf sparkConf = new SparkConf().setMaster("local[8]").setAppName("SeqVecTests");
    sc = new JavaSparkContext(sparkConf);
}

Example 59 with SparkConf

use of org.apache.spark.SparkConf in project deeplearning4j by deeplearning4j.

the class TestKryoWarning method testKryoMessageCGCorrectConfigNoKryo.

@Test
@Ignore
public void testKryoMessageCGCorrectConfigNoKryo() {
    //Should NOT print warning message
    SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest");
    doTestCG(sparkConf);
}

Example 60 with SparkConf

use of org.apache.spark.SparkConf in project deeplearning4j by deeplearning4j.

the class TestCompareParameterAveragingSparkVsSingleMachine method getContext.

private static JavaSparkContext getContext(int nWorkers) {
    SparkConf sparkConf = new SparkConf();
    sparkConf.setMaster("local[" + nWorkers + "]");
    sparkConf.setAppName("Test");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    return sc;
}