Examples with LocalPlume com.tdunning.plume.local.eager.LocalPlume used on opensource projects

Example 1 with LocalPlume

use of com.tdunning.plume.local.eager.LocalPlume in project Plume by tdunning.

the class WordCountTest method wordCount.

@Test
public void wordCount() throws IOException {
    Plume p = new LocalPlume();
    countWords(p.readResourceFile("simple-text.txt"));
}

Example 2 with LocalPlume

use of com.tdunning.plume.local.eager.LocalPlume in project Plume by tdunning.

the class LogParseTest method parseGroupSort.

@Test
public void parseGroupSort() throws IOException {
    Plume p = new LocalPlume();
    PCollection<String> logs = p.readResourceFile("log.txt");
    PTable<String, Event> events = logs.map(new DoFn<String, Pair<String, Event>>() {

        @Override
        public void process(String logLine, EmitFn<Pair<String, Event>> emitter) {
            Event e = new Event(logLine);
            emitter.emit(new Pair<String, Event>(e.getName(), e));
        }
    }, Plume.tableOf(strings(), strings()));
    PTable<String, Iterable<Event>> byName = events.groupByKey(new Ordering<Event>() {
    });
}

Example 3 with LocalPlume

use of com.tdunning.plume.local.eager.LocalPlume in project Plume by tdunning.

the class FlattenTest method flatten.

@Test
public void flatten() {
    Set<Integer> x1 = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        x1.add(i);
    }
    List<Integer> x2 = Lists.newArrayList();
    for (int i = 5; i < 15; i++) {
        x2.add(i);
    }
    Plume p = new LocalPlume();
    PCollection<Integer> x3 = p.flatten(p.fromJava(x1), p.fromJava(x2));
    PTable<Integer, Integer> x4 = x3.count();
    Map<Integer, Integer> r = Maps.newHashMap();
    for (Pair<Integer, Integer> pair : x4) {
        r.put(pair.getKey(), pair.getValue());
    }
    for (int i = 0; i < 5; i++) {
        Assert.assertEquals(new Integer(1), r.get(i));
    }
    for (int i = 5; i < 10; i++) {
        Assert.assertEquals(new Integer(2), r.get(i));
    }
    for (int i = 10; i < 15; i++) {
        Assert.assertEquals(new Integer(1), r.get(i));
    }
    PTable<Integer, String> x5 = x4.map(new DoFn<Pair<Integer, Integer>, Pair<Integer, String>>() {

        @Override
        public void process(Pair<Integer, Integer> v, EmitFn<Pair<Integer, String>> emitter) {
            emitter.emit(new Pair<Integer, String>(v.getKey(), v.getValue().toString()));
        }
    }, tableOf(integers(), strings()));
    for (Pair<Integer, String> pair : x5) {
        Assert.assertEquals(r.get(pair.getKey()).toString(), pair.getValue());
    }
    PCollection<String> x6 = x4.map(new DoFn<Pair<Integer, Integer>, String>() {

        @Override
        public void process(Pair<Integer, Integer> v, EmitFn<String> emitter) {
            emitter.emit(v.getValue().toString());
        }
    }, collectionOf(strings()));
    Map<String, Integer> r2 = Maps.newHashMap();
    for (Pair<String, Integer> v : x6.count()) {
        r2.put(v.getKey(), v.getValue());
    }
}

Example 4 with LocalPlume

use of com.tdunning.plume.local.eager.LocalPlume in project Plume by tdunning.

the class WordCountTest method wordCountAvro.

@Test
public void wordCountAvro() throws IOException {
    Plume p = new LocalPlume();
    String file = Resources.getResource("simple-text.avro").getPath();
    countWords(p.readAvroFile(file, strings()));
}