use of com.tdunning.plume.Plume in project Plume by tdunning.
the class LogParseTest method parseGroupSort.
@Test
public void parseGroupSort() throws IOException {
Plume p = new LocalPlume();
PCollection<String> logs = p.readResourceFile("log.txt");
PTable<String, Event> events = logs.map(new DoFn<String, Pair<String, Event>>() {
@Override
public void process(String logLine, EmitFn<Pair<String, Event>> emitter) {
Event e = new Event(logLine);
emitter.emit(new Pair<String, Event>(e.getName(), e));
}
}, Plume.tableOf(strings(), strings()));
PTable<String, Iterable<Event>> byName = events.groupByKey(new Ordering<Event>() {
});
}
use of com.tdunning.plume.Plume in project Plume by tdunning.
the class FlattenTest method flatten.
@Test
public void flatten() {
Set<Integer> x1 = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
x1.add(i);
}
List<Integer> x2 = Lists.newArrayList();
for (int i = 5; i < 15; i++) {
x2.add(i);
}
Plume p = new LocalPlume();
PCollection<Integer> x3 = p.flatten(p.fromJava(x1), p.fromJava(x2));
PTable<Integer, Integer> x4 = x3.count();
Map<Integer, Integer> r = Maps.newHashMap();
for (Pair<Integer, Integer> pair : x4) {
r.put(pair.getKey(), pair.getValue());
}
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new Integer(1), r.get(i));
}
for (int i = 5; i < 10; i++) {
Assert.assertEquals(new Integer(2), r.get(i));
}
for (int i = 10; i < 15; i++) {
Assert.assertEquals(new Integer(1), r.get(i));
}
PTable<Integer, String> x5 = x4.map(new DoFn<Pair<Integer, Integer>, Pair<Integer, String>>() {
@Override
public void process(Pair<Integer, Integer> v, EmitFn<Pair<Integer, String>> emitter) {
emitter.emit(new Pair<Integer, String>(v.getKey(), v.getValue().toString()));
}
}, tableOf(integers(), strings()));
for (Pair<Integer, String> pair : x5) {
Assert.assertEquals(r.get(pair.getKey()).toString(), pair.getValue());
}
PCollection<String> x6 = x4.map(new DoFn<Pair<Integer, Integer>, String>() {
@Override
public void process(Pair<Integer, Integer> v, EmitFn<String> emitter) {
emitter.emit(v.getValue().toString());
}
}, collectionOf(strings()));
Map<String, Integer> r2 = Maps.newHashMap();
for (Pair<String, Integer> v : x6.count()) {
r2.put(v.getKey(), v.getValue());
}
}
Aggregations