Examples with MRPipeline org.apache.crunch.impl.mr.MRPipeline used on opensource projects

Example 21 with MRPipeline

use of org.apache.crunch.impl.mr.MRPipeline in project crunch by cloudera.

the class AggregateTest method testCollectUrls.

@Test
public void testCollectUrls() throws Exception {
    Pipeline p = new MRPipeline(AggregateTest.class);
    String urlsInputPath = FileHelper.createTempCopyOf("urls.txt");
    PTable<String, Collection<String>> urls = Aggregate.collectValues(p.readTextFile(urlsInputPath).parallelDo(new SplitFn(), tableOf(strings(), strings())));
    for (Pair<String, Collection<String>> e : urls.materialize()) {
        String key = e.first();
        int expectedSize = 0;
        if ("www.A.com".equals(key)) {
            expectedSize = 4;
        } else if ("www.B.com".equals(key) || "www.F.com".equals(key)) {
            expectedSize = 2;
        } else if ("www.C.com".equals(key) || "www.D.com".equals(key) || "www.E.com".equals(key)) {
            expectedSize = 1;
        }
        assertEquals("Checking key = " + key, expectedSize, e.second().size());
        p.done();
    }
}

Example 22 with MRPipeline

use of org.apache.crunch.impl.mr.MRPipeline in project crunch by cloudera.

the class AggregateTest method testCollectValues_Writables.

@Test
public void testCollectValues_Writables() throws IOException {
    Pipeline pipeline = new MRPipeline(AggregateTest.class);
    Map<Integer, Collection<Text>> collectionMap = pipeline.readTextFile(FileHelper.createTempCopyOf("set2.txt")).parallelDo(new MapStringToTextPair(), Writables.tableOf(Writables.ints(), Writables.writables(Text.class))).collectValues().materializeToMap();
    assertEquals(1, collectionMap.size());
    assertEquals(Lists.newArrayList(new Text("c"), new Text("d"), new Text("a")), collectionMap.get(1));
}

Example 23 with MRPipeline

use of org.apache.crunch.impl.mr.MRPipeline in project crunch by cloudera.

the class AvroTypeSortTest method testSortAvroTypesBySelectedFields.

@Test
public void testSortAvroTypesBySelectedFields() throws Exception {
    MRPipeline pipeline = new MRPipeline(AvroTypeSortTest.class);
    Person ccc10 = createPerson("CCC", 10);
    Person bbb20 = createPerson("BBB", 20);
    Person aaa30 = createPerson("AAA", 30);
    writeAvroFile(Lists.newArrayList(ccc10, bbb20, aaa30), avroFile);
    PCollection<Person> unsorted = pipeline.read(At.avroFile(avroFile.getAbsolutePath(), records(Person.class)));
    // Sort by Name
    MapFn<Person, String> nameExtractor = new MapFn<Person, String>() {

        @Override
        public String map(Person input) {
            return input.getName().toString();
        }
    };
    PCollection<Person> sortedByName = unsorted.by(nameExtractor, strings()).groupByKey().ungroup().values();
    List<Person> sortedByNameList = Lists.newArrayList(sortedByName.materialize());
    assertEquals(3, sortedByNameList.size());
    assertEquals(aaa30, sortedByNameList.get(0));
    assertEquals(bbb20, sortedByNameList.get(1));
    assertEquals(ccc10, sortedByNameList.get(2));
    // Sort by Age
    MapFn<Person, Integer> ageExtractor = new MapFn<Person, Integer>() {

        @Override
        public Integer map(Person input) {
            return input.getAge();
        }
    };
    PCollection<Person> sortedByAge = unsorted.by(ageExtractor, ints()).groupByKey().ungroup().values();
    List<Person> sortedByAgeList = Lists.newArrayList(sortedByAge.materialize());
    assertEquals(3, sortedByAgeList.size());
    assertEquals(ccc10, sortedByAgeList.get(0));
    assertEquals(bbb20, sortedByAgeList.get(1));
    assertEquals(aaa30, sortedByAgeList.get(2));
    pipeline.done();
}

Example 24 with MRPipeline

use of org.apache.crunch.impl.mr.MRPipeline in project crunch by cloudera.

the class SetTest method setUp.

@Before
public void setUp() throws IOException {
    String set1InputPath = FileHelper.createTempCopyOf("set1.txt");
    String set2InputPath = FileHelper.createTempCopyOf("set2.txt");
    pipeline = new MRPipeline(SetTest.class);
    set1 = pipeline.read(At.textFile(set1InputPath, typeFamily.strings()));
    set2 = pipeline.read(At.textFile(set2InputPath, typeFamily.strings()));
}

Example 25 with MRPipeline

use of org.apache.crunch.impl.mr.MRPipeline in project crunch by cloudera.

the class AvroFileSourceTargetTest method testReflect.

@Test
public void testReflect() throws IOException {
    Schema pojoPersonSchema = ReflectData.get().getSchema(PojoPerson.class);
    GenericRecord savedRecord = new GenericData.Record(pojoPersonSchema);
    savedRecord.put("name", "John Doe");
    populateGenericFile(Lists.newArrayList(savedRecord), pojoPersonSchema);
    Pipeline pipeline = new MRPipeline(AvroFileSourceTargetTest.class);
    PCollection<PojoPerson> personCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(), Avros.reflects(PojoPerson.class)));
    List<PojoPerson> recordList = Lists.newArrayList(personCollection.materialize());
    assertEquals(1, recordList.size());
    PojoPerson person = recordList.get(0);
    assertEquals("John Doe", person.getName());
}