Examples with IntWritable org.apache.hadoop.io.IntWritable used on opensource projects

Example 41 with IntWritable

use of org.apache.hadoop.io.IntWritable in project mongo-hadoop by mongodb.

the class EnronMailReducer method reduce.

@Override
public void reduce(final MailPair pKey, final Iterable<IntWritable> pValues, final Context pContext) throws IOException, InterruptedException {
    int sum = 0;
    for (final IntWritable value : pValues) {
        sum += value.get();
    }
    BSONObject outDoc = BasicDBObjectBuilder.start().add("f", pKey.getFrom()).add("t", pKey.getTo()).get();
    reduceResult.setDoc(outDoc);
    intw.set(sum);
    pContext.write(reduceResult, intw);
}

Example 42 with IntWritable

use of org.apache.hadoop.io.IntWritable in project mongo-hadoop by mongodb.

the class LogCombiner method reduce.

@Override
public void reduce(final Text key, final Iterator<IntWritable> values, final OutputCollector<Text, IntWritable> output, final Reporter reporter) throws IOException {
    int count = 0;
    while (values.hasNext()) {
        count += values.next().get();
    }
    output.collect(key, new IntWritable(count));
}

Example 43 with IntWritable

use of org.apache.hadoop.io.IntWritable in project mongo-hadoop by mongodb.

the class LogReducer method reduce.

@Override
public void reduce(final Text pKey, final Iterable<IntWritable> pValues, final Context pContext) throws IOException, InterruptedException {
    int count = 0;
    for (IntWritable val : pValues) {
        count += val.get();
    }
    BasicBSONObject query = new BasicBSONObject("devices", new ObjectId(pKey.toString()));
    BasicBSONObject update = new BasicBSONObject("$inc", new BasicBSONObject("logs_count", count));
    if (LOG.isDebugEnabled()) {
        LOG.debug("query: " + query);
        LOG.debug("update: " + update);
    }
    reduceResult.setQuery(query);
    reduceResult.setModifiers(update);
    pContext.write(null, reduceResult);
}

Example 44 with IntWritable

use of org.apache.hadoop.io.IntWritable in project goldenorb by jzachr.

the class OutboundVertexThread method testOutBoundVertexQueue.

@Test
public void testOutBoundVertexQueue() throws Exception {
    // OutboundVertexQueue settings
    int numberOfPartitions = 100;
    // max number of Vertices to be sent by a Thread
    int numOfVerticesToSendPerThread = 10500;
    // max number of Vertices to trigger a send operation by the queue
    int numOfVerticesPerBlock = 1000;
    int partitionId = 1;
    Class<? extends Vertex<?, ?, ?>> vertexClass = TestVertex.class;
    Map<Integer, OrbPartitionCommunicationProtocol> orbClients = new HashMap<Integer, OrbPartitionCommunicationProtocol>();
    for (int i = 0; i < numberOfPartitions; i++) {
        orbClients.put(new Integer(i), infoCollector);
    }
    OutboundVertexQueue ovq = new OutboundVertexQueue(numberOfPartitions, numOfVerticesPerBlock, orbClients, vertexClass, partitionId);
    // initialize the Threads and pass them their test Vertices
    CountDownLatch startLatch = new CountDownLatch(1);
    CountDownLatch everyoneDoneLatch = new CountDownLatch(numberOfPartitions);
    for (int i = 0; i < numberOfPartitions; i++) {
        Vertices vrts = new Vertices(vertexClass);
        for (int p = 0; p < numOfVerticesToSendPerThread; p++) {
            String vertexID = "vertex " + p;
            IntWritable vertexValue = new IntWritable(p);
            List<Edge<IntWritable>> edgesList = new ArrayList<Edge<IntWritable>>();
            TestVertex vrt = new TestVertex(vertexID, vertexValue, edgesList);
            vrts.add(vrt);
        }
        OutboundVertexThread obmThread = new OutboundVertexThread(vrts, ovq, startLatch, everyoneDoneLatch);
        // initialize a Thread
        obmThread.start();
    }
    // start all Threads simultaneously
    startLatch.countDown();
    // wait until all Threads are done
    everyoneDoneLatch.await();
    ovq.sendRemainingVertices();
    System.out.println(infoCollector.vList.size());
    assertThat(ovq, notNullValue());
    assertTrue(infoCollector.vList.size() == (numberOfPartitions * numOfVerticesToSendPerThread));
}

Example 45 with IntWritable

use of org.apache.hadoop.io.IntWritable in project goldenorb by jzachr.

the class SingleSourceShortestPathReader method buildVertex.

@Override
public SingleSourceShortestPathVertex buildVertex(LongWritable key, Text value) {
    String[] values = value.toString().split("\t");
    ArrayList<Edge<IntWritable>> edgeCollection = new ArrayList<Edge<IntWritable>>();
    for (int i = 1; i < values.length; i++) {
        String[] split = values[i].split(":");
        Edge<IntWritable> edge = new Edge<IntWritable>(Integer.toString(Integer.parseInt(split[0].trim())), new IntWritable(Integer.parseInt(split[1])));
        edgeCollection.add(edge);
    }
    PathWritable emptyPathWritable = new PathWritable();
    SingleSourceShortestPathVertex vertex = new SingleSourceShortestPathVertex(Integer.toString(Integer.parseInt(values[0].trim())), emptyPathWritable, edgeCollection);
    return vertex;
}