Examples with OutputCollector org.apache.storm.task.OutputCollector used on opensource projects

Example 31 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class TestHiveBolt method testTickTuple.

@Test
public void testTickTuple() {
    JsonRecordHiveMapper mapper = new JsonRecordHiveMapper().withColumnFields(new Fields(colNames1)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(2);
    bolt = new TestingHiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Tuple tuple1 = generateTestTuple(1, "SJC", "Sunnyvale", "CA");
    Tuple tuple2 = generateTestTuple(2, "SFO", "San Jose", "CA");
    bolt.execute(tuple1);
    // The tick should cause tuple1 to be ack'd
    Tuple mockTick = MockTupleHelpers.mockTickTuple();
    bolt.execute(mockTick);
    verify(collector).ack(tuple1);
    // The second tuple should NOT be ack'd because the batch should be cleared and this will be
    // the first transaction in the new batch
    bolt.execute(tuple2);
    verify(collector, never()).ack(tuple2);
    bolt.cleanup();
}

Example 32 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class TestHiveBolt method testData.

@Test
public void testData() throws Exception {
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(1);
    bolt = new TestingHiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Integer id = 1;
    String msg = "SJC";
    String city = "Sunnyvale";
    String state = "CA";
    Tuple tuple1 = generateTestTuple(id, msg, city, state);
    bolt.execute(tuple1);
    verify(collector).ack(tuple1);
    List<String> partVals = Lists.newArrayList(city, state);
    List<byte[]> recordsWritten = bolt.getRecordWritten(partVals);
    Assert.assertNotNull(recordsWritten);
    Assert.assertEquals(1, recordsWritten.size());
    byte[] mapped = generateDelimiteredRecord(Lists.newArrayList(id, msg), mapper.getFieldDelimiter());
    Assert.assertArrayEquals(mapped, recordsWritten.get(0));
    bolt.cleanup();
}

Example 33 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class TestHiveBolt method testMultiPartitionTuples.

@Test
public void testMultiPartitionTuples() throws Exception {
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(10);
    bolt = new TestingHiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Integer id = 1;
    String msg = "test";
    String city = "San Jose";
    String state = "CA";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        Tuple tuple = generateTestTuple(id, msg, city, state);
        tuples.add(tuple);
        bolt.execute(tuple);
    }
    for (Tuple t : tuples) {
        verify(collector).ack(t);
    }
    List<String> partVals = Lists.newArrayList(city, state);
    List<byte[]> recordsWritten = bolt.getRecordWritten(partVals);
    Assert.assertNotNull(recordsWritten);
    Assert.assertEquals(100, recordsWritten.size());
    byte[] mapped = generateDelimiteredRecord(Lists.newArrayList(id, msg), mapper.getFieldDelimiter());
    for (byte[] record : recordsWritten) {
        Assert.assertArrayEquals(mapped, record);
    }
    bolt.cleanup();
}

Example 34 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class KafkaBoltTest method testSimple.

@Test
public void testSimple() {
    MockProducer<String, String> producer = new MockProducer<>(Cluster.empty(), false, null, null, null);
    KafkaBolt<String, String> bolt = makeBolt(producer);
    OutputCollector collector = mock(OutputCollector.class);
    TopologyContext context = mock(TopologyContext.class);
    Map<String, Object> conf = new HashMap<>();
    bolt.prepare(conf, context, collector);
    String key = "KEY";
    String value = "VALUE";
    Tuple testTuple = createTestTuple(key, value);
    bolt.execute(testTuple);
    assertThat(producer.history().size(), is(1));
    ProducerRecord<String, String> arg = producer.history().get(0);
    LOG.info("GOT {} ->", arg);
    LOG.info("{}, {}, {}", arg.topic(), arg.key(), arg.value());
    assertThat(arg.topic(), is("MY_TOPIC"));
    assertThat(arg.key(), is(key));
    assertThat(arg.value(), is(value));
    // Complete the send
    producer.completeNext();
    verify(collector).ack(testTuple);
}

Example 35 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class TridentBoltExecutor method prepare.

@Override
public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
    messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    lastRotate = System.currentTimeMillis();
    batches = new RotatingMap<>(2);
    this.context = context;
    this.collector = collector;
    coordCollector = new CoordinatedOutputCollector(collector);
    coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(coordCollector));
    coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (coordConditions == null) {
        coordConditions = new HashMap<>();
        for (String batchGroup : coordSpecs.keySet()) {
            CoordSpec spec = coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(), coordStream(batchGroup), new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("coordConditions", coordConditions);
    }
    bolt.prepare(conf, context, coordOutputCollector);
}