Examples with HiveOptions org.apache.storm.hive.common.HiveOptions used on opensource projects

Example 11 with HiveOptions

use of org.apache.storm.hive.common.HiveOptions in project storm by apache.

the class TestHiveBolt method testData.

@Test
public void testData() throws Exception {
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(1);
    bolt = new HiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Tuple tuple1 = generateTestTuple(1, "SJC", "Sunnyvale", "CA");
    //Tuple tuple2 = generateTestTuple(2,"SFO","San Jose","CA");
    bolt.execute(tuple1);
    verify(collector).ack(tuple1);
    //bolt.execute(tuple2);
    //verify(collector).ack(tuple2);
    checkDataWritten(tblName, dbName, "1,SJC,Sunnyvale,CA");
    bolt.cleanup();
}

Example 12 with HiveOptions

use of org.apache.storm.hive.common.HiveOptions in project storm by apache.

the class TestHiveBolt method testTickTuple.

@Test
public void testTickTuple() {
    JsonRecordHiveMapper mapper = new JsonRecordHiveMapper().withColumnFields(new Fields(colNames1)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(2);
    bolt = new HiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Tuple tuple1 = generateTestTuple(1, "SJC", "Sunnyvale", "CA");
    Tuple tuple2 = generateTestTuple(2, "SFO", "San Jose", "CA");
    bolt.execute(tuple1);
    //The tick should cause tuple1 to be ack'd
    Tuple mockTick = MockTupleHelpers.mockTickTuple();
    bolt.execute(mockTick);
    verify(collector).ack(tuple1);
    //The second tuple should NOT be ack'd because the batch should be cleared and this will be
    //the first transaction in the new batch
    bolt.execute(tuple2);
    verify(collector, never()).ack(tuple2);
    bolt.cleanup();
}

Example 13 with HiveOptions

use of org.apache.storm.hive.common.HiveOptions in project storm by apache.

the class TestHiveBolt method testNoAcksIfFlushFails.

@Test
public void testNoAcksIfFlushFails() throws Exception {
    JsonRecordHiveMapper mapper = new JsonRecordHiveMapper().withColumnFields(new Fields(colNames1)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(2);
    HiveBolt spyBolt = Mockito.spy(new HiveBolt(hiveOptions));
    //This forces a failure of all the flush attempts
    doThrow(new InterruptedException()).when(spyBolt).flushAllWriters(true);
    spyBolt.prepare(config, null, new OutputCollector(collector));
    Tuple tuple1 = generateTestTuple(1, "SJC", "Sunnyvale", "CA");
    Tuple tuple2 = generateTestTuple(2, "SFO", "San Jose", "CA");
    spyBolt.execute(tuple1);
    spyBolt.execute(tuple2);
    verify(collector, never()).ack(tuple1);
    verify(collector, never()).ack(tuple2);
    spyBolt.cleanup();
}

Example 14 with HiveOptions

use of org.apache.storm.hive.common.HiveOptions in project storm by apache.

the class TestHiveBolt method testMultiPartitionTuples.

@Test
public void testMultiPartitionTuples() throws Exception {
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(10);
    bolt = new HiveBolt(hiveOptions);
    bolt.prepare(config, null, new OutputCollector(collector));
    Integer id = 1;
    String msg = "test";
    String city = "San Jose";
    String state = "CA";
    checkRecordCountInTable(tblName, dbName, 0);
    Set<Tuple> tupleSet = new HashSet<Tuple>();
    for (int i = 0; i < 100; i++) {
        Tuple tuple = generateTestTuple(id, msg, city, state);
        tupleSet.add(tuple);
        bolt.execute(tuple);
    }
    checkRecordCountInTable(tblName, dbName, 100);
    for (Tuple t : tupleSet) verify(collector).ack(t);
    bolt.cleanup();
}