Examples with DelimitedRecordHiveMapper org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper used on opensource projects

Example 1 with DelimitedRecordHiveMapper

use of org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper in project storm by apache.

the class TestHiveWriter method testInstantiate.

@Test
public void testInstantiate() throws Exception {
    DelimitedRecordHiveMapper mapper = new MockedDelemiteredRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, Arrays.asList(partitionVals));
    TestingHiveWriter writer = new TestingHiveWriter(endPoint, 10, true, timeout, callTimeoutPool, mapper, ugi, false);
    writer.close();
}

Example 2 with DelimitedRecordHiveMapper

use of org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper in project storm by apache.

the class TestHiveWriter method testWriteBasic.

@Test
public void testWriteBasic() throws Exception {
    DelimitedRecordHiveMapper mapper = new MockedDelemiteredRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, Arrays.asList(partitionVals));
    TestingHiveWriter writer = new TestingHiveWriter(endPoint, 10, true, timeout, callTimeoutPool, mapper, ugi, false);
    writeTuples(writer, mapper, 3);
    writer.flush(false);
    writer.close();
    Mockito.verify(writer.getMockedTxBatch(), Mockito.times(3)).write(Mockito.any(byte[].class));
}

Example 3 with DelimitedRecordHiveMapper

use of org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper in project storm by apache.

the class HiveTopology method main.

public static void main(String[] args) throws Exception {
    String metaStoreUri = args[0];
    String dbName = args[1];
    String tblName = args[2];
    String[] colNames = { "id", "name", "phone", "street", "city", "state" };
    Config config = new Config();
    config.setNumWorkers(1);
    UserDataSpout spout = new UserDataSpout();
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames));
    HiveOptions hiveOptions;
    if (args.length == 6) {
        hiveOptions = new HiveOptions(metaStoreUri, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(100).withIdleTimeout(10).withKerberosKeytab(args[4]).withKerberosPrincipal(args[5]);
    } else {
        hiveOptions = new HiveOptions(metaStoreUri, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(100).withIdleTimeout(10).withMaxOpenConnections(1);
    }
    HiveBolt hiveBolt = new HiveBolt(hiveOptions);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(USER_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, hiveBolt, 1).shuffleGrouping(USER_SPOUT_ID);
    String topoName = TOPOLOGY_NAME;
    if (args.length >= 4) {
        topoName = args[3];
    }
    StormSubmitter.submitTopology(topoName, config, builder.createTopology());
}

Example 4 with DelimitedRecordHiveMapper

use of org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper in project storm by apache.

the class HiveTopologyPartitioned method main.

public static void main(String[] args) throws Exception {
    String metaStoreUri = args[0];
    String dbName = args[1];
    String tblName = args[2];
    String[] partNames = { "city", "state" };
    String[] colNames = { "id", "name", "phone", "street" };
    Config config = new Config();
    config.setNumWorkers(1);
    UserDataSpout spout = new UserDataSpout();
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions;
    if (args.length == 6) {
        hiveOptions = new HiveOptions(metaStoreUri, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(1000).withIdleTimeout(10).withKerberosKeytab(args[4]).withKerberosPrincipal(args[5]);
    } else {
        hiveOptions = new HiveOptions(metaStoreUri, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(1000).withIdleTimeout(10);
    }
    HiveBolt hiveBolt = new HiveBolt(hiveOptions);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(USER_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, hiveBolt, 1).shuffleGrouping(USER_SPOUT_ID);
    String topoName = TOPOLOGY_NAME;
    if (args.length > 3) {
        topoName = args[3];
    }
    StormSubmitter.submitTopology(topoName, config, builder.createTopology());
}

Example 5 with DelimitedRecordHiveMapper

use of org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper in project storm by apache.

the class TestHiveBolt method testWithByteArrayIdandMessage.

@Test
public void testWithByteArrayIdandMessage() throws Exception {
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(2);
    bolt = new TestingHiveBolt(hiveOptions);
    bolt.prepare(config, null, collector);
    Integer id = 100;
    String msg = "test-123";
    String city = "sunnyvale";
    String state = "ca";
    Set<Tuple> tupleSet = new HashSet<Tuple>();
    for (int i = 0; i < 4; i++) {
        Tuple tuple = generateTestTuple(id, msg, city, state);
        bolt.execute(tuple);
        tupleSet.add(tuple);
    }
    List<String> partVals = Lists.newArrayList(city, state);
    for (Tuple t : tupleSet) {
        verify(collector).ack(t);
    }
    Assert.assertEquals(4, bolt.getRecordWritten(partVals).size());
    bolt.cleanup();
}