use of org.apache.flume.Transaction in project phoenix by apache.
the class JsonEventSerializerIT method testInnerColumns.
@Test
public void testInnerColumns() throws EventDeliveryException, SQLException {
final String fullTableName = "FLUME_JSON_TEST";
String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + " (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + " CONSTRAINT pk PRIMARY KEY (flume_time))\n";
String columns = "col1,col2,col3,col4";
String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
String columnsMapping = "{\"col1\":\"col1\",\"col2\":\"x.y\",\"col3\":\"a.b1.c\",\"col4\":\"col4\"}";
initSinkContext(fullTableName, ddl, columns, columnsMapping, rowkeyType, null);
sink = new PhoenixSink();
Configurables.configure(sink, sinkContext);
assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
final Channel channel = this.initChannel();
sink.setChannel(channel);
sink.start();
final String eventBody = "{\"col1\" : \"kalyan\", \"x\" : {\"y\" : 10.5}, \"a\" : {\"b1\" : {\"c\" : [\"abc\",\"pqr\",\"xyz\"] }, \"b2\" : 111}, \"col4\" : [1,2,3,4]}";
final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
// put event in channel
Transaction transaction = channel.getTransaction();
transaction.begin();
channel.put(event);
transaction.commit();
transaction.close();
sink.process();
int rowsInDb = countRows(fullTableName);
assertEquals(1, rowsInDb);
sink.stop();
assertEquals(LifecycleState.STOP, sink.getLifecycleState());
dropTable(fullTableName);
}
use of org.apache.flume.Transaction in project phoenix by apache.
the class JsonEventSerializerIT method testInnerColumnsWithArrayMapping.
@Test
public void testInnerColumnsWithArrayMapping() throws EventDeliveryException, SQLException {
final String fullTableName = "FLUME_JSON_TEST";
String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + " (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + " CONSTRAINT pk PRIMARY KEY (flume_time))\n";
String columns = "col1,col2,col3,col4";
String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
String columnsMapping = "{\"col1\":\"col1\",\"col2\":\"x.y\",\"col3\":\"a.b[*].c\",\"col4\":\"col4\"}";
initSinkContext(fullTableName, ddl, columns, columnsMapping, rowkeyType, null);
sink = new PhoenixSink();
Configurables.configure(sink, sinkContext);
assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
final Channel channel = this.initChannel();
sink.setChannel(channel);
sink.start();
final String eventBody = "{\"col1\" : \"kalyan\", \"x\" : {\"y\" : 10.5}, \"a\" : {\"b\" : [{\"c\" : \"abc\"}, {\"c\" : \"pqr\"}, {\"c\" : \"xyz\"}] , \"b2\" : 111}, \"col4\" : [1,2,3,4]}";
final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
// put event in channel
Transaction transaction = channel.getTransaction();
transaction.begin();
channel.put(event);
transaction.commit();
transaction.close();
sink.process();
int rowsInDb = countRows(fullTableName);
assertEquals(1, rowsInDb);
sink.stop();
assertEquals(LifecycleState.STOP, sink.getLifecycleState());
dropTable(fullTableName);
}
use of org.apache.flume.Transaction in project phoenix by apache.
the class CsvEventSerializerIT method testEventsWithHeaders.
@Test
public void testEventsWithHeaders() throws Exception {
sinkContext = new Context();
final String fullTableName = "FLUME_CSV_TEST";
final String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + " (rowkey VARCHAR not null, col1 varchar , col2 double, col3 varchar[], col4 integer[], host varchar , source varchar \n" + " CONSTRAINT pk PRIMARY KEY (rowkey))\n";
String columns = "col1,col2,col3,col4";
String rowkeyType = DefaultKeyGenerator.UUID.name();
String headers = "host,source";
initSinkContext(fullTableName, ddl, columns, null, null, null, null, rowkeyType, headers);
sink = new PhoenixSink();
Configurables.configure(sink, sinkContext);
assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
final Channel channel = this.initChannel();
sink.setChannel(channel);
sink.start();
int numEvents = 10;
String col1 = "val1";
String a1 = "\"aaa,bbb,ccc\"";
String a2 = "\"1,2,3,4\"";
String hostHeader = "host1";
String sourceHeader = "source1";
String eventBody = null;
List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
for (int i = 0; i < numEvents; i++) {
eventBody = (col1 + i) + "," + i * 10.5 + "," + a1 + "," + a2;
Map<String, String> headerMap = Maps.newHashMapWithExpectedSize(2);
headerMap.put("host", hostHeader);
headerMap.put("source", sourceHeader);
Event event = EventBuilder.withBody(Bytes.toBytes(eventBody), headerMap);
eventList.add(event);
}
// put event in channel
Transaction transaction = channel.getTransaction();
transaction.begin();
for (Event event : eventList) {
channel.put(event);
}
transaction.commit();
transaction.close();
sink.process();
final String query = " SELECT * FROM \n " + fullTableName;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
final ResultSet rs;
final Connection conn = DriverManager.getConnection(getUrl(), props);
try {
rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
assertEquals("host1", rs.getString("host"));
assertEquals("source1", rs.getString("source"));
assertTrue(rs.next());
assertEquals("host1", rs.getString("host"));
assertEquals("source1", rs.getString("source"));
} finally {
if (conn != null) {
conn.close();
}
}
sink.stop();
assertEquals(LifecycleState.STOP, sink.getLifecycleState());
dropTable(fullTableName);
}
use of org.apache.flume.Transaction in project phoenix by apache.
the class CsvEventSerializerIT method testMissingColumnsInEvent.
@Test
public void testMissingColumnsInEvent() throws EventDeliveryException, SQLException {
final String fullTableName = "FLUME_CSV_TEST";
initSinkContextWithDefaults(fullTableName);
sink = new PhoenixSink();
Configurables.configure(sink, sinkContext);
assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
final Channel channel = this.initChannel();
sink.setChannel(channel);
sink.start();
final String eventBody = "kalyan,\"abc,pqr,xyz\",\"1,2,3,4\"";
final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
// put event in channel
Transaction transaction = channel.getTransaction();
transaction.begin();
channel.put(event);
transaction.commit();
transaction.close();
sink.process();
int rowsInDb = countRows(fullTableName);
assertEquals(0, rowsInDb);
sink.stop();
assertEquals(LifecycleState.STOP, sink.getLifecycleState());
dropTable(fullTableName);
}
use of org.apache.flume.Transaction in project logging-log4j2 by apache.
the class FlumeAppenderTest method testIncompleteBatch2.
@Test
public void testIncompleteBatch2() throws IOException {
final Agent[] agents = new Agent[] { Agent.createAgent("localhost", testPort) };
final FlumeAppender avroAppender = FlumeAppender.createAppender(agents, null, null, "false", "Avro", null, "1000", "1000", "1", "500", "avro", "false", null, null, null, null, null, "true", "10", null, null, null, null);
avroAppender.start();
avroLogger.addAppender(avroAppender);
avroLogger.setLevel(Level.ALL);
Assert.assertNotNull(avroLogger);
avroLogger.info("Test message 0");
final Transaction transaction = channel.getTransaction();
transaction.begin();
avroLogger.info("Test message 1");
avroLogger.info("Test message 2");
avroAppender.stop();
for (int i = 0; i < 3; ++i) {
final Event event = channel.take();
Assert.assertNotNull("No event for item " + i, event);
Assert.assertTrue("Channel contained event, but not expected message. Received : " + getBody(event), getBody(event).endsWith("Test message " + i));
}
transaction.commit();
transaction.close();
eventSource.stop();
}
Aggregations