Search in sources :

Example 6 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class PhoenixSinkIT method testSinkLifecycle.

@Test
public void testSinkLifecycle() {
    String tableName = generateUniqueName();
    String ddl = "CREATE TABLE " + tableName + "  (flume_time timestamp not null, col1 varchar , col2 varchar" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, tableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    Assert.assertEquals(LifecycleState.START, sink.getLifecycleState());
    sink.stop();
    Assert.assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}
Also used : Context(org.apache.flume.Context) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) NullPhoenixSink(org.apache.phoenix.flume.sink.NullPhoenixSink) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 7 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testBatchEvents.

@Test
public void testBatchEvents() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    initSinkContextWithDefaults(fullTableName);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 150;
    String col1 = "val1";
    String a1 = "[aaa,bbb,ccc]";
    String a2 = "[1,2,3,4]";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < eventList.size(); i++) {
        eventBody = "{\"col1\" : \"" + (col1 + i) + "\", \"col2\" : " + i * 10.5 + " , \"col3\" : " + a1 + " , \"col4\" : " + a2 + "}";
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(eventList.size(), rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 8 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testWithOutColumnsMapping.

@Test
public void testWithOutColumnsMapping() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    String columns = "col1,col2,col3,col4";
    String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
    initSinkContext(fullTableName, ddl, columns, null, rowkeyType, null);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "{\"col1\" : \"kalyan\", \"col2\" : 10.5, \"col3\" : [\"abc\",\"pqr\",\"xyz\"], \"col4\" : [1,2,3,4]}";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(1, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 9 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testInnerColumns.

@Test
public void testInnerColumns() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    String columns = "col1,col2,col3,col4";
    String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
    String columnsMapping = "{\"col1\":\"col1\",\"col2\":\"x.y\",\"col3\":\"a.b1.c\",\"col4\":\"col4\"}";
    initSinkContext(fullTableName, ddl, columns, columnsMapping, rowkeyType, null);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "{\"col1\" : \"kalyan\", \"x\" : {\"y\" : 10.5}, \"a\" : {\"b1\" : {\"c\" : [\"abc\",\"pqr\",\"xyz\"] }, \"b2\" : 111}, \"col4\" : [1,2,3,4]}";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(1, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 10 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method initChannel.

private Channel initChannel() {
    // Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");
    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}
Also used : Context(org.apache.flume.Context) MemoryChannel(org.apache.flume.channel.MemoryChannel) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel)

Aggregations

Channel (org.apache.flume.Channel)38 MemoryChannel (org.apache.flume.channel.MemoryChannel)33 Transaction (org.apache.flume.Transaction)29 Test (org.junit.Test)27 Event (org.apache.flume.Event)26 PhoenixSink (org.apache.phoenix.flume.sink.PhoenixSink)25 Context (org.apache.flume.Context)16 EventDeliveryException (org.apache.flume.EventDeliveryException)8 Connection (java.sql.Connection)5 ResultSet (java.sql.ResultSet)5 Properties (java.util.Properties)5 SQLException (java.sql.SQLException)4 ArrayList (java.util.ArrayList)4 NullPhoenixSink (org.apache.phoenix.flume.sink.NullPhoenixSink)4 ChannelSelector (org.apache.flume.ChannelSelector)3 ChannelProcessor (org.apache.flume.channel.ChannelProcessor)3 ReplicatingChannelSelector (org.apache.flume.channel.ReplicatingChannelSelector)3 UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 ChannelException (org.apache.flume.ChannelException)2