Search in sources :

Example 26 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testMissingColumnsInEvent.

@Test
public void testMissingColumnsInEvent() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    initSinkContextWithDefaults(fullTableName);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "{\"col1\" : \"kalyan\", \"col3\" : [\"abc\",\"pqr\",\"xyz\"], \"col4\" : [1,2,3,4]}";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(0, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 27 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testEventsWithHeaders.

@Test
public void testEventsWithHeaders() throws Exception {
    sinkContext = new Context();
    final String fullTableName = "FLUME_JSON_TEST";
    final String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (rowkey VARCHAR not null, col1 varchar , col2 double, col3 varchar[], col4 integer[], host varchar , source varchar \n" + "  CONSTRAINT pk PRIMARY KEY (rowkey))\n";
    String columns = "col1,col2,col3,col4";
    String columnsMapping = "{\"col1\":\"col1\",\"col2\":\"col2\",\"col3\":\"col3\",\"col4\":\"col4\"}";
    String rowkeyType = DefaultKeyGenerator.UUID.name();
    String headers = "host,source";
    initSinkContext(fullTableName, ddl, columns, columnsMapping, rowkeyType, headers);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 10;
    String col1 = "val1";
    String a1 = "[aaa,bbb,ccc]";
    String a2 = "[1,2,3,4]";
    String hostHeader = "host1";
    String sourceHeader = "source1";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < numEvents; i++) {
        eventBody = "{\"col1\" : \"" + (col1 + i) + "\", \"col2\" : " + i * 10.5 + " , \"col3\" : " + a1 + " , \"col4\" : " + a2 + "}";
        Map<String, String> headerMap = Maps.newHashMapWithExpectedSize(2);
        headerMap.put("host", hostHeader);
        headerMap.put("source", sourceHeader);
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody), headerMap);
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try {
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
    } finally {
        if (conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Context(org.apache.flume.Context) Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) ResultSet(java.sql.ResultSet) Connection(java.sql.Connection) Event(org.apache.flume.Event) Properties(java.util.Properties) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 28 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testMismatchKeyGenerator.

@Test
public void testMismatchKeyGenerator() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    initSinkContextWithDefaults(fullTableName);
    setConfig(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "{\"col1\" : \"kalyan\", \"col2\" : 10.5, \"col3\" : [\"abc\",\"pqr\",\"xyz\"], \"col4\" : [1,2,3,4]}";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    try {
        sink.process();
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getCause().getMessage().contains("java.lang.IllegalArgumentException: Invalid format:"));
    }
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) SQLException(java.sql.SQLException) EventDeliveryException(org.apache.flume.EventDeliveryException) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 29 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class PhoenixSinkIT method testCreateTable.

@Test
public void testCreateTable() throws Exception {
    String tableName = generateUniqueName();
    String ddl = "CREATE TABLE " + tableName + " " + "  (flume_time timestamp not null, col1 varchar , col2 varchar" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    final String fullTableName = tableName;
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
    try {
        boolean exists = admin.tableExists(fullTableName);
        Assert.assertTrue(exists);
    } finally {
        admin.close();
    }
}
Also used : Context(org.apache.flume.Context) HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) NullPhoenixSink(org.apache.phoenix.flume.sink.NullPhoenixSink) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 30 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class PhoenixSinkIT method initChannel.

private Channel initChannel() {
    //Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");
    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}
Also used : Context(org.apache.flume.Context) MemoryChannel(org.apache.flume.channel.MemoryChannel) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel)

Aggregations

Channel (org.apache.flume.Channel)38 MemoryChannel (org.apache.flume.channel.MemoryChannel)33 Transaction (org.apache.flume.Transaction)29 Test (org.junit.Test)27 Event (org.apache.flume.Event)26 PhoenixSink (org.apache.phoenix.flume.sink.PhoenixSink)25 Context (org.apache.flume.Context)16 EventDeliveryException (org.apache.flume.EventDeliveryException)8 Connection (java.sql.Connection)5 ResultSet (java.sql.ResultSet)5 Properties (java.util.Properties)5 SQLException (java.sql.SQLException)4 ArrayList (java.util.ArrayList)4 NullPhoenixSink (org.apache.phoenix.flume.sink.NullPhoenixSink)4 ChannelSelector (org.apache.flume.ChannelSelector)3 ChannelProcessor (org.apache.flume.channel.ChannelProcessor)3 ReplicatingChannelSelector (org.apache.flume.channel.ReplicatingChannelSelector)3 UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 ChannelException (org.apache.flume.ChannelException)2