Search in sources :

Example 21 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class JsonEventSerializerIT method testDifferentColumnNames.

@Test
public void testDifferentColumnNames() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_JSON_TEST";
    String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    String columns = "col1,col2,col3,col4";
    String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
    String columnsMapping = "{\"col1\":\"col1\",\"col2\":\"f2\",\"col3\":\"f3\",\"col4\":\"col4\"}";
    initSinkContext(fullTableName, ddl, columns, columnsMapping, rowkeyType, null);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "{\"col1\" : \"kalyan\", \"f2\" : 10.5, \"f3\" : [\"abc\",\"pqr\",\"xyz\"], \"col4\" : [1,2,3,4]}";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(1, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 22 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class CsvEventSerializerIT method testKeyGenerator.

@Test
public void testKeyGenerator() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_CSV_TEST";
    initSinkContextWithDefaults(fullTableName);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "kalyan,10.5,\"abc,pqr,xyz\",\"1,2,3,4\"";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(1, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 23 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class CsvEventSerializerIT method testMismatchKeyGenerator.

@Test
public void testMismatchKeyGenerator() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_CSV_TEST";
    initSinkContextWithDefaults(fullTableName);
    setConfig(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "kalyan,10.5,\"abc,pqr,xyz\",\"1,2,3,4\"";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    try {
        sink.process();
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getCause().getMessage().contains("java.lang.IllegalArgumentException: Invalid format:"));
    }
    dropTable(fullTableName);
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) SQLException(java.sql.SQLException) EventDeliveryException(org.apache.flume.EventDeliveryException) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 24 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class RegexEventSerializerIT method testMismatchKeyGenerator.

@Test
public void testMismatchKeyGenerator() throws EventDeliveryException, SQLException {
    final String fullTableName = generateUniqueName();
    initSinkContextWithDefaults(fullTableName);
    setConfig(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "val1\tval2";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    try {
        sink.process();
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getCause().getMessage().contains("java.lang.IllegalArgumentException: Invalid format:"));
    }
}
Also used : Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) Event(org.apache.flume.Event) SQLException(java.sql.SQLException) EventDeliveryException(org.apache.flume.EventDeliveryException) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Example 25 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class RegexEventSerializerIT method testEventsWithHeaders.

@Test
public void testEventsWithHeaders() throws Exception {
    sinkContext = new Context();
    final String fullTableName = generateUniqueName();
    final String ddl = "CREATE TABLE " + fullTableName + "  (rowkey VARCHAR not null, col1 varchar , cf1.col2 varchar , host varchar , source varchar \n" + "  CONSTRAINT pk PRIMARY KEY (rowkey))\n";
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,cf1.col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_HEADER_NAMES, "host,source");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 10;
    String col1 = "val1";
    String col2 = "val2";
    String hostHeader = "host1";
    String sourceHeader = "source1";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < numEvents; i++) {
        eventBody = (col1 + i) + "\t" + (col2 + i);
        Map<String, String> headerMap = Maps.newHashMapWithExpectedSize(2);
        headerMap.put("host", hostHeader);
        headerMap.put("source", sourceHeader);
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody), headerMap);
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try {
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
    } finally {
        if (conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}
Also used : Context(org.apache.flume.Context) Transaction(org.apache.flume.Transaction) MemoryChannel(org.apache.flume.channel.MemoryChannel) Channel(org.apache.flume.Channel) ResultSet(java.sql.ResultSet) Connection(java.sql.Connection) Event(org.apache.flume.Event) Properties(java.util.Properties) PhoenixSink(org.apache.phoenix.flume.sink.PhoenixSink) Test(org.junit.Test)

Aggregations

Channel (org.apache.flume.Channel)38 MemoryChannel (org.apache.flume.channel.MemoryChannel)33 Transaction (org.apache.flume.Transaction)29 Test (org.junit.Test)27 Event (org.apache.flume.Event)26 PhoenixSink (org.apache.phoenix.flume.sink.PhoenixSink)25 Context (org.apache.flume.Context)16 EventDeliveryException (org.apache.flume.EventDeliveryException)8 Connection (java.sql.Connection)5 ResultSet (java.sql.ResultSet)5 Properties (java.util.Properties)5 SQLException (java.sql.SQLException)4 ArrayList (java.util.ArrayList)4 NullPhoenixSink (org.apache.phoenix.flume.sink.NullPhoenixSink)4 ChannelSelector (org.apache.flume.ChannelSelector)3 ChannelProcessor (org.apache.flume.channel.ChannelProcessor)3 ReplicatingChannelSelector (org.apache.flume.channel.ReplicatingChannelSelector)3 UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 ChannelException (org.apache.flume.ChannelException)2