Examples with Channel org.apache.flume.Channel used on opensource projects

Example 16 with Channel

use of org.apache.flume.Channel in project MSEC by Tencent.

the class TestProfobufSource method testBasic.

@Test
public void testBasic() throws Exception {
    Context context = new Context();
    context.put("bind", "localhost");
    context.put("port", "45673");
    context.put("threads", "1");
    ProtobufSource source = new ProtobufSource();
    source.configure(context);
    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    // for faster tests
    channelContext.put("keep-alive", "0");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context(channelContext));
    Sink sink = new LoggerSink();
    sink.setChannel(channel);
    sink.start();
    DefaultSinkProcessor proc = new DefaultSinkProcessor();
    proc.setSinks(Collections.singletonList(sink));
    SinkRunner sinkRunner = new SinkRunner(proc);
    sinkRunner.start();
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    ChannelProcessor chp = new ChannelProcessor(rcs);
    source.setChannelProcessor(chp);
    source.start();
    Thread.sleep(5000);
    source.stop();
    sinkRunner.stop();
    sink.stop();
}

Example 17 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class CsvEventSerializerIT method testBatchEvents.

@Test
public void testBatchEvents() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_CSV_TEST";
    initSinkContextWithDefaults(fullTableName);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 150;
    String col1 = "val1";
    String a1 = "\"aaa,bbb,ccc\"";
    String a2 = "\"1,2,3,4\"";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < eventList.size(); i++) {
        eventBody = (col1 + i) + "," + i * 10.5 + "," + a1 + "," + a2;
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(eventList.size(), rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}

Example 18 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class CsvEventSerializerIT method initChannel.

private Channel initChannel() {
    // Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");
    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}

Example 19 with Channel

use of org.apache.flume.Channel in project phoenix by apache.

the class CsvEventSerializerIT method testWithDefaultDelimiters.

@Test
public void testWithDefaultDelimiters() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_CSV_TEST";
    String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (flume_time timestamp not null, col1 varchar , col2 double, col3 varchar[], col4 integer[]" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    String columns = "col1,col2,col3,col4";
    String rowkeyType = DefaultKeyGenerator.TIMESTAMP.name();
    initSinkContext(fullTableName, ddl, columns, null, null, null, null, rowkeyType, null);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "kalyan,10.5,\"abc,pqr,xyz\",\"1,2,3,4\"";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    sink.process();
    int rowsInDb = countRows(fullTableName);
    assertEquals(1, rowsInDb);
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}

Example 20 with Channel

use of org.apache.flume.Channel in project logging-log4j2 by apache.

the class FlumeAppenderTest method setUp.

@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
         * Clear out all other appenders associated with this logger to ensure
         * we're only hitting the Avro appender.
         */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);
    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);
    eventSource.setChannelProcessor(new ChannelProcessor(cs));
    eventSource.start();
    Assert.assertTrue("Reached start or error", LifecycleController.waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START, eventSource.getLifecycleState());
}