Examples with Context org.apache.flume.Context used on opensource projects

Example 16 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class CsvEventSerializerIT method initChannel.

private Channel initChannel() {
    // Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");
    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}

Example 17 with Context

use of org.apache.flume.Context in project logging-log4j2 by apache.

the class FlumeAppenderTest method setUp.

@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
         * Clear out all other appenders associated with this logger to ensure
         * we're only hitting the Avro appender.
         */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);
    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);
    eventSource.setChannelProcessor(new ChannelProcessor(cs));
    eventSource.start();
    Assert.assertTrue("Reached start or error", LifecycleController.waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START, eventSource.getLifecycleState());
}

Example 18 with Context

use of org.apache.flume.Context in project logging-log4j2 by apache.

the class FlumeAppenderTest method testNotConnected.

@Test
public void testNotConnected() throws Exception {
    eventSource.stop();
    final String altPort = Integer.toString(Integer.parseInt(testPort) + 1);
    final Agent[] agents = new Agent[] { Agent.createAgent("localhost", testPort), Agent.createAgent("localhost", altPort) };
    final FlumeAppender avroAppender = FlumeAppender.createAppender(agents, null, null, "false", "Avro", null, "1000", "1000", "1", "1000", "avro", "false", null, null, null, null, null, "true", "1", null, null, null, null);
    avroAppender.start();
    Assert.assertTrue("Appender Not started", avroAppender.isStarted());
    avroLogger.addAppender(avroAppender);
    avroLogger.setLevel(Level.ALL);
    try {
        avroLogger.info("Test message");
        Assert.fail("Exception should have been thrown");
    } catch (final Exception ex) {
    }
    try {
        final Context context = new Context();
        context.put("port", altPort);
        context.put("bind", "0.0.0.0");
        Configurables.configure(eventSource, context);
        eventSource.start();
    } catch (final ChannelException e) {
        Assert.fail("Caught exception while resetting port to " + altPort + " : " + e.getMessage());
    }
    avroLogger.info("Test message 2");
    final Transaction transaction = channel.getTransaction();
    transaction.begin();
    final Event event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertTrue("Channel contained event, but not expected message", getBody(event).endsWith("Test message 2"));
    transaction.commit();
    transaction.close();
}

Example 19 with Context

use of org.apache.flume.Context in project logging-log4j2 by apache.

the class FlumeAppenderTest method testReconnect.

@Test
public void testReconnect() throws Exception {
    final String altPort = Integer.toString(Integer.parseInt(testPort) + 1);
    final Agent[] agents = new Agent[] { Agent.createAgent("localhost", testPort), Agent.createAgent("localhost", altPort) };
    final FlumeAppender avroAppender = FlumeAppender.createAppender(agents, null, null, "false", "Avro", null, "1000", "1000", "1", "1000", "avro", "false", null, null, null, null, null, "true", "1", null, null, null, null);
    avroAppender.start();
    avroLogger.addAppender(avroAppender);
    avroLogger.setLevel(Level.ALL);
    avroLogger.info("Test message");
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertTrue("Channel contained event, but not expected message. Received : " + getBody(event), getBody(event).endsWith("Test message"));
    transaction.commit();
    transaction.close();
    eventSource.stop();
    try {
        final Context context = new Context();
        context.put("port", altPort);
        context.put("bind", "0.0.0.0");
        Configurables.configure(eventSource, context);
        eventSource.start();
    } catch (final ChannelException e) {
        Assert.fail("Caught exception while resetting port to " + altPort + " : " + e.getMessage());
    }
    avroLogger.info("Test message 2");
    transaction = channel.getTransaction();
    transaction.begin();
    event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertTrue("Channel contained event, but not expected message", getBody(event).endsWith("Test message 2"));
    transaction.commit();
    transaction.close();
}

Example 20 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class RegexEventSerializerIT method testEventsWithHeaders.

@Test
public void testEventsWithHeaders() throws Exception {
    sinkContext = new Context();
    final String fullTableName = generateUniqueName();
    final String ddl = "CREATE TABLE " + fullTableName + "  (rowkey VARCHAR not null, col1 varchar , cf1.col2 varchar , host varchar , source varchar \n" + "  CONSTRAINT pk PRIMARY KEY (rowkey))\n";
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,cf1.col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_HEADER_NAMES, "host,source");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 10;
    String col1 = "val1";
    String col2 = "val2";
    String hostHeader = "host1";
    String sourceHeader = "source1";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < numEvents; i++) {
        eventBody = (col1 + i) + "\t" + (col2 + i);
        Map<String, String> headerMap = Maps.newHashMapWithExpectedSize(2);
        headerMap.put("host", hostHeader);
        headerMap.put("source", sourceHeader);
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody), headerMap);
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try {
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
    } finally {
        if (conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}