Examples with Context org.apache.flume.Context used on opensource projects

Example 6 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testConfiguration.

@Test
public void testConfiguration() {
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "test");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
}

Example 7 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testSinkLifecycle.

@Test
public void testSinkLifecycle() {
    String tableName = generateUniqueName();
    String ddl = "CREATE TABLE " + tableName + "  (flume_time timestamp not null, col1 varchar , col2 varchar" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, tableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    Assert.assertEquals(LifecycleState.START, sink.getLifecycleState());
    sink.stop();
    Assert.assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}

Example 8 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class JsonEventSerializerIT method initChannel.

private Channel initChannel() {
    // Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");
    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}

Example 9 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class CsvEventSerializerIT method initSinkContext.

private void initSinkContext(final String fullTableName, final String ddl, final String columns, final String csvDelimiter, final String csvQuote, final String csvEscape, final String csvArrayDelimiter, final String rowkeyType, final String headers) {
    Preconditions.checkNotNull(fullTableName);
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.CSV.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, columns);
    if (null != csvDelimiter)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CSV_DELIMITER, csvDelimiter);
    if (null != csvQuote)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CSV_QUOTE, csvQuote);
    if (null != csvEscape)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CSV_ESCAPE, csvEscape);
    if (null != csvArrayDelimiter)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CSV_ARRAY_DELIMITER, csvArrayDelimiter);
    if (null != rowkeyType)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, rowkeyType);
    if (null != headers)
        sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_HEADER_NAMES, headers);
}

Example 10 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class CsvEventSerializerIT method testEventsWithHeaders.

@Test
public void testEventsWithHeaders() throws Exception {
    sinkContext = new Context();
    final String fullTableName = "FLUME_CSV_TEST";
    final String ddl = "CREATE TABLE IF NOT EXISTS " + fullTableName + "  (rowkey VARCHAR not null, col1 varchar , col2 double, col3 varchar[], col4 integer[], host varchar , source varchar \n" + "  CONSTRAINT pk PRIMARY KEY (rowkey))\n";
    String columns = "col1,col2,col3,col4";
    String rowkeyType = DefaultKeyGenerator.UUID.name();
    String headers = "host,source";
    initSinkContext(fullTableName, ddl, columns, null, null, null, null, rowkeyType, headers);
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    int numEvents = 10;
    String col1 = "val1";
    String a1 = "\"aaa,bbb,ccc\"";
    String a2 = "\"1,2,3,4\"";
    String hostHeader = "host1";
    String sourceHeader = "source1";
    String eventBody = null;
    List<Event> eventList = Lists.newArrayListWithCapacity(numEvents);
    for (int i = 0; i < numEvents; i++) {
        eventBody = (col1 + i) + "," + i * 10.5 + "," + a1 + "," + a2;
        Map<String, String> headerMap = Maps.newHashMapWithExpectedSize(2);
        headerMap.put("host", hostHeader);
        headerMap.put("source", sourceHeader);
        Event event = EventBuilder.withBody(Bytes.toBytes(eventBody), headerMap);
        eventList.add(event);
    }
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    for (Event event : eventList) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();
    sink.process();
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try {
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
        assertTrue(rs.next());
        assertEquals("host1", rs.getString("host"));
        assertEquals("source1", rs.getString("source"));
    } finally {
        if (conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    dropTable(fullTableName);
}