Examples with Context org.apache.flume.Context used on opensource projects

Example 1 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class RegexEventSerializerIT method testApacheLogRegex.

@Test
public void testApacheLogRegex() throws Exception {
    sinkContext = new Context();
    final String fullTableName = generateUniqueName();
    final String logRegex = "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) \"([^ ]+) ([^ ]+)" + " ([^\"]+)\" (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\")" + " ([^ \"]*|\"[^\"]*\"))?";
    final String columns = "host,identity,user,time,method,request,protocol,status,size,referer,agent";
    String ddl = "CREATE TABLE " + fullTableName + "  (uid VARCHAR NOT NULL, user VARCHAR, time varchar, host varchar , identity varchar, method varchar, request varchar , protocol varchar," + "  status integer , size integer , referer varchar , agent varchar CONSTRAINT pk PRIMARY KEY (uid))\n";
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, logRegex);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, columns);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    String message1 = "33.22.11.00 - user1 [12/Dec/2013:07:01:19 +0000] " + "\"GET /wp-admin/css/install.css HTTP/1.0\" 200 813 " + "\"http://www.google.com\" \"Mozilla/5.0 (comp" + "atible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\"";
    String message2 = "192.168.20.1 - user2 [13/Dec/2013:06:05:19 +0000] " + "\"GET /wp-admin/css/install.css HTTP/1.0\" 400 363 " + "\"http://www.salesforce.com/in/?ir=1\" \"Mozilla/5.0 (comp" + "atible;)\"";
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final Event event1 = EventBuilder.withBody(Bytes.toBytes(message1));
    final Event event2 = EventBuilder.withBody(Bytes.toBytes(message2));
    final Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event1);
    channel.put(event2);
    transaction.commit();
    transaction.close();
    sink.process();
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try {
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertTrue(rs.next());
    } finally {
        if (conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}

Example 2 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class RegexEventSerializerIT method initSinkContextWithDefaults.

private void initSinkContextWithDefaults(final String fullTableName) {
    Preconditions.checkNotNull(fullTableName);
    sinkContext = new Context();
    String ddl = "CREATE TABLE " + fullTableName + "  (flume_time timestamp not null, col1 varchar , col2 varchar" + "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION, "^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
}

Example 3 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testInvalidTable.

@Test
public void testInvalidTable() {
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "flume_test");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    try {
        sink.start();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage(), e.getMessage().contains("ERROR 1012 (42M03): Table undefined."));
    }
}

Example 4 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testExtendedSink.

@Test
public void testExtendedSink() throws Exception {
    // Create a mock NullPhoenixSink which extends PhoenixSink, and verify configure is invoked()
    PhoenixSink sink = mock(NullPhoenixSink.class);
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "FLUME_TEST_EXTENDED");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, CustomSerializer.class.getName());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "ID, COUNTS");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    Configurables.configure(sink, sinkContext);
    verify(sink).configure(sinkContext);
}

Example 5 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testConfiguration.

@Test
public void testConfiguration() {
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "test");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
}