Examples with Context org.apache.flume.Context used on opensource projects

Example 26 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testExtendedSerializer.

@Test
public void testExtendedSerializer() throws Exception {
    /*
        Sadly, we can't mock a serializer, as the PhoenixSink does a Class.forName() to instantiate
        it. Instead. we'll setup a Flume channel and verify the data our custom serializer wrote.
        */
    final String fullTableName = "FLUME_TEST_EXTENDED";
    final String ddl = "CREATE TABLE " + fullTableName + " (ID BIGINT NOT NULL PRIMARY KEY, COUNTS UNSIGNED_LONG)";
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    conn.createStatement().execute(ddl);
    conn.commit();
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "FLUME_TEST_EXTENDED");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, CustomSerializer.class.getName());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "ID, COUNTS");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    PhoenixSink sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    // Send a test event through Flume, using our custom serializer
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(EventBuilder.withBody(Bytes.toBytes("test event")));
    transaction.commit();
    transaction.close();
    sink.process();
    sink.stop();
    // Verify our serializer wrote out data
    ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM FLUME_TEST_EXTENDED");
    assertTrue(rs.next());
    assertTrue(rs.getLong(1) == 1L);
}

Example 27 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testInvalidConfiguration.

@Test(expected = NullPointerException.class)
public void testInvalidConfiguration() {
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
}

Example 28 with Context

use of org.apache.flume.Context in project phoenix by apache.

the class PhoenixSinkIT method testInvalidConfigurationOfSerializer.

@Test(expected = RuntimeException.class)
public void testInvalidConfigurationOfSerializer() {
    sinkContext = new Context();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "test");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER, "unknown");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES, "col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.TIMESTAMP.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
}

Example 29 with Context

use of org.apache.flume.Context in project ignite by apache.

the class IgniteSinkTest method testSink.

/**
 * @throws Exception {@link Exception}.
 */
public void testSink() throws Exception {
    IgniteConfiguration cfg = loadConfiguration("modules/flume/src/test/resources/example-ignite.xml");
    cfg.setClientMode(false);
    final Ignite grid = startGrid("igniteServerNode", cfg);
    Context channelContext = new Context();
    channelContext.put("capacity", String.valueOf(EVENT_CNT));
    channelContext.put("transactionCapacity", String.valueOf(EVENT_CNT));
    Channel memoryChannel = new MemoryChannel();
    Configurables.configure(memoryChannel, channelContext);
    final CountDownLatch latch = new CountDownLatch(EVENT_CNT);
    final IgnitePredicate<Event> putLsnr = new IgnitePredicate<Event>() {

        @Override
        public boolean apply(Event evt) {
            assert evt != null;
            latch.countDown();
            return true;
        }
    };
    IgniteSink sink = new IgniteSink() {

        // Setting the listener on cache before sink processing starts.
        @Override
        public synchronized void start() {
            super.start();
            grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).localListen(putLsnr, EVT_CACHE_OBJECT_PUT);
        }
    };
    sink.setName("IgniteSink");
    sink.setChannel(memoryChannel);
    Context ctx = new Context();
    ctx.put(IgniteSinkConstants.CFG_CACHE_NAME, CACHE_NAME);
    ctx.put(IgniteSinkConstants.CFG_PATH, "example-ignite.xml");
    ctx.put(IgniteSinkConstants.CFG_EVENT_TRANSFORMER, "org.apache.ignite.stream.flume.TestEventTransformer");
    Configurables.configure(sink, ctx);
    sink.start();
    try {
        Transaction tx = memoryChannel.getTransaction();
        tx.begin();
        for (int i = 0; i < EVENT_CNT; i++) memoryChannel.put(EventBuilder.withBody((String.valueOf(i) + ": " + i).getBytes()));
        tx.commit();
        tx.close();
        Sink.Status status = Sink.Status.READY;
        while (status != Sink.Status.BACKOFF) {
            status = sink.process();
        }
    } finally {
        sink.stop();
    }
    // Checks that 10000 events successfully processed in 10 seconds.
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).stopLocalListen(putLsnr);
    IgniteCache<String, Integer> cache = grid.cache(CACHE_NAME);
    // Checks that each event was processed properly.
    for (int i = 0; i < EVENT_CNT; i++) {
        assertEquals(i, (int) cache.get(String.valueOf(i)));
    }
    assertEquals(EVENT_CNT, cache.size(CachePeekMode.PRIMARY));
}

Example 30 with Context

use of org.apache.flume.Context in project apex-malhar by apache.

the class FlumeSink method configure.

/* End Configurable Interface */
@SuppressWarnings({ "UseSpecificCatch", "BroadCatchBlock", "TooBroadCatch" })
private static <T> T configure(String key, Class<T> clazz, Context context) {
    String classname = context.getString(key);
    if (classname == null) {
        return null;
    }
    try {
        Class<?> loadClass = Thread.currentThread().getContextClassLoader().loadClass(classname);
        if (clazz.isAssignableFrom(loadClass)) {
            @SuppressWarnings("unchecked") T object = (T) loadClass.newInstance();
            if (object instanceof Configurable) {
                Context context1 = new Context(context.getSubProperties(key + '.'));
                String id = context1.getString(Storage.ID);
                if (id == null) {
                    id = context.getString(Storage.ID);
                    logger.debug("{} inherited id={} from sink", key, id);
                    context1.put(Storage.ID, id);
                }
                ((Configurable) object).configure(context1);
            }
            return object;
        } else {
            logger.error("key class {} does not implement {} interface", classname, Storage.class.getCanonicalName());
            throw new Error("Invalid storage " + classname);
        }
    } catch (Error error) {
        throw error;
    } catch (RuntimeException re) {
        throw re;
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}