Examples with Context org.apache.flume.Context used on opensource projects

Example 36 with Context

use of org.apache.flume.Context in project apex-malhar by apache.

the class InterceptorTestHelper method testIntercept_Event.

public void testIntercept_Event() {
    builder.configure(new Context(context));
    Interceptor interceptor = builder.build();
    assertArrayEquals("Empty Bytes", "\001\001\001".getBytes(), interceptor.intercept(new MyEvent("".getBytes())).getBody());
    assertArrayEquals("One Separator", "\001\001\001".getBytes(), interceptor.intercept(new MyEvent("\002".getBytes())).getBody());
    assertArrayEquals("Two Separators", "\001\001\001".getBytes(), interceptor.intercept(new MyEvent("\002\002".getBytes())).getBody());
    assertArrayEquals("One Field", "\001\001\001".getBytes(), interceptor.intercept(new MyEvent("First".getBytes())).getBody());
    assertArrayEquals("Two Fields", "First\001\001\001".getBytes(), interceptor.intercept(new MyEvent("\002First".getBytes())).getBody());
    assertArrayEquals("Two Fields", "\001\001\001".getBytes(), interceptor.intercept(new MyEvent("First\001".getBytes())).getBody());
    assertArrayEquals("Two Fields", "Second\001\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002Second".getBytes())).getBody());
    assertArrayEquals("Three Fields", "Second\001\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002Second\002".getBytes())).getBody());
    assertArrayEquals("Three Fields", "\001Second\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002\002Second".getBytes())).getBody());
    assertArrayEquals("Four Fields", "\001Second\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002\002Second\002".getBytes())).getBody());
    assertArrayEquals("Five Fields", "\001Second\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002\002Second\002\002".getBytes())).getBody());
    assertArrayEquals("Six Fields", "\001Second\001\001".getBytes(), interceptor.intercept(new MyEvent("First\002\002Second\002\002\002".getBytes())).getBody());
}

Example 37 with Context

use of org.apache.flume.Context in project apex-malhar by apache.

the class HDFSStorageTest method getStorage.

private HDFSStorage getStorage(String id, boolean restore) {
    Context ctx = new Context();
    STORAGE_DIRECTORY = testMeta.baseDir;
    ctx.put(HDFSStorage.BASE_DIR_KEY, testMeta.baseDir);
    ctx.put(HDFSStorage.RESTORE_KEY, Boolean.toString(restore));
    ctx.put(HDFSStorage.ID, id);
    ctx.put(HDFSStorage.BLOCKSIZE, "256");
    HDFSStorage lstorage = new HDFSStorage();
    lstorage.configure(ctx);
    lstorage.setup(null);
    return lstorage;
}

Example 38 with Context

use of org.apache.flume.Context in project rocketmq-externals by apache.

the class RocketMQSinkTest method testEvent.

@Test
public void testEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
    /*
        start sink
         */
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    RocketMQSink sink = new RocketMQSink();
    Configurables.configure(sink, context);
    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();
    /*
        mock flume source
         */
    String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
    channel.put(event);
    tx.commit();
    tx.close();
    log.info("publish message : {}", sendMsg);
    Sink.Status status = sink.process();
    if (status == Sink.Status.BACKOFF) {
        fail("Error");
    }
    sink.stop();
    /*
        consumer message
         */
    consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setNamesrvAddr(nameServer);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
    consumer.start();
    String receiveMsg = null;
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
    for (MessageQueue queue : queues) {
        long offset = getMessageQueueOffset(queue);
        PullResult pullResult = consumer.pull(queue, tag, offset, 32);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            for (MessageExt message : pullResult.getMsgFoundList()) {
                byte[] body = message.getBody();
                receiveMsg = new String(body, "UTF-8");
                log.info("receive message : {}", receiveMsg);
            }
            long nextBeginOffset = pullResult.getNextBeginOffset();
            putMessageQueueOffset(queue, nextBeginOffset);
        }
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    consumer.shutdown();
    assertEquals(sendMsg, receiveMsg);
}

Example 39 with Context

use of org.apache.flume.Context in project rocketmq-externals by apache.

the class RocketMQSourceTest method testEvent.

@Test
public void testEvent() throws EventDeliveryException, MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    // publish test message
    DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
    producer.setNamesrvAddr(nameServer);
    String sendMsg = "\"Hello Flume\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    try {
        producer.start();
        Message msg = new Message(TOPIC_DEFAULT, tag, sendMsg.getBytes("UTF-8"));
        SendResult sendResult = producer.send(msg);
        log.info("publish message : {}, sendResult:{}", sendMsg, sendResult);
    } catch (Exception e) {
        throw new MQClientException("Failed to publish messages", e);
    } finally {
        producer.shutdown();
    }
    // start source
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector channelSelector = new ReplicatingChannelSelector();
    channelSelector.setChannels(channels);
    ChannelProcessor channelProcessor = new ChannelProcessor(channelSelector);
    RocketMQSource source = new RocketMQSource();
    source.setChannelProcessor(channelProcessor);
    Configurables.configure(source, context);
    source.start();
    PollableSource.Status status = source.process();
    if (status == PollableSource.Status.BACKOFF) {
        fail("Error");
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    source.stop();
    /*
        mock flume sink
         */
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    if (event == null) {
        transaction.commit();
        fail("Error");
    }
    byte[] body = event.getBody();
    String receiveMsg = new String(body, "UTF-8");
    log.info("receive message : {}", receiveMsg);
    assertEquals(sendMsg, receiveMsg);
}