use of org.apache.flume.Sink in project phoenix by apache.
the class PhoenixSinkIT method testSinkCreation.
@Test
public void testSinkCreation() {
SinkFactory factory = new DefaultSinkFactory();
Sink sink = factory.create("PhoenixSink__", "org.apache.phoenix.flume.sink.PhoenixSink");
Assert.assertNotNull(sink);
Assert.assertTrue(PhoenixSink.class.isInstance(sink));
}
use of org.apache.flume.Sink in project rocketmq-externals by apache.
the class RocketMQSinkTest method testBatchEvent.
@Test
public void testBatchEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
/*
start sink
*/
Context context = new Context();
context.put(NAME_SERVER_CONFIG, nameServer);
context.put(TAG_CONFIG, tag);
context.put(BATCH_SIZE_CONFIG, String.valueOf(batchSize));
RocketMQSink sink = new RocketMQSink();
Configurables.configure(sink, context);
MemoryChannel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
/*
mock flume source
*/
Map<String, String> msgs = new HashMap<>();
Transaction tx = channel.getTransaction();
tx.begin();
int sendNum = 0;
for (int i = 0; i < batchSize; i++) {
String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss:SSSS");
Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
channel.put(event);
log.info("publish message : {}", sendMsg);
String[] sendMsgKv = sendMsg.split(",");
msgs.put(sendMsgKv[1], sendMsgKv[0]);
sendNum++;
Thread.sleep(10);
}
log.info("send message num={}", sendNum);
tx.commit();
tx.close();
Sink.Status status = sink.process();
if (status == Sink.Status.BACKOFF) {
fail("Error");
}
sink.stop();
/*
consumer message
*/
consumer = new DefaultMQPullConsumer(consumerGroup);
consumer.setNamesrvAddr(nameServer);
consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
consumer.start();
int receiveNum = 0;
String receiveMsg = null;
Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
for (MessageQueue queue : queues) {
long offset = getMessageQueueOffset(queue);
PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
if (pullResult.getPullStatus() == PullStatus.FOUND) {
for (MessageExt message : pullResult.getMsgFoundList()) {
byte[] body = message.getBody();
receiveMsg = new String(body, "UTF-8");
String[] receiveMsgKv = receiveMsg.split(",");
msgs.remove(receiveMsgKv[1]);
log.info("receive message : {}", receiveMsg);
receiveNum++;
}
long nextBeginOffset = pullResult.getNextBeginOffset();
putMessageQueueOffset(queue, nextBeginOffset);
}
}
log.info("receive message num={}", receiveNum);
/*
wait for processQueueTable init
*/
Thread.sleep(1000);
consumer.shutdown();
assertEquals(msgs.size(), 0);
}
use of org.apache.flume.Sink in project rocketmq-externals by apache.
the class RocketMQSinkTest method testNullEvent.
@Test
public void testNullEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
/*
start sink
*/
Context context = new Context();
context.put(NAME_SERVER_CONFIG, nameServer);
context.put(TAG_CONFIG, tag);
RocketMQSink sink = new RocketMQSink();
Configurables.configure(sink, context);
MemoryChannel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
Sink.Status status = sink.process();
assertEquals(status, Sink.Status.BACKOFF);
sink.stop();
}
use of org.apache.flume.Sink in project ignite by apache.
the class IgniteSinkTest method testSink.
/**
* @throws Exception {@link Exception}.
*/
public void testSink() throws Exception {
IgniteConfiguration cfg = loadConfiguration("modules/flume/src/test/resources/example-ignite.xml");
cfg.setClientMode(false);
final Ignite grid = startGrid("igniteServerNode", cfg);
Context channelContext = new Context();
channelContext.put("capacity", String.valueOf(EVENT_CNT));
channelContext.put("transactionCapacity", String.valueOf(EVENT_CNT));
Channel memoryChannel = new MemoryChannel();
Configurables.configure(memoryChannel, channelContext);
final CountDownLatch latch = new CountDownLatch(EVENT_CNT);
final IgnitePredicate<Event> putLsnr = new IgnitePredicate<Event>() {
@Override
public boolean apply(Event evt) {
assert evt != null;
latch.countDown();
return true;
}
};
IgniteSink sink = new IgniteSink() {
// Setting the listener on cache before sink processing starts.
@Override
public synchronized void start() {
super.start();
grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).localListen(putLsnr, EVT_CACHE_OBJECT_PUT);
}
};
sink.setName("IgniteSink");
sink.setChannel(memoryChannel);
Context ctx = new Context();
ctx.put(IgniteSinkConstants.CFG_CACHE_NAME, CACHE_NAME);
ctx.put(IgniteSinkConstants.CFG_PATH, "example-ignite.xml");
ctx.put(IgniteSinkConstants.CFG_EVENT_TRANSFORMER, "org.apache.ignite.stream.flume.TestEventTransformer");
Configurables.configure(sink, ctx);
sink.start();
try {
Transaction tx = memoryChannel.getTransaction();
tx.begin();
for (int i = 0; i < EVENT_CNT; i++) memoryChannel.put(EventBuilder.withBody((String.valueOf(i) + ": " + i).getBytes()));
tx.commit();
tx.close();
Sink.Status status = Sink.Status.READY;
while (status != Sink.Status.BACKOFF) {
status = sink.process();
}
} finally {
sink.stop();
}
// Checks that 10000 events successfully processed in 10 seconds.
assertTrue(latch.await(10, TimeUnit.SECONDS));
grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).stopLocalListen(putLsnr);
IgniteCache<String, Integer> cache = grid.cache(CACHE_NAME);
// Checks that each event was processed properly.
for (int i = 0; i < EVENT_CNT; i++) {
assertEquals(i, (int) cache.get(String.valueOf(i)));
}
assertEquals(EVENT_CNT, cache.size(CachePeekMode.PRIMARY));
}
use of org.apache.flume.Sink in project MSEC by Tencent.
the class TestProfobufSource method testBasic.
@Test
public void testBasic() throws Exception {
Context context = new Context();
context.put("bind", "localhost");
context.put("port", "45673");
context.put("threads", "1");
ProtobufSource source = new ProtobufSource();
source.configure(context);
Map<String, String> channelContext = new HashMap();
channelContext.put("capacity", "1000000");
// for faster tests
channelContext.put("keep-alive", "0");
Channel channel = new MemoryChannel();
Configurables.configure(channel, new Context(channelContext));
Sink sink = new LoggerSink();
sink.setChannel(channel);
sink.start();
DefaultSinkProcessor proc = new DefaultSinkProcessor();
proc.setSinks(Collections.singletonList(sink));
SinkRunner sinkRunner = new SinkRunner(proc);
sinkRunner.start();
ChannelSelector rcs = new ReplicatingChannelSelector();
rcs.setChannels(Collections.singletonList(channel));
ChannelProcessor chp = new ChannelProcessor(rcs);
source.setChannelProcessor(chp);
source.start();
Thread.sleep(5000);
source.stop();
sinkRunner.stop();
sink.stop();
}
Aggregations