Examples with PreparePublish com.continuuity.weave.kafka.client.PreparePublish used on opensource projects

Example 1 with PreparePublish

use of com.continuuity.weave.kafka.client.PreparePublish in project weave by continuuity.

the class SimpleKafkaClient method preparePublish.

@Override
public PreparePublish preparePublish(final String topic, final Compression compression) {
    final Map<Integer, MessageSetEncoder> encoders = Maps.newHashMap();
    return new PreparePublish() {

        @Override
        public PreparePublish add(byte[] payload, Object partitionKey) {
            return add(ByteBuffer.wrap(payload), partitionKey);
        }

        @Override
        public PreparePublish add(ByteBuffer payload, Object partitionKey) {
            // TODO: Partition
            int partition = 0;
            MessageSetEncoder encoder = encoders.get(partition);
            if (encoder == null) {
                encoder = getEncoder(compression);
                encoders.put(partition, encoder);
            }
            encoder.add(ChannelBuffers.wrappedBuffer(payload));
            return this;
        }

        @Override
        public ListenableFuture<?> publish() {
            List<ListenableFuture<?>> futures = Lists.newArrayListWithCapacity(encoders.size());
            for (Map.Entry<Integer, MessageSetEncoder> entry : encoders.entrySet()) {
                futures.add(doPublish(topic, entry.getKey(), entry.getValue().finish()));
            }
            encoders.clear();
            return Futures.allAsList(futures);
        }

        private ListenableFuture<?> doPublish(String topic, int partition, ChannelBuffer messageSet) {
            final KafkaRequest request = KafkaRequest.createProduce(topic, partition, messageSet);
            final SettableFuture<?> result = SettableFuture.create();
            final ConnectionPool.ConnectResult connection = connectionPool.connect(getTopicBroker(topic, partition).getAddress());
            connection.getChannelFuture().addListener(new ChannelFutureListener() {

                @Override
                public void operationComplete(ChannelFuture future) throws Exception {
                    try {
                        future.getChannel().write(request).addListener(getPublishChannelFutureListener(result, null, connection));
                    } catch (Exception e) {
                        result.setException(e);
                    }
                }
            });
            return result;
        }
    };
}

Example 2 with PreparePublish

use of com.continuuity.weave.kafka.client.PreparePublish in project weave by continuuity.

the class KafkaAppender method publishLogs.

private ListenableFuture<Integer> publishLogs() {
    // If the publisher is not available, simply returns a completed future.
    PreparePublish publisher = KafkaAppender.this.publisher.get();
    if (publisher == null) {
        return Futures.immediateFuture(0);
    }
    int count = 0;
    for (String json : Iterables.consumingIterable(buffer)) {
        publisher.add(Charsets.UTF_8.encode(json), 0);
        count++;
    }
    // Nothing to publish, simply returns a completed future.
    if (count == 0) {
        return Futures.immediateFuture(0);
    }
    bufferedSize.set(0);
    final int finalCount = count;
    return Futures.transform(publisher.publish(), new Function<Object, Integer>() {

        @Override
        public Integer apply(Object input) {
            return finalCount;
        }
    });
}