Examples with JdkZlibEncoder io.netty.handler.codec.compression.JdkZlibEncoder used on opensource projects

Example 1 with JdkZlibEncoder

use of io.netty.handler.codec.compression.JdkZlibEncoder in project vert.x by eclipse.

the class HttpCompressionTest method setUp.

public void setUp() throws Exception {
    super.setUp();
    EmbeddedChannel channel = new EmbeddedChannel();
    channel.pipeline().addFirst(new JdkZlibEncoder(ZlibWrapper.GZIP, compressionLevel));
    channel.writeAndFlush(Unpooled.copiedBuffer(COMPRESS_TEST_STRING, StandardCharsets.UTF_8));
    channel.close();
    Queue<Object> messages = channel.outboundMessages();
    compressedTestString = Buffer.buffer();
    ByteBuf buf;
    while ((buf = (ByteBuf) messages.poll()) != null) {
        byte[] tmp = new byte[buf.readableBytes()];
        buf.readBytes(tmp);
        buf.release();
        compressedTestString.appendBytes(tmp);
    }
}

Example 2 with JdkZlibEncoder

use of io.netty.handler.codec.compression.JdkZlibEncoder in project mantis by Netflix.

the class RemoteObservable method createTcpConnectionToServer.

private static <K, V> Observable<GroupedObservable<K, V>> createTcpConnectionToServer(final ConnectToGroupedObservable<K, V> params, final RemoteUnsubscribe remoteUnsubscribe, final RxMetrics metrics, final Action0 connectionDisconnectCallback, Observable<Integer> closeTrigger) {
    final Decoder<K> keyDecoder = params.getKeyDecoder();
    final Decoder<V> valueDecoder = params.getValueDecoder();
    loadFastProperties();
    return RxNetty.createTcpClient(params.getHost(), params.getPort(), new PipelineConfiguratorComposite<RemoteRxEvent, List<RemoteRxEvent>>(new PipelineConfigurator<RemoteRxEvent, List<RemoteRxEvent>>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            if (enableNettyLogging) {
                // uncomment to enable debug logging
                pipeline.addFirst(new LoggingHandler(LogLevel.ERROR));
            }
            if (enableHeartBeating) {
                pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
                pipeline.addLast("heartbeat", new HeartbeatHandler());
            }
            if (enableCompression) {
                pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
                pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            }
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(maxFrameLength, 0, 4, 0, 4));
        }
    }, new BatchedRxEventPipelineConfigurator())).connect().flatMap(new Func1<ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>>, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
            // send subscribe event to server
            connection.writeAndFlush(RemoteRxEvent.subscribed(params.getName(), params.getSubscribeParameters()));
            remoteUnsubscribe.setConnection(connection);
            return connection.getInput().lift(new DropOperator<RemoteRxEvent>("incoming_" + RemoteObservable.class.getCanonicalName() + "_createTcpConnectionToServerGroups"));
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            // connection completed
            logger.warn("Detected connection completed when trying to connect to host: " + params.getHost() + " port: " + params.getPort());
            connectionDisconnectCallback.call();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(Throwable t1) {
            logger.warn("Detected connection error when trying to connect to host: " + params.getHost() + " port: " + params.getPort(), t1);
            connectionDisconnectCallback.call();
            // complete if error occurs
            return Observable.empty();
        }
    }).takeUntil(closeTrigger).map(new Func1<RemoteRxEvent, Notification<byte[]>>() {

        @Override
        public Notification<byte[]> call(RemoteRxEvent rxEvent) {
            if (rxEvent.getType() == RemoteRxEvent.Type.next) {
                metrics.incrementNextCount();
                return Notification.createOnNext(rxEvent.getData());
            } else if (rxEvent.getType() == RemoteRxEvent.Type.error) {
                metrics.incrementErrorCount();
                return Notification.createOnError(fromBytesToThrowable(rxEvent.getData()));
            } else if (rxEvent.getType() == RemoteRxEvent.Type.completed) {
                metrics.incrementCompletedCount();
                return Notification.createOnCompleted();
            } else {
                throw new RuntimeException("RemoteRxEvent of type:" + rxEvent.getType() + ", not supported.");
            }
        }
    }).<byte[]>dematerialize().groupBy(new Func1<byte[], K>() {

        @Override
        public K call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            // ignore notification type in key selector
            buff.get();
            int keyLength = buff.getInt();
            byte[] key = new byte[keyLength];
            buff.get(key);
            return keyDecoder.decode(key);
        }
    }, new Func1<byte[], Notification<V>>() {

        @Override
        public Notification<V> call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            byte notificationType = buff.get();
            if (notificationType == 1) {
                int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] valueBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(valueBytes, 0, dataLength);
                V value = valueDecoder.decode(valueBytes);
                return Notification.createOnNext(value);
            } else if (notificationType == 2) {
                return Notification.createOnCompleted();
            } else if (notificationType == 3) {
                int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] errorBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(errorBytes, 0, dataLength);
                return Notification.createOnError(fromBytesToThrowable(errorBytes));
            } else {
                throw new RuntimeException("Notification encoding not support: " + notificationType);
            }
        }
    }).map(new Func1<GroupedObservable<K, Notification<V>>, GroupedObservable<K, V>>() {

        @Override
        public GroupedObservable<K, V> call(GroupedObservable<K, Notification<V>> group) {
            return GroupedObservableUtils.createGroupedObservable(group.getKey(), group.<V>dematerialize());
        }
    }).doOnEach(new Observer<GroupedObservable<K, V>>() {

        @Override
        public void onCompleted() {
            logger.info("RemoteRxEvent, name: {} onCompleted()", params.getName());
        }

        @Override
        public void onError(Throwable e) {
            logger.error("RemoteRxEvent, name: {} onError()", params.getName(), e);
        }

        @Override
        public void onNext(GroupedObservable<K, V> group) {
            if (logger.isDebugEnabled()) {
                logger.debug("RemoteRxEvent, name: {} new key: {}", params.getName(), group.getKey());
            }
        }
    });
}

Example 3 with JdkZlibEncoder

use of io.netty.handler.codec.compression.JdkZlibEncoder in project mantis by Netflix.

the class RemoteObservable method createTcpConnectionToGOServer.

private static <K, V> Observable<MantisGroup<K, V>> createTcpConnectionToGOServer(final ConnectToGroupedObservable<K, V> params, final RemoteUnsubscribe remoteUnsubscribe, final RxMetrics metrics, final Action0 connectionDisconnectCallback, Observable<Integer> closeTrigger, final SpscArrayQueue<MantisGroup<?, ?>> inputQueue) {
    final Decoder<K> keyDecoder = params.getKeyDecoder();
    final Decoder<V> valueDecoder = params.getValueDecoder();
    loadFastProperties();
    return RxNetty.createTcpClient(params.getHost(), params.getPort(), new PipelineConfiguratorComposite<RemoteRxEvent, List<RemoteRxEvent>>(new PipelineConfigurator<RemoteRxEvent, List<RemoteRxEvent>>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            if (enableNettyLogging) {
                // uncomment to enable debug logging
                pipeline.addFirst(new LoggingHandler(LogLevel.ERROR));
            }
            if (enableHeartBeating) {
                pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
                pipeline.addLast("heartbeat", new HeartbeatHandler());
            }
            if (enableCompression) {
                pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
                pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            }
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(maxFrameLength, 0, 4, 0, 4));
        }
    }, new BatchedRxEventPipelineConfigurator())).connect().flatMap(new Func1<ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>>, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
            // send subscribe event to server
            connection.writeAndFlush(RemoteRxEvent.subscribed(params.getName(), params.getSubscribeParameters()));
            remoteUnsubscribe.setConnection(connection);
            return connection.getInput().lift(new DropOperator<RemoteRxEvent>("incoming_" + RemoteObservable.class.getCanonicalName() + "_createTcpConnectionToServerGroups"));
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            // connection completed
            logger.warn("Detected connection completed when trying to connect to host: " + params.getHost() + " port: " + params.getPort());
            connectionDisconnectCallback.call();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(Throwable t1) {
            logger.warn("Detected connection error when trying to connect to host: " + params.getHost() + " port: " + params.getPort(), t1);
            connectionDisconnectCallback.call();
            // complete if error occurs
            return Observable.empty();
        }
    }).takeUntil(closeTrigger).filter(new Func1<RemoteRxEvent, Boolean>() {

        @Override
        public Boolean call(RemoteRxEvent rxEvent) {
            return (rxEvent.getType() == RemoteRxEvent.Type.next);
        }
    }).map(new Func1<RemoteRxEvent, Notification<byte[]>>() {

        @Override
        public Notification<byte[]> call(RemoteRxEvent rxEvent) {
            metrics.incrementNextCount();
            return Notification.createOnNext(rxEvent.getData());
        }
    }).<byte[]>dematerialize().map(new Func1<byte[], MantisGroup<K, V>>() {

        @Override
        public MantisGroup<K, V> call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            // ignore notification type in key selector
            buff.get();
            int keyLength = buff.getInt();
            byte[] key = new byte[keyLength];
            buff.get(key);
            K keyVal = keyDecoder.decode(key);
            V value = null;
            buff = ByteBuffer.wrap((byte[]) bytes);
            byte notificationType = buff.get();
            if (notificationType == 1) {
                // int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] valueBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(valueBytes, 0, dataLength);
                value = valueDecoder.decode(valueBytes);
            } else {
                throw new RuntimeException("Notification encoding not support: " + notificationType);
            }
            return new MantisGroup<K, V>(keyVal, value);
        }
    }).doOnEach(new Observer<MantisGroup<K, V>>() {

        @Override
        public void onCompleted() {
            logger.info("RemoteRxEvent, name: " + params.getName() + " onCompleted()");
        }

        @Override
        public void onError(Throwable e) {
            logger.error("RemoteRxEvent, name: " + params.getName() + " onError()", e);
        }

        @Override
        public void onNext(MantisGroup<K, V> group) {
            if (logger.isDebugEnabled()) {
                logger.debug("RemoteRxEvent, name: " + params.getName() + " new key: " + group.getKeyValue());
            }
        }
    });
}

Example 4 with JdkZlibEncoder

use of io.netty.handler.codec.compression.JdkZlibEncoder in project mantis by Netflix.

the class RemoteObservable method createTcpConnectionToServer.

private static <T> Observable<T> createTcpConnectionToServer(final ConnectToObservable<T> params, final RemoteUnsubscribe remoteUnsubscribe, final RxMetrics metrics, final Action0 connectionDisconnectCallback, Observable<Integer> closeTrigger) {
    final Decoder<T> decoder = params.getDecoder();
    loadFastProperties();
    return RxNetty.createTcpClient(params.getHost(), params.getPort(), new PipelineConfiguratorComposite<RemoteRxEvent, List<RemoteRxEvent>>(new PipelineConfigurator<RemoteRxEvent, List<RemoteRxEvent>>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            if (enableNettyLogging) {
                // uncomment to enable debug logging
                pipeline.addFirst(new LoggingHandler(LogLevel.ERROR));
            }
            if (enableHeartBeating) {
                pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
                pipeline.addLast("heartbeat", new HeartbeatHandler());
            }
            if (enableCompression) {
                pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
                pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            }
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(maxFrameLength, 0, 4, 0, 4));
        }
    }, new BatchedRxEventPipelineConfigurator())).connect().flatMap(new Func1<ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>>, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
            // send subscribe event to server
            connection.writeAndFlush(RemoteRxEvent.subscribed(params.getName(), params.getSubscribeParameters()));
            remoteUnsubscribe.setConnection(connection);
            return connection.getInput().lift(new DropOperator<RemoteRxEvent>("incoming_" + RemoteObservable.class.getCanonicalName() + "_createTcpConnectionToServer"));
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            // connection completed
            logger.warn("Detected connection completed when trying to connect to host: " + params.getHost() + " port: " + params.getPort());
            connectionDisconnectCallback.call();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(Throwable t1) {
            logger.warn("Detected connection error when trying to connect to host: " + params.getHost() + " port: " + params.getPort(), t1);
            connectionDisconnectCallback.call();
            // complete if error occurs
            return Observable.empty();
        }
    }).takeUntil(closeTrigger).map(new Func1<RemoteRxEvent, Notification<T>>() {

        @Override
        public Notification<T> call(RemoteRxEvent rxEvent) {
            if (rxEvent.getType() == RemoteRxEvent.Type.next) {
                metrics.incrementNextCount();
                return Notification.createOnNext(decoder.decode(rxEvent.getData()));
            } else if (rxEvent.getType() == RemoteRxEvent.Type.error) {
                metrics.incrementErrorCount();
                return Notification.createOnError(fromBytesToThrowable(rxEvent.getData()));
            } else if (rxEvent.getType() == RemoteRxEvent.Type.completed) {
                metrics.incrementCompletedCount();
                return Notification.createOnCompleted();
            } else {
                throw new RuntimeException("RemoteRxEvent of type: " + rxEvent.getType() + ", not supported.");
            }
        }
    }).<T>dematerialize().doOnEach(new Observer<T>() {

        @Override
        public void onCompleted() {
            logger.info("RemoteRxEvent: " + params.getName() + " onCompleted()");
        }

        @Override
        public void onError(Throwable e) {
            logger.error("RemoteRxEvent: " + params.getName() + " onError()", e);
        }

        @Override
        public void onNext(T t) {
            if (logger.isDebugEnabled()) {
                logger.debug("RemoteRxEvent: " + params.getName() + " onNext(): " + t);
            }
        }
    });
}

Example 5 with JdkZlibEncoder

use of io.netty.handler.codec.compression.JdkZlibEncoder in project mantis by Netflix.

the class LegacyTcpPushServer method createServer.

@Override
public RxServer<?, ?> createServer() {
    RxServer<RemoteRxEvent, RemoteRxEvent> server = RxNetty.newTcpServerBuilder(port, new ConnectionHandler<RemoteRxEvent, RemoteRxEvent>() {

        @Override
        public Observable<Void> handle(final ObservableConnection<RemoteRxEvent, RemoteRxEvent> newConnection) {
            final InetSocketAddress socketAddress = (InetSocketAddress) newConnection.getChannel().remoteAddress();
            // extract groupId, id, predicate from incoming byte[]
            return newConnection.getInput().flatMap(new Func1<RemoteRxEvent, Observable<Void>>() {

                @Override
                public Observable<Void> call(RemoteRxEvent incomingRequest) {
                    if (incomingRequest.getType() == RemoteRxEvent.Type.subscribed) {
                        Map<String, String> params = incomingRequest.getSubscribeParameters();
                        // client state
                        String id = null;
                        String slotId = null;
                        String groupId = null;
                        // sample state
                        boolean enableSampling = false;
                        long samplingTimeMsec = 0;
                        // predicate state
                        Map<String, List<String>> predicateParams = null;
                        if (params != null && !params.isEmpty()) {
                            predicateParams = new HashMap<String, List<String>>();
                            for (Entry<String, String> entry : params.entrySet()) {
                                List<String> values = new LinkedList<>();
                                values.add(entry.getValue());
                                predicateParams.put(entry.getKey(), values);
                            }
                            if (params.containsKey("id")) {
                                id = params.get("id");
                            }
                            if (params.containsKey("slotId")) {
                                slotId = params.get("slotId");
                            }
                            if (params.containsKey("groupId")) {
                                groupId = params.get("groupId");
                            }
                            if (params.containsKey("sample")) {
                                samplingTimeMsec = Long.parseLong(params.get("sample")) * 1000;
                                if (samplingTimeMsec < 50) {
                                    throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
                                }
                                enableSampling = true;
                            }
                            if (params.containsKey("sampleMSec")) {
                                samplingTimeMsec = Long.parseLong(params.get("sampleMSec"));
                                if (samplingTimeMsec < 50) {
                                    throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
                                }
                                enableSampling = true;
                            }
                        }
                        Func1<T, Boolean> predicateFunction = null;
                        if (predicate != null) {
                            predicateFunction = predicate.call(predicateParams);
                        }
                        // support legacy metrics per connection
                        Metrics sseSinkMetrics = new Metrics.Builder().name("DropOperator_outgoing_subject_" + slotId).addCounter("onNext").addCounter("dropped").build();
                        sseSinkMetrics = metricsRegistry.registerAndGet(sseSinkMetrics);
                        Counter legacyMsgProcessedCounter = sseSinkMetrics.getCounter("onNext");
                        Counter legacyDroppedWrites = sseSinkMetrics.getCounter("dropped");
                        return manageConnection(newConnection, socketAddress.getHostString(), socketAddress.getPort(), groupId, slotId, id, null, false, null, enableSampling, samplingTimeMsec, predicateFunction, null, legacyMsgProcessedCounter, legacyDroppedWrites, null);
                    }
                    return null;
                }
            });
        }
    }).pipelineConfigurator(new PipelineConfiguratorComposite<RemoteRxEvent, RemoteRxEvent>(new PipelineConfigurator<RemoteRxEvent, RemoteRxEvent>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            // pipeline.addLast(new LoggingHandler(LogLevel.ERROR)); // uncomment to enable debug logging
            pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
            pipeline.addLast("heartbeat", new HeartbeatHandler());
            pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
            pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(5242880, 0, 4, 0, 4));
        }
    }, new LegacyTcpPipelineConfigurator(name))).channelOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(1024 * 1024, 5 * 1024 * 1024)).build();
    return server;
}