Examples with BaseFlowableOperator com.dexels.navajo.document.stream.io.BaseFlowableOperator used on opensource projects

Example 1 with BaseFlowableOperator

use of com.dexels.navajo.document.stream.io.BaseFlowableOperator in project navajo by Dexels.

the class StreamCompress method genericCompress2.

// public static FlowableOperator<Flowable<byte[]>, byte[]> genericCompress(String type, boolean debug) {
// final int COMPRESSION_BUFFER_SIZE = 16384;
// 
// Deflater deflater = type.equals("gzip") ? new Deflater(Deflater.DEFAULT_COMPRESSION, true) : new Deflater();
// 
// return new BaseFlowableOperator<Flowable<byte[]>, byte[]>(10) {
// 
// @Override
// public Subscriber<? super byte[]> apply(Subscriber<? super Flowable<byte[]>> child) throws Exception {
// 
// final CRC32 crc = new CRC32();
// AtomicInteger totalIn = new AtomicInteger();
// 
// return new Subscriber<byte[]>() {
// 
// private OutputStream debugDump = null;
// @Override
// public void onComplete() {
// 
// deflater.finish();
// 
// byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
// Iterable<byte[]> output = new Iterable<byte[]>(){
// 
// @Override
// public Iterator<byte[]> iterator() {
// return new Iterator<byte[]>() {
// int read;
// boolean first = true;
// @Override
// public boolean hasNext() {
// read = deflater.deflate(buffer,0,buffer.length,FLUSH_MODE);
// boolean needsInput = deflater.needsInput();
// boolean hasMore = (first || needsInput) && read > 0;
// first = false;
// return hasMore;
// }
// 
// @Override
// public byte[] next() {
// return Arrays.copyOfRange(buffer, 0, read);
// }
// };
// }
// };
// 
// List<byte[]> items = new ArrayList<>();
// output.forEach(items::add);
// 
// byte[] suffix = writeTrailer((int) crc.getValue(), deflater.getTotalIn());
// if("gzip".equals(type)) {
// queue.offer(Flowable.fromIterable(items).concatWith(Flowable.just(suffix)));
// } else {
// queue.offer(Flowable.fromIterable(items));
// }
// operatorComplete(child);
// }
// 
// @Override
// public void onError(Throwable t) {
// if(debug) {
// try {
// debugDump.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// 
// operatorError(t, child);
// }
// 
// @Override
// public void onSubscribe(Subscription s) {
// operatorSubscribe(s, child);
// if("gzip".equals(type)) {
// byte[] hdr = writeHeader();
// totalIn.addAndGet(hdr.length);
// offer(Flowable.just(hdr));
// }
// }
// 
// @Override
// public void onNext(byte[] dataIn) {
// crc.update(dataIn);
// totalIn.addAndGet(dataIn.length);
// deflater.setInput(dataIn);
// byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
// Iterator<byte[]> it =  new Iterator<byte[]>() {
// int read;
// boolean first = true;
// @Override
// public boolean hasNext() {
// read = deflater.deflate(buffer,0,buffer.length,FLUSH_MODE);
// boolean needsInput = deflater.needsInput();
// boolean hasMore = (first || needsInput) && read > 0;
// first = false;
// return hasMore;
// }
// 
// @Override
// public byte[] next() {
// byte[] compressedData = Arrays.copyOfRange(buffer, 0, read);
// return compressedData;
// }
// };
// List<byte[]> result = new ArrayList<byte[]>();
// it.forEachRemaining(result::add);
// queue.offer(Flowable.fromIterable(result));
// drain(child);
// }
// };
// }
// };
// }
// 
private static FlowableOperator<Flowable<byte[]>, byte[]> genericCompress2(CompressionType type, boolean debug) {
    final int COMPRESSION_BUFFER_SIZE = 4096;
    Deflater deflater = type == CompressionType.GZIP ? new Deflater(Deflater.DEFAULT_COMPRESSION, true) : new Deflater();
    return new BaseFlowableOperator<Flowable<byte[]>, byte[]>(10) {

        @Override
        public Subscriber<? super byte[]> apply(Subscriber<? super Flowable<byte[]>> child) throws Exception {
            final CRC32 crc = new CRC32();
            AtomicInteger totalIn = new AtomicInteger();
            return new Subscriber<byte[]>() {

                private OutputStream debugDump = null;

                @Override
                public void onComplete() {
                    deflater.finish();
                    byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
                    List<byte[]> output = new ArrayList<>();
                    int read = 0;
                    do {
                        // read = deflater.deflate(buffer,0,buffer.length,Deflater.FULL_FLUSH);
                        read = deflater.deflate(buffer);
                        byte[] compressedData = Arrays.copyOfRange(buffer, 0, read);
                        output.add(compressedData);
                    } while (read > 0);
                    byte[] suffix = writeTrailer((int) crc.getValue(), deflater.getTotalIn());
                    if (type == CompressionType.GZIP) {
                        queue.offer(Flowable.fromIterable(output).concatWith(Flowable.just(suffix)));
                    } else {
                        queue.offer(Flowable.fromIterable(output));
                    }
                    operatorComplete(child);
                }

                @Override
                public void onError(Throwable t) {
                    if (debug) {
                        try {
                            debugDump.close();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }
                    operatorError(t, child);
                }

                @Override
                public void onSubscribe(Subscription s) {
                    operatorSubscribe(s, child);
                    if (type == CompressionType.GZIP) {
                        byte[] hdr = writeHeader();
                        totalIn.addAndGet(hdr.length);
                        offer(Flowable.just(hdr));
                    }
                }

                @Override
                public void onNext(byte[] dataIn) {
                    crc.update(dataIn);
                    totalIn.addAndGet(dataIn.length);
                    deflater.setInput(dataIn);
                    List<byte[]> output = new ArrayList<>();
                    byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
                    int read = 0;
                    do {
                        read = deflater.deflate(buffer);
                        byte[] compressedData = Arrays.copyOfRange(buffer, 0, read);
                        output.add(compressedData);
                    } while (read > 0);
                    queue.offer(Flowable.fromIterable(output));
                    drain(child);
                }
            };
        }
    };
}

Example 2 with BaseFlowableOperator

use of com.dexels.navajo.document.stream.io.BaseFlowableOperator in project navajo by Dexels.

the class StreamDocument method gatherBinary.

public static FlowableOperator<Binary, String> gatherBinary() {
    return new BaseFlowableOperator<Binary, String>(1) {

        @Override
        public Subscriber<? super String> apply(Subscriber<? super Binary> child) throws Exception {
            return new Subscriber<String>() {

                Binary result = null;

                @Override
                public void onComplete() {
                    if (result != null) {
                        try {
                            result.finishPushContent();
                            operatorNext("", r -> result, child);
                        } catch (IOException e) {
                            operatorError(e, child);
                        }
                    }
                    operatorComplete(child);
                }

                @Override
                public void onError(Throwable t) {
                    operatorError(t, child);
                }

                @Override
                public void onNext(String e) {
                    try {
                        result.pushContent(e);
                        operatorRequest(1);
                    } catch (IOException ex) {
                        operatorError(ex, child);
                    }
                }

                @Override
                public void onSubscribe(Subscription s) {
                    operatorSubscribe(s, child);
                    result = createBinary();
                }

                private Binary createBinary() {
                    Binary result = new Binary();
                    try {
                        result.startPushRead();
                    } catch (IOException e1) {
                        logger.error("Error: ", e1);
                    }
                    return result;
                }
            };
        }
    };
}

Example 3 with BaseFlowableOperator

use of com.dexels.navajo.document.stream.io.BaseFlowableOperator in project navajo by Dexels.

the class StreamDocument method messageWithPath.

public static FlowableOperator<NavajoStreamEvent, NavajoStreamEvent> messageWithPath(final String messagePath, final Function<Msg, Msg> operation, boolean filterOthers) {
    return new BaseFlowableOperator<NavajoStreamEvent, NavajoStreamEvent>(1) {

        @Override
        public Subscriber<? super NavajoStreamEvent> apply(Subscriber<? super NavajoStreamEvent> child) throws Exception {
            return new Subscriber<NavajoStreamEvent>() {

                private final Stack<String> pathStack = new Stack<>();

                @Override
                public void onComplete() {
                    operatorComplete(child);
                }

                @Override
                public void onError(Throwable e) {
                    operatorError(e, child);
                }

                @Override
                public void onNext(NavajoStreamEvent event) {
                    switch(event.type()) {
                        case MESSAGE_STARTED:
                            pathStack.push(event.path());
                            // operatorRequest(1);
                            break;
                        case ARRAY_ELEMENT_STARTED:
                            // operatorRequest(1);
                            break;
                        case MESSAGE:
                            if (matches(messagePath, pathStack)) {
                                Msg transformed;
                                try {
                                    transformed = operation.apply((Msg) event.body());
                                    operatorNext(event, e -> {
                                        return Events.message(transformed, event.path(), event.attributes());
                                    }, child);
                                } catch (Exception e1) {
                                    logger.error("Unexpected error: ", e1);
                                }
                                return;
                            }
                            pathStack.pop();
                            break;
                        case ARRAY_ELEMENT:
                            if (matches(messagePath, pathStack)) {
                                Msg transformed;
                                try {
                                    transformed = operation.apply((Msg) event.body());
                                    operatorNext(event, e -> {
                                        return Events.arrayElement(transformed, event.attributes());
                                    }, child);
                                } catch (Exception e1) {
                                    logger.error("Very unexpected exception: ", e1);
                                    e1.printStackTrace();
                                }
                                return;
                            }
                            break;
                        // TODO Support these?
                        case ARRAY_STARTED:
                            pathStack.push(event.path());
                            break;
                        case ARRAY_DONE:
                            pathStack.pop();
                            break;
                        default:
                            break;
                    }
                    if (!filterOthers) {
                        operatorNext(event, e -> e, child);
                    } else {
                        operatorRequest(1);
                    }
                }

                private boolean matches(String path, Stack<String> pathStack) {
                    String joined = String.join("/", pathStack);
                    return path.equals(joined);
                }

                @Override
                public void onSubscribe(Subscription s) {
                    operatorSubscribe(s, child);
                }
            };
        }
    };
}

Example 4 with BaseFlowableOperator

use of com.dexels.navajo.document.stream.io.BaseFlowableOperator in project navajo by Dexels.

the class XML method parseFlowable.

public static FlowableOperator<Flowable<XMLEvent>, byte[]> parseFlowable(int queueSize) {
    return new BaseFlowableOperator<Flowable<XMLEvent>, byte[]>(queueSize) {

        @Override
        public Subscriber<? super byte[]> apply(Subscriber<? super Flowable<XMLEvent>> child) throws Exception {
            return new Subscriber<byte[]>() {

                private final SaxXmlFeeder feeder = new SaxXmlFeeder();

                @Override
                public void onError(Throwable t) {
                    error = t;
                    done = true;
                    drain(child);
                }

                @Override
                public void onComplete() {
                    feeder.endOfInput();
                    done = true;
                    drain(child);
                }

                @Override
                public void onNext(byte[] buffer) {
                    Flowable<XMLEvent> fromIterable;
                    try {
                        List<XMLEvent> x = StreamSupport.stream(feeder.parse(buffer).spliterator(), false).filter(elt -> elt != null).collect(Collectors.toList());
                        fromIterable = Flowable.fromIterable(x);
                        queue.offer(fromIterable);
                        drain(child);
                    } catch (XMLStreamException e) {
                        child.onError(e);
                        return;
                    }
                }

                @Override
                public void onSubscribe(Subscription s) {
                    subscription = s;
                    child.onSubscribe(new Subscription() {

                        @Override
                        public void request(long n) {
                            BackpressureHelper.add(requested, n);
                            drain(child);
                        }

                        @Override
                        public void cancel() {
                            cancelled = true;
                            s.cancel();
                        }
                    });
                    s.request(1);
                }
            };
        }
    };
}

Example 5 with BaseFlowableOperator

use of com.dexels.navajo.document.stream.io.BaseFlowableOperator in project navajo by Dexels.

the class StreamCompress method genericDecompress2.

private static FlowableOperator<Flowable<byte[]>, byte[]> genericDecompress2(CompressionType type, boolean debug) {
    final int COMPRESSION_BUFFER_SIZE = 4096;
    Inflater inflater = type == CompressionType.GZIP ? new Inflater(true) : new Inflater();
    CRC32 crc = new CRC32();
    AtomicBoolean first = new AtomicBoolean(true);
    return new BaseFlowableOperator<Flowable<byte[]>, byte[]>(1) {

        @Override
        public Subscriber<? super byte[]> apply(Subscriber<? super Flowable<byte[]>> child) throws Exception {
            return new Subscriber<byte[]>() {

                private OutputStream debugDump = null;

                private final List<byte[]> stashedData = new ArrayList<>();

                @Override
                public void onComplete() {
                    // inflater.finish();
                    byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
                    List<byte[]> output = new ArrayList<>();
                    int read = 0;
                    try {
                        do {
                            read = inflater.inflate(buffer);
                            byte[] compressedData = Arrays.copyOfRange(buffer, 0, read);
                            output.add(compressedData);
                        } while (read > 0);
                    } catch (DataFormatException e) {
                        e.printStackTrace();
                    }
                    // byte[] suffix = writeTrailer((int) crc.getValue(), deflater.getTotalIn());
                    // if(type ==CompressionType.GZIP) {
                    // queue.offer(Flowable.fromIterable(output).concatWith(Flowable.just(suffix)));
                    // } else {
                    queue.offer(Flowable.fromIterable(output));
                    // }
                    operatorComplete(child);
                }

                @Override
                public void onError(Throwable t) {
                    if (debug) {
                        try {
                            debugDump.close();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }
                    operatorError(t, child);
                }

                @Override
                public void onSubscribe(Subscription s) {
                    operatorSubscribe(s, child);
                    if (type == CompressionType.GZIP) {
                    // byte[] hdr = writeHeader();
                    // totalIn.addAndGet(hdr.length);
                    // offer(Flowable.just(hdr));
                    }
                }

                @Override
                public void onNext(byte[] data) {
                    if (first.get() && CompressionType.GZIP == type) {
                        byte[] joinedData;
                        if (!stashedData.isEmpty()) {
                            int totalSize = stashedData.stream().map(b -> b.length).collect(Collectors.summingInt(i -> i));
                            joinedData = new byte[totalSize];
                            int count = 0;
                            for (byte[] e : stashedData) {
                                System.arraycopy(e, 0, joinedData, count, e.length);
                                count += e.length;
                            }
                        } else {
                            joinedData = data;
                        }
                        ByteArrayInputStream bais = new ByteArrayInputStream(joinedData);
                        int offset = 0;
                        try {
                            offset = readHeader(bais, crc);
                            first.set(false);
                            if (joinedData.length - offset > 0) {
                                inflater.setInput(joinedData, offset, joinedData.length - offset);
                            } else {
                                operatorRequest(1);
                                return;
                            }
                        } catch (EOFException e2) {
                            // stashedData.add(data);
                            e2.printStackTrace();
                            operatorError(e2, child);
                            // operatorRequest(1);
                            return;
                        } catch (IOException e1) {
                            e1.printStackTrace();
                            operatorError(e1, child);
                        }
                    } else {
                        inflater.setInput(data);
                    }
                    try {
                        // inflater.setInput(data);
                        List<byte[]> output = new ArrayList<>();
                        byte[] buffer = new byte[COMPRESSION_BUFFER_SIZE];
                        int read = 0;
                        do {
                            read = inflater.inflate(buffer);
                            byte[] compressedData = Arrays.copyOfRange(buffer, 0, read);
                            output.add(compressedData);
                        } while (read > 0);
                        queue.offer(Flowable.fromIterable(output));
                        drain(child);
                    } catch (DataFormatException e) {
                        e.printStackTrace();
                        onError(e);
                    }
                }
            };
        }
    };
}