Examples with FSTStruct org.nustaq.offheap.structs.FSTStruct used on opensource projects

Example 1 with FSTStruct

use of org.nustaq.offheap.structs.FSTStruct in project fast-cast by RuedigerMoeller.

the class MPPublisher method main.

public static void main(String[] arg) {
    // 5 chars MAX !!
    FastCast.getFastCast().setNodeId("MPUB");
    configureFastCast();
    FCPublisher pub = FastCast.getFastCast().onTransport("default").publish(// unique-per-transport topic id
    new PublisherConf(1).numPacketHistory(// how many packets are kept for retransmission requests
    33_000).pps(// packets per second rate limit.
    10_000));
    MultipleProtocol.initStructFactory();
    FSTStructAllocator onHeapAlloc = new FSTStructAllocator(10_000);
    AMessage aMsg = onHeapAlloc.newStruct(new AMessage());
    OtherMessage other = onHeapAlloc.newStruct(new OtherMessage());
    ComposedMessage composed = onHeapAlloc.newStruct(new ComposedMessage());
    ThreadLocalRandom random = ThreadLocalRandom.current();
    // could directly send raw on publisher
    RateMeasure measure = new RateMeasure("msg/s");
    int count = 0;
    FSTStruct msg = null;
    while (true) {
        measure.count();
        // fill in data
        switch(random.nextInt(3)) {
            case 0:
                for (int i = 0; i < aMsg.stringArrayLen(); i++) {
                    aMsg.stringArray(i).setString("Hello " + i);
                }
                aMsg.setL(count++);
                msg = aMsg;
                break;
            case 1:
                other.setQuantity(count++);
                other.setValue(random.nextDouble());
                msg = other;
                break;
            case 2:
                aMsg.stringArray(0).setString("Hello !");
                aMsg.stringArray(1).setString("Hello !!");
                aMsg.setL(count++);
                other.setQuantity(count++);
                other.setValue(random.nextDouble());
                // does a copy !
                composed.setMegA(aMsg);
                // does a copy !
                composed.setMsgB(other);
                msg = composed;
                break;
        }
        // send message
        while (!pub.offer(null, msg.getBase(), msg.getOffset(), msg.getByteSize(), false)) {
        /* spin */
        }
    }
}

Example 2 with FSTStruct

use of org.nustaq.offheap.structs.FSTStruct in project fast-cast by RuedigerMoeller.

the class AsyncPubStruct method initFastCast.

public void initFastCast() throws Exception {
    fastCast = FastCast.getFastCast();
    fastCast.setNodeId("PUB");
    fastCast.loadConfig(CFG_FILE_PATH);
    pub = fastCast.onTransport("default").publish("stream");
    // pointer to message
    final FSTStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(FSTStruct.class);
    fastCast.onTransport("back").subscribe("back", new FCSubscriber() {

        @Override
        public void messageReceived(String sender, long sequence, Bytez b, long off, int len) {
            // as structs decode literally in zero time, we can decode inside receiver thread
            msg.baseOn(b, (int) off);
            Class type = msg.getPointedClass();
            if (type == StructString.class) {
                // sequence end, print histogram
                final Histogram oldHi = hi;
                hi = new Histogram(TimeUnit.SECONDS.toNanos(2), 3);
                // no lambdas to stay 1.7 compatible
                // move printing out of the receiving thread
                dumper.execute(new Runnable() {

                    @Override
                    public void run() {
                        oldHi.outputPercentileDistribution(System.out, 1000.0);
                    }
                });
            } else {
                // a regular message, record latency
                AsyncLatMessageStruct mdata = msg.cast();
                long value = System.nanoTime() - mdata.getSendTimeStampNanos();
                if (value < 1_000_000_000) {
                    // avoid AIOB during init + JITTING
                    hi.recordValue(value);
                }
            // a real app would need to copy the message (recycle byte arrays/objects !) and
            // run msg processing in an executor to get out of the receiving thread.
            // mdata gets invalid on finish of this method
            }
        }

        @Override
        public boolean dropped() {
            System.out.println("DROPPED");
            System.exit(-1);
            return false;
        }

        @Override
        public void senderTerminated(String senderNodeId) {
        }

        @Override
        public void senderBootstrapped(String receivesFrom, long seqNo) {
        }
    });
}

Example 3 with FSTStruct

use of org.nustaq.offheap.structs.FSTStruct in project fast-cast by RuedigerMoeller.

the class AsyncSubStruct method initFastCast.

public void initFastCast() throws Exception {
    fastCast = FastCast.getFastCast();
    fastCast.setNodeId("SUB");
    fastCast.loadConfig(AsyncLatPublisher.CFG_FILE_PATH);
    backPub = fastCast.onTransport("back").publish("back");
    // pointer to message
    final FSTStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(FSTStruct.class);
    final RateMeasure measure = new RateMeasure("receive rate");
    fastCast.onTransport("default").subscribe("stream", new FCSubscriber() {

        int count = 0;

        @Override
        public void messageReceived(String sender, long sequence, Bytez b, long off, final int len) {
            measure.count();
            if (++count == 10 || len < 30) /*"END" marker*/
            {
                count = 0;
                final byte[] copy = pool.getBA();
                if (// prevent segfault :) !
                len < copy.length) {
                    b.getArr(off, copy, 0, len);
                    count = 0;
                    // do bounce back in different thread, else blocking on send will pressure back to
                    // sender resulting in whacky behaviour+throughput
                    bounceBackExec.execute(new Runnable() {

                        @Override
                        public void run() {
                            while (!backPub.offer(null, copy, 0, len, true)) {
                            // spin
                            }
                            // give back to pool
                            pool.returnBA(copy);
                        }
                    });
                } else {
                    throw new RuntimeException("was soll das ?");
                }
            }
        }

        @Override
        public boolean dropped() {
            // fatal, exit
            System.out.println("process dropped ");
            System.exit(-1);
            return false;
        }

        @Override
        public void senderTerminated(String senderNodeId) {
        }

        @Override
        public void senderBootstrapped(String receivesFrom, long seqNo) {
        }
    });
}

Example 4 with FSTStruct

use of org.nustaq.offheap.structs.FSTStruct in project fast-cast by RuedigerMoeller.

the class MPSubscriber method main.

public static void main(String[] arg) {
    MultipleProtocol.initStructFactory();
    // 5 chars MAX !!
    FastCast.getFastCast().setNodeId("MSUB");
    MPPublisher.configureFastCast();
    final RateMeasure rateMeasure = new RateMeasure("receive rate");
    FastCast.getFastCast().onTransport("default").subscribe(new SubscriberConf(1).receiveBufferPackets(33_000), new FCSubscriber() {

        FSTStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(FSTStruct.class);

        int count = 0;

        @Override
        public void messageReceived(String sender, long sequence, Bytez b, long off, int len) {
            rateMeasure.count();
            msg.baseOn(b, (int) off);
            Class type = msg.getPointedClass();
            if (type == AMessage.class) {
                AMessage am = msg.cast();
            // am is valid until another pointer is used. use "am = am.detach" in order to get a non-cached pointer
            // note msg data is valid only inside this method (reused). You need to copy data in case processing should 
            // be done in a different thread (usually do basic filtering in-thread, then do processing on a dedicated thread)
            } else if (type == OtherMessage.class) {
                OtherMessage om = msg.cast();
            // ..
            } else if (type == ComposedMessage.class) {
                ComposedMessage cm = msg.cast();
                if (count++ == 500_000) {
                    System.out.println("Other" + cm);
                    count = 0;
                }
            }
        }

        @Override
        public boolean dropped() {
            System.out.println("fatal, could not keep up. exiting");
            System.exit(0);
            return false;
        }

        @Override
        public void senderTerminated(String senderNodeId) {
            System.out.println("sender died " + senderNodeId);
        }

        @Override
        public void senderBootstrapped(String receivesFrom, long seqNo) {
            System.out.println("bootstrap " + receivesFrom);
        }
    });
}