Examples with Bytes net.openhft.chronicle.bytes.Bytes used on opensource projects

Example 46 with Bytes

use of net.openhft.chronicle.bytes.Bytes in project Chronicle-Queue by OpenHFT.

the class ChronicleReader method execute.

public void execute() {
    long lastObservedTailIndex;
    long highestReachedIndex = 0L;
    boolean isFirstIteration = true;
    boolean retryLastOperation;
    boolean queueHasBeenModified;
    final AtomicLong matchCounter = new AtomicLong(0L);
    do {
        try (final ChronicleQueue queue = createQueue();
            final QueueEntryHandler messageConverter = entryHandlerFactory.get()) {
            final ExcerptTailer tailer = queue.createTailer();
            MessageHistory.set(new VanillaMessageHistory());
            tlTailer = ThreadLocal.withInitial(queue::createTailer);
            do {
                if (highestReachedIndex != 0L) {
                    tailer.moveToIndex(highestReachedIndex);
                }
                final Bytes textConversionTarget = Bytes.elasticByteBuffer();
                try {
                    moveToSpecifiedPosition(queue, tailer, isFirstIteration);
                    lastObservedTailIndex = tailer.index();
                    Consumer<String> messageConsumer = text -> applyFiltersAndLog(text, tailer.index(), matchCounter);
                    BooleanSupplier readOne;
                    if (methodReaderInterface == null) {
                        readOne = () -> readOne(messageConverter, tailer, messageConsumer);
                    } else {
                        // TODO: consider unifying this with messageConverter
                        Bytes<ByteBuffer> bytes = Bytes.elasticHeapByteBuffer(256);
                        Wire wire = wireType.apply(bytes);
                        if (wire instanceof TextWire)
                            ((TextWire) wire).useTextDocuments();
                        MethodWriterBuilder<?> mwb = wire.methodWriterBuilder(methodReaderInterface);
                        if (showMessageHistory)
                            mwb.updateInterceptor((methodName, t) -> {
                                MessageHistory messageHistory = MessageHistory.get();
                                // this is an attempt to recognise that no MH was read and instead the method reader called reset(...) on it
                                if (messageHistory.sources() != 1 || messageHistory.timings() != 1)
                                    bytes.append(messageHistory + System.lineSeparator());
                                return true;
                            });
                        MethodReader methodReader = tailer.methodReader(mwb.build());
                        readOne = () -> {
                            boolean found = methodReader.readOne();
                            if (found)
                                messageConsumer.accept(bytes.toString());
                            bytes.clear();
                            return found;
                        };
                    }
                    while (!Thread.currentThread().isInterrupted()) {
                        boolean found = readOne.getAsBoolean();
                        if (!found) {
                            if (tailInputSource) {
                                pauser.pause();
                            }
                            break;
                        } else {
                            if (matchLimitReached(matchCounter.get())) {
                                break;
                            }
                        }
                        pauser.reset();
                    }
                } finally {
                    textConversionTarget.releaseLast();
                    highestReachedIndex = tailer.index();
                    isFirstIteration = false;
                }
                queueHasBeenModified = queueHasBeenModifiedSinceLastCheck(lastObservedTailIndex);
                retryLastOperation = false;
                if (!running || matchLimitReached(matchCounter.get()))
                    return;
            } while (tailerDirection != BACKWARD && (tailInputSource || queueHasBeenModified));
        } catch (final RuntimeException e) {
            if (e.getCause() != null && e.getCause() instanceof DateTimeParseException) {
                // ignore this error - due to a race condition between
                // the reader creating a Queue (with default roll-cycle due to no files on disk)
                // and the writer appending to the Queue with a non-default roll-cycle
                retryLastOperation = true;
            } else {
                throw e;
            }
        }
    } while (retryLastOperation);
}

Example 47 with Bytes

use of net.openhft.chronicle.bytes.Bytes in project Chronicle-Queue by OpenHFT.

the class ChronicleAppenderCycleTest method testAppenderCycle.

@Test
public void testAppenderCycle() throws IOException {
    String id = "testAppenderCycle";
    Bytes msg = Bytes.allocateDirect(64);
    try {
        int n = 20;
        for (int i = 0; i < n; ++i) runTest(id + '-' + i, msg);
    } finally {
        msg.releaseLast();
    }
}

Example 48 with Bytes

use of net.openhft.chronicle.bytes.Bytes in project Chronicle-Queue by OpenHFT.

the class ChronicleQueueTwoThreadsTest method doTest.

void doTest(boolean buffered) throws InterruptedException {
    File name = getTmpDir();
    AtomicLong counter = new AtomicLong();
    Thread tailerThread = new Thread(() -> {
        AffinityLock rlock = AffinityLock.acquireLock();
        Bytes bytes = NativeBytes.nativeBytes(BYTES_LENGTH).unchecked(true);
        try (ChronicleQueue rqueue = SingleChronicleQueueBuilder.fieldlessBinary(name).testBlockSize().build()) {
            ExcerptTailer tailer = rqueue.createTailer();
            while (!Thread.interrupted()) {
                bytes.clear();
                if (tailer.readBytes(bytes)) {
                    counter.incrementAndGet();
                }
            }
        } finally {
            if (rlock != null) {
                rlock.release();
            }
        // System.out.printf("Read %,d messages", counter.intValue());
        }
    }, "tailer thread");
    long runs = 50_000;
    Thread appenderThread = new Thread(() -> {
        AffinityLock wlock = AffinityLock.acquireLock();
        try {
            ChronicleQueue wqueue = SingleChronicleQueueBuilder.fieldlessBinary(name).rollCycle(SMALL_DAILY).testBlockSize().writeBufferMode(buffered ? BufferMode.Asynchronous : BufferMode.None).build();
            ExcerptAppender appender = wqueue.acquireAppender();
            Bytes bytes = Bytes.allocateDirect(BYTES_LENGTH).unchecked(true);
            long next = System.nanoTime() + INTERVAL_US * 1000;
            for (int i = 0; i < runs; i++) {
                while (System.nanoTime() < next) /* busy wait*/
                ;
                long start = next;
                bytes.readPositionRemaining(0, BYTES_LENGTH);
                bytes.writeLong(0L, start);
                appender.writeBytes(bytes);
                next += INTERVAL_US * 1000;
            }
            wqueue.close();
        } finally {
            if (wlock != null) {
                wlock.release();
            }
        }
    }, "appender thread");
    tailerThread.start();
    Jvm.pause(100);
    appenderThread.start();
    appenderThread.join();
    // Pause to allow tailer to catch up (if needed)
    for (int i = 0; i < 10; i++) {
        if (runs != counter.get())
            Jvm.pause(Jvm.isDebug() ? 10000 : 100);
    }
    for (int i = 0; i < 10; i++) {
        tailerThread.interrupt();
        tailerThread.join(100);
    }
    assertEquals(runs, counter.get());
}

Example 49 with Bytes

use of net.openhft.chronicle.bytes.Bytes in project Chronicle-Queue by OpenHFT.

the class MethodReaderObjectReuseTest method testOneOne.

@Test
public void testOneOne() {
    ClassAliasPool.CLASS_ALIASES.addAlias(PingDTO.class);
    try (ChronicleQueue cq = SingleChronicleQueueBuilder.single(OS.getTarget() + "/MethodReaderObjectReuseTest-" + Time.uniqueId()).build()) {
        PingDTO.constructionExpected++;
        PingDTO pdtio = new PingDTO();
        PingDTO.constructionExpected++;
        Pinger pinger = cq.acquireAppender().methodWriter(Pinger.class);
        for (int i = 0; i < 5; i++) {
            pinger.ping(pdtio);
            assertEquals(PingDTO.constructionExpected, PingDTO.constructionCounter);
            pdtio.bytes.append("hi");
        }
        StringBuilder sb = new StringBuilder();
        PingDTO.constructionExpected++;
        MethodReader reader = cq.createTailer().methodReader((Pinger) pingDTO -> sb.append("ping ").append(pingDTO));
        while (reader.readOne()) ;
        // moved this assert below the readOne as object may be constructed lazily
        assertEquals(PingDTO.constructionExpected, PingDTO.constructionCounter);
        assertEquals("ping !PingDTO {\n" + "  bytes: \"\"\n" + "}\n" + "ping !PingDTO {\n" + "  bytes: hi\n" + "}\n" + "ping !PingDTO {\n" + "  bytes: hihi\n" + "}\n" + "ping !PingDTO {\n" + "  bytes: hihihi\n" + "}\n" + "ping !PingDTO {\n" + "  bytes: hihihihi\n" + "}\n", sb.toString());
    }
}

Example 50 with Bytes

use of net.openhft.chronicle.bytes.Bytes in project cassandra by apache.

the class Dump method dump.

public static void dump(List<String> arguments, String rollCycle, boolean follow) {
    StringBuilder sb = new StringBuilder();
    ReadMarshallable reader = wireIn -> {
        sb.setLength(0);
        int version = wireIn.read(BinLog.VERSION).int16();
        if (version > FullQueryLogger.CURRENT_VERSION) {
            throw new IORuntimeException("Unsupported record version [" + version + "] - highest supported version is [" + FullQueryLogger.CURRENT_VERSION + ']');
        }
        String type = wireIn.read(BinLog.TYPE).text();
        if (!FullQueryLogger.SINGLE_QUERY.equals((type)) && !FullQueryLogger.BATCH.equals((type))) {
            throw new IORuntimeException("Unsupported record type field [" + type + "] - supported record types are [" + FullQueryLogger.SINGLE_QUERY + ", " + FullQueryLogger.BATCH + ']');
        }
        sb.append("Type: ").append(type).append(System.lineSeparator());
        long queryStartTime = wireIn.read(FullQueryLogger.QUERY_START_TIME).int64();
        sb.append("Query start time: ").append(queryStartTime).append(System.lineSeparator());
        int protocolVersion = wireIn.read(FullQueryLogger.PROTOCOL_VERSION).int32();
        sb.append("Protocol version: ").append(protocolVersion).append(System.lineSeparator());
        QueryOptions options = QueryOptions.codec.decode(Unpooled.wrappedBuffer(wireIn.read(FullQueryLogger.QUERY_OPTIONS).bytes()), ProtocolVersion.decode(protocolVersion, true));
        long generatedTimestamp = wireIn.read(FullQueryLogger.GENERATED_TIMESTAMP).int64();
        sb.append("Generated timestamp:").append(generatedTimestamp).append(System.lineSeparator());
        int generatedNowInSeconds = wireIn.read(FullQueryLogger.GENERATED_NOW_IN_SECONDS).int32();
        sb.append("Generated nowInSeconds:").append(generatedNowInSeconds).append(System.lineSeparator());
        switch(type) {
            case (FullQueryLogger.SINGLE_QUERY):
                dumpQuery(options, wireIn, sb);
                break;
            case (FullQueryLogger.BATCH):
                dumpBatch(options, wireIn, sb);
                break;
            default:
                throw new IORuntimeException("Log entry of unsupported type " + type);
        }
        System.out.print(sb.toString());
        System.out.flush();
    };
    // Backoff strategy for spinning on the queue, not aggressive at all as this doesn't need to be low latency
    Pauser pauser = Pauser.millis(100);
    List<ChronicleQueue> queues = arguments.stream().distinct().map(path -> SingleChronicleQueueBuilder.single(new File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build()).collect(Collectors.toList());
    List<ExcerptTailer> tailers = queues.stream().map(ChronicleQueue::createTailer).collect(Collectors.toList());
    boolean hadWork = true;
    while (hadWork) {
        hadWork = false;
        for (ExcerptTailer tailer : tailers) {
            while (tailer.readDocument(reader)) {
                hadWork = true;
            }
        }
        if (follow) {
            if (!hadWork) {
                // Chronicle queue doesn't support blocking so use this backoff strategy
                pauser.pause();
            }
            // Don't terminate the loop even if there wasn't work
            hadWork = true;
        }
    }
}