Examples with LineTcpSender io.questdb.cutlass.line.LineTcpSender used on opensource projects

Example 1 with LineTcpSender

use of io.questdb.cutlass.line.LineTcpSender in project questdb by bluestreak01.

the class LineTCPSenderMain method main.

public static void main(String[] args) {
    final long count = 10_000_000;
    String hostIPv4 = "127.0.0.1";
    // 8089 influx
    int port = 9009;
    int bufferCapacity = 256 * 1024;
    final Rnd rnd = new Rnd();
    long start = System.nanoTime();
    try (LineTcpSender sender = new LineTcpSender(Net.parseIPv4(hostIPv4), port, bufferCapacity)) {
        for (int i = 0; i < count; i++) {
            // if ((i & 0x1) == 0) {
            sender.metric("weather");
            // } else {
            // sender.metric("weather2");
            // }
            sender.tag("location", "london").tag("by", rnd.nextString(5)).field("temp", rnd.nextPositiveLong()).field("ok", rnd.nextPositiveInt()).$(rnd.nextLong(5000000000000L));
        // sender.$();
        }
        sender.flush();
    }
    System.out.println("Actual rate: " + (count * 1_000_000_000L / (System.nanoTime() - start)));
}

Example 2 with LineTcpSender

use of io.questdb.cutlass.line.LineTcpSender in project questdb by bluestreak01.

the class LineTcpReceiverTest method testStringsWithTcpSenderWithNewLineChars.

@Test
public void testStringsWithTcpSenderWithNewLineChars() throws Exception {
    runInContext((receiver) -> {
        send(receiver, "table", WAIT_ENGINE_TABLE_RELEASE, () -> {
            try (LineTcpSender lineTcpSender = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, msgBufferSize)) {
                lineTcpSender.metric("table").tag("tag1", "value 1").tag("tag=2", "значение 2").field("поле=3", "{\"ключ\": \n \"число\", \r\n \"key2\": \"value2\"}\n").$(0);
                lineTcpSender.flush();
            }
        });
        String expected = "tag1\ttag=2\tполе=3\ttimestamp\n" + "value 1\tзначение 2\t{\"ключ\": \n" + " \"число\", \r\n" + " \"key2\": \"value2\"}\n\t1970-01-01T00:00:00.000000Z\n";
        assertTable(expected, "table");
    });
}

Example 3 with LineTcpSender

use of io.questdb.cutlass.line.LineTcpSender in project questdb by bluestreak01.

the class LineTcpReceiverTest method test.

private void test(String authKeyId, PrivateKey authPrivateKey, int msgBufferSize, final int nRows, boolean expectDisconnect) throws Exception {
    this.authKeyId = authKeyId;
    this.msgBufferSize = msgBufferSize;
    assertMemoryLeak(() -> {
        final String[] locations = { "x london", "paris", "rome" };
        final CharSequenceHashSet tables = new CharSequenceHashSet();
        tables.add("weather1");
        tables.add("weather2");
        tables.add("weather3");
        SOCountDownLatch tablesCreated = new SOCountDownLatch();
        tablesCreated.setCount(tables.size());
        final Rnd rand = new Rnd();
        final StringBuilder[] expectedSbs = new StringBuilder[tables.size()];
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN) {
                if (tables.contains(name)) {
                    tablesCreated.countDown();
                }
            }
        });
        minIdleMsBeforeWriterRelease = 100;
        try (LineTcpReceiver ignored = LineTcpReceiver.create(lineConfiguration, sharedWorkerPool, LOG, engine)) {
            long startEpochMs = System.currentTimeMillis();
            sharedWorkerPool.assignCleaner(Path.CLEANER);
            sharedWorkerPool.start(LOG);
            try {
                final AbstractLineSender[] senders = new AbstractLineSender[tables.size()];
                for (int n = 0; n < senders.length; n++) {
                    if (null != authKeyId) {
                        AuthenticatedLineTcpSender sender = new AuthenticatedLineTcpSender(authKeyId, authPrivateKey, Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                        sender.authenticate();
                        senders[n] = sender;
                    } else {
                        senders[n] = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                    }
                    StringBuilder sb = new StringBuilder((nRows + 1) * lineConfiguration.getMaxMeasurementSize());
                    sb.append("location\ttemp\ttimestamp\n");
                    expectedSbs[n] = sb;
                }
                long ts = Os.currentTimeMicros();
                StringSink tsSink = new StringSink();
                for (int nRow = 0; nRow < nRows; nRow++) {
                    int nTable = nRow < tables.size() ? nRow : rand.nextInt(tables.size());
                    AbstractLineSender sender = senders[nTable];
                    StringBuilder sb = expectedSbs[nTable];
                    CharSequence tableName = tables.get(nTable);
                    sender.metric(tableName);
                    String location = locations[rand.nextInt(locations.length)];
                    sb.append(location);
                    sb.append('\t');
                    sender.tag("location", location);
                    int temp = rand.nextInt(100);
                    sb.append(temp);
                    sb.append('\t');
                    sender.field("temp", temp);
                    tsSink.clear();
                    TimestampFormatUtils.appendDateTimeUSec(tsSink, ts);
                    sb.append(tsSink);
                    sb.append('\n');
                    sender.$(ts * 1000);
                    sender.flush();
                    if (expectDisconnect) {
                        // To prevent all data being buffered before the expected disconnect slow sending
                        Os.sleep(100);
                    }
                    ts += rand.nextInt(1000);
                }
                for (int n = 0; n < senders.length; n++) {
                    AbstractLineSender sender = senders[n];
                    sender.close();
                }
                Assert.assertFalse(expectDisconnect);
                boolean ready = tablesCreated.await(TimeUnit.MINUTES.toNanos(1));
                if (!ready) {
                    throw new IllegalStateException("Timeout waiting for tables to be created");
                }
                int nRowsWritten;
                do {
                    nRowsWritten = 0;
                    long timeTakenMs = System.currentTimeMillis() - startEpochMs;
                    if (timeTakenMs > TEST_TIMEOUT_IN_MS) {
                        LOG.error().$("after ").$(timeTakenMs).$("ms tables only had ").$(nRowsWritten).$(" rows out of ").$(nRows).$();
                        break;
                    }
                    Thread.yield();
                    for (int n = 0; n < tables.size(); n++) {
                        CharSequence tableName = tables.get(n);
                        while (true) {
                            try (TableReader reader = engine.getReader(AllowAllCairoSecurityContext.INSTANCE, tableName)) {
                                TableReaderRecordCursor cursor = reader.getCursor();
                                while (cursor.hasNext()) {
                                    nRowsWritten++;
                                }
                                break;
                            } catch (EntryLockedException ex) {
                                LOG.info().$("retrying read for ").$(tableName).$();
                                LockSupport.parkNanos(1);
                            }
                        }
                    }
                } while (nRowsWritten < nRows);
                LOG.info().$(nRowsWritten).$(" rows written").$();
            } finally {
                sharedWorkerPool.halt();
            }
        } finally {
            engine.setPoolListener(null);
        }
        for (int n = 0; n < tables.size(); n++) {
            CharSequence tableName = tables.get(n);
            LOG.info().$("checking table ").$(tableName).$();
            assertTable(expectedSbs[n], tableName);
        }
    });
}

Example 4 with LineTcpSender

use of io.questdb.cutlass.line.LineTcpSender in project questdb by bluestreak01.

the class LineTcpReceiverTest method testTcpSenderWithNewLineCharsInFieldName.

@Test
public void testTcpSenderWithNewLineCharsInFieldName() throws Exception {
    if (engine.getConfiguration().getFilesFacade().isRestrictedFileSystem()) {
        // Windows (NTFS) cannot crate files with new line in file name.
        return;
    }
    runInContext((receiver) -> {
        String tableName = "table";
        send(receiver, tableName, WAIT_ENGINE_TABLE_RELEASE, () -> {
            try (LineTcpSender lineTcpSender = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, msgBufferSize)) {
                lineTcpSender.metric(tableName).tag("tag\n1", "value 1").field("tag\n2", "value 2").$(0);
                lineTcpSender.flush();
            }
        });
        String expected = "tag\n" + "1\ttag\n" + "2\ttimestamp\n" + "value 1\tvalue 2\t1970-01-01T00:00:00.000000Z\n";
        assertTable(expected, tableName);
    });
}

Example 5 with LineTcpSender

use of io.questdb.cutlass.line.LineTcpSender in project questdb by bluestreak01.

the class LineTcpReceiverTest method testNewPartitionRowCancelledTwice.

@Test
public void testNewPartitionRowCancelledTwice() throws Exception {
    runInContext((receiver) -> {
        send(receiver, "table", WAIT_ENGINE_TABLE_RELEASE, () -> {
            try (LineTcpSender lineTcpSender = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, msgBufferSize)) {
                lineTcpSender.metric("table").tag("tag1", "value 1").tag("tag=2", "значение 2").field("поле=3", "{\"ключ\": \"число\"}").$(0);
                lineTcpSender.metric("table").tag("tag1", "value 2").$(0);
                lineTcpSender.metric("table").tag("tag 2", // Invalid column name, last line is not saved
                "value=\2").$(Timestamps.DAY_MICROS * 1000L);
                // Repeat
                lineTcpSender.metric("table").tag("tag 2", // Invalid column name, last line is not saved
                "value=\2").$(Timestamps.DAY_MICROS * 1000L);
                lineTcpSender.flush();
            }
        });
        String expected = "tag1\ttag=2\tполе=3\ttimestamp\n" + "value 1\tзначение 2\t{\"ключ\": \"число\"}\t1970-01-01T00:00:00.000000Z\n" + "value 2\t\t\t1970-01-01T00:00:00.000000Z\n";
        assertTable(expected, "table");
    });
}