Examples with AbstractLineSender io.questdb.cutlass.line.AbstractLineSender used on opensource projects

Example 1 with AbstractLineSender

use of io.questdb.cutlass.line.AbstractLineSender in project questdb by bluestreak01.

the class LineTcpReceiverTest method test.

private void test(String authKeyId, PrivateKey authPrivateKey, int msgBufferSize, final int nRows, boolean expectDisconnect) throws Exception {
    this.authKeyId = authKeyId;
    this.msgBufferSize = msgBufferSize;
    assertMemoryLeak(() -> {
        final String[] locations = { "x london", "paris", "rome" };
        final CharSequenceHashSet tables = new CharSequenceHashSet();
        tables.add("weather1");
        tables.add("weather2");
        tables.add("weather3");
        SOCountDownLatch tablesCreated = new SOCountDownLatch();
        tablesCreated.setCount(tables.size());
        final Rnd rand = new Rnd();
        final StringBuilder[] expectedSbs = new StringBuilder[tables.size()];
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN) {
                if (tables.contains(name)) {
                    tablesCreated.countDown();
                }
            }
        });
        minIdleMsBeforeWriterRelease = 100;
        try (LineTcpReceiver ignored = LineTcpReceiver.create(lineConfiguration, sharedWorkerPool, LOG, engine)) {
            long startEpochMs = System.currentTimeMillis();
            sharedWorkerPool.assignCleaner(Path.CLEANER);
            sharedWorkerPool.start(LOG);
            try {
                final AbstractLineSender[] senders = new AbstractLineSender[tables.size()];
                for (int n = 0; n < senders.length; n++) {
                    if (null != authKeyId) {
                        AuthenticatedLineTcpSender sender = new AuthenticatedLineTcpSender(authKeyId, authPrivateKey, Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                        sender.authenticate();
                        senders[n] = sender;
                    } else {
                        senders[n] = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                    }
                    StringBuilder sb = new StringBuilder((nRows + 1) * lineConfiguration.getMaxMeasurementSize());
                    sb.append("location\ttemp\ttimestamp\n");
                    expectedSbs[n] = sb;
                }
                long ts = Os.currentTimeMicros();
                StringSink tsSink = new StringSink();
                for (int nRow = 0; nRow < nRows; nRow++) {
                    int nTable = nRow < tables.size() ? nRow : rand.nextInt(tables.size());
                    AbstractLineSender sender = senders[nTable];
                    StringBuilder sb = expectedSbs[nTable];
                    CharSequence tableName = tables.get(nTable);
                    sender.metric(tableName);
                    String location = locations[rand.nextInt(locations.length)];
                    sb.append(location);
                    sb.append('\t');
                    sender.tag("location", location);
                    int temp = rand.nextInt(100);
                    sb.append(temp);
                    sb.append('\t');
                    sender.field("temp", temp);
                    tsSink.clear();
                    TimestampFormatUtils.appendDateTimeUSec(tsSink, ts);
                    sb.append(tsSink);
                    sb.append('\n');
                    sender.$(ts * 1000);
                    sender.flush();
                    if (expectDisconnect) {
                        // To prevent all data being buffered before the expected disconnect slow sending
                        Os.sleep(100);
                    }
                    ts += rand.nextInt(1000);
                }
                for (int n = 0; n < senders.length; n++) {
                    AbstractLineSender sender = senders[n];
                    sender.close();
                }
                Assert.assertFalse(expectDisconnect);
                boolean ready = tablesCreated.await(TimeUnit.MINUTES.toNanos(1));
                if (!ready) {
                    throw new IllegalStateException("Timeout waiting for tables to be created");
                }
                int nRowsWritten;
                do {
                    nRowsWritten = 0;
                    long timeTakenMs = System.currentTimeMillis() - startEpochMs;
                    if (timeTakenMs > TEST_TIMEOUT_IN_MS) {
                        LOG.error().$("after ").$(timeTakenMs).$("ms tables only had ").$(nRowsWritten).$(" rows out of ").$(nRows).$();
                        break;
                    }
                    Thread.yield();
                    for (int n = 0; n < tables.size(); n++) {
                        CharSequence tableName = tables.get(n);
                        while (true) {
                            try (TableReader reader = engine.getReader(AllowAllCairoSecurityContext.INSTANCE, tableName)) {
                                TableReaderRecordCursor cursor = reader.getCursor();
                                while (cursor.hasNext()) {
                                    nRowsWritten++;
                                }
                                break;
                            } catch (EntryLockedException ex) {
                                LOG.info().$("retrying read for ").$(tableName).$();
                                LockSupport.parkNanos(1);
                            }
                        }
                    }
                } while (nRowsWritten < nRows);
                LOG.info().$(nRowsWritten).$(" rows written").$();
            } finally {
                sharedWorkerPool.halt();
            }
        } finally {
            engine.setPoolListener(null);
        }
        for (int n = 0; n < tables.size(); n++) {
            CharSequence tableName = tables.get(n);
            LOG.info().$("checking table ").$(tableName).$();
            assertTable(expectedSbs[n], tableName);
        }
    });
}

Example 2 with AbstractLineSender

use of io.questdb.cutlass.line.AbstractLineSender in project questdb by bluestreak01.

the class LineUdpInsertTest method assertType.

protected static void assertType(String tableName, String targetColumnName, int columnType, String expected, Consumer<AbstractLineSender> senderConsumer, String... expectedExtraStringColumns) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (CairoEngine engine = new CairoEngine(configuration)) {
            final SOCountDownLatch waitForData = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (event == PoolListener.EV_RETURN && Chars.equals(tableName, name)) {
                    waitForData.countDown();
                }
            });
            try (AbstractLineProtoUdpReceiver receiver = createLineProtoReceiver(engine)) {
                if (columnType != ColumnType.UNDEFINED) {
                    try (TableModel model = new TableModel(configuration, tableName, PartitionBy.NONE)) {
                        CairoTestUtils.create(model.col(targetColumnName, columnType).timestamp());
                    }
                }
                receiver.start();
                try (AbstractLineSender sender = createLineProtoSender()) {
                    senderConsumer.accept(sender);
                    sender.flush();
                }
                // allow reader to hit the readout
                Os.sleep(250L);
            }
            if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
                Assert.fail();
            }
            assertReader(tableName, expected, expectedExtraStringColumns);
        }
    });
}

Example 3 with AbstractLineSender

use of io.questdb.cutlass.line.AbstractLineSender in project questdb by bluestreak01.

the class LineUdpParserSupportTest method testColumnType.

private void testColumnType(int columnType, String expected, Consumer<AbstractLineSender> senderConsumer) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (CairoEngine engine = new CairoEngine(configuration)) {
            final SOCountDownLatch waitForData = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (event == PoolListener.EV_RETURN && Chars.equals(tableName, name)) {
                    waitForData.countDown();
                }
            });
            try (AbstractLineProtoUdpReceiver receiver = createLineProtoReceiver(engine)) {
                try (TableModel model = new TableModel(configuration, tableName, PartitionBy.NONE)) {
                    CairoTestUtils.create(model.col(targetColumnName, columnType).col(locationColumnName, ColumnType.getGeoHashTypeWithBits(30)).timestamp());
                }
                receiver.start();
                try (AbstractLineSender sender = createLineProtoSender()) {
                    senderConsumer.accept(sender);
                    sender.flush();
                }
                Os.sleep(250L);
            }
            if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
                Assert.fail();
            }
        }
        assertReader(tableName, expected);
    });
}