Examples with TestResult io.openmessaging.benchmark.TestResult used on opensource projects

Example 1 with TestResult

use of io.openmessaging.benchmark.TestResult in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaInstanceScalingSmallTest method testConnections.

@ParameterizedTest(name = "tesConnections_m{0}_r{1}_p{2}_c{3}")
@CsvSource({ "100, 50, 8, 8, 4Gi, 750m", "100, 50, 1, 15, 4Gi, 750m", "100, 50, 15, 1, 4Gi, 750m" })
void testConnections(int maxConnections, int connectionCreationRate, int numProducers, int numConsumers, String ombWorkerMem, String ombWorkerCpu, TestInfo info) throws Exception {
    int messageSize = 1024;
    int targetRate = 2_000;
    int workersPerInstance = 2;
    ManagedKafkaCapacity capacity = kafkaProvisioner.defaultCapacity((long) targetRate * messageSize * 2);
    capacity.setMaxConnectionAttemptsPerSec(connectionCreationRate);
    capacity.setTotalMaxConnections(maxConnections);
    ManagedKafkaDeployment kafkaDeployment = kafkaProvisioner.deployCluster("cluster1", capacity, AdopterProfile.SMALL_VALUE_PROD);
    omb.setWorkerContainerMemory(Quantity.parse(ombWorkerMem));
    omb.setWorkerCpu(Quantity.parse(ombWorkerCpu));
    workers = omb.deployWorkers(workersPerInstance);
    Map<ManagedKafkaDeployment, List<String>> workerMapping = new HashMap<>();
    Iterator<String> workerIt = workers.iterator();
    Map<ManagedKafkaDeployment, String> instanceBootstrap = new HashMap<>();
    List<String> ws = new ArrayList<>();
    for (int w = 0; w < workersPerInstance; w++) {
        ws.add(workerIt.next());
    }
    workerMapping.put(kafkaDeployment, ws);
    instanceBootstrap.put(kafkaDeployment, kafkaDeployment.waitUntilReady());
    ExecutorService executorService = Executors.newFixedThreadPool(1);
    AtomicInteger timeout = new AtomicInteger();
    List<TestResult> testResults = new ArrayList<>();
    try {
        List<Future<OMBWorkloadResult>> results = new ArrayList<>();
        for (Map.Entry<ManagedKafkaDeployment, String> entry : instanceBootstrap.entrySet()) {
            File ombDir = new File(instanceDir, entry.getKey().getManagedKafka().getMetadata().getName());
            Files.createDirectories(ombDir.toPath());
            OMBDriver driver = new OMBDriver().setReplicationFactor(3).setTopicConfig("min.insync.replicas=2\n").setCommonConfigWithBootstrapUrl(entry.getValue()).setProducerConfig("acks=all\n").setConsumerConfig("auto.offset.reset=earliest\nenable.auto.commit=false\n");
            OMBWorkload workload = new OMBWorkload().setName(String.format("Kafka Cluster: %s", entry.getKey().getManagedKafka().getMetadata().getName())).setTopics(1).setPartitionsPerTopic(99).setMessageSize(messageSize).setPayloadFile("src/test/resources/payload/payload-1Kb.data").setSubscriptionsPerTopic(numConsumers).setConsumerPerSubscription(1).setProducersPerTopic(numProducers).setProducerRate(targetRate).setConsumerBacklogSizeGB(0);
            timeout.set(Math.max(workload.getTestDurationMinutes() + workload.getWarmupDurationMinutes(), timeout.get()));
            results.add(executorService.submit(() -> {
                OMBWorkloadResult result = omb.runWorkload(ombDir, driver, workerMapping.get(entry.getKey()), workload);
                LOGGER.info("Result stored in {}", result.getResultFile().getAbsolutePath());
                return result;
            }));
        }
        for (Future<OMBWorkloadResult> result : results) {
            testResults.add(result.get(timeout.get() * 2L, TimeUnit.MINUTES).getTestResult());
        }
    } finally {
        executorService.shutdown();
        executorService.awaitTermination(1, TimeUnit.MINUTES);
    }
}

Example 2 with TestResult

use of io.openmessaging.benchmark.TestResult in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class InstanceProfiler method determineLatency.

protected LatencyResult determineLatency(Profile profile, Consumer<Workload> setupModifier, BiConsumer<Profile, TestResult> resultConsumer) throws Exception {
    int pub = nominalPartitionCount;
    OMBWorkload load = createBasicWorkload(lowThroughput, pub, profile);
    load.warmupDurationMinutes = 2;
    load.testDurationMinutes = 4;
    setupModifier.accept(load);
    LatencyResult result = new LatencyResult();
    result.targetIngressMBs = load.producerRate * load.messageSize / (double) ONE_MB;
    result.targetEgressMBs = result.targetIngressMBs * load.subscriptionsPerTopic;
    try {
        OMBWorkloadResult loadResult = doTestRun(load.name, load, profile);
        TestResult loadTestResult = loadResult.getTestResult();
        double avgPublishRate = TestUtils.getAvg(loadTestResult.publishRate);
        double avgPublishThrougput = avgPublishRate * profile.messageSize;
        if (!isThroughputAcceptable(load, loadTestResult)) {
            result.error = new ErrorResult();
            result.error.message = "latency result may not acceptable due to throughput inconsistency";
            result.error.throughput = toThroughputResult(profile, loadTestResult);
        }
        result.aggregatedEndToEndLatency50pct = loadTestResult.aggregatedEndToEndLatency50pct;
        // the 99% tile can be very noisy, we'll use the median instead of the aggregated value.  This will
        // always be significantly higher than the 95%, but not truly reflect the 99% - however we are not yet able to
        result.medianEndToEndLatency99pct = TestUtils.getMedian(loadTestResult.endToEndLatency99pct);
        result.aggregatedPublishLatency50pct = loadTestResult.aggregatedPublishLatency50pct;
        result.aggregatedPublishLatency99pct = loadTestResult.aggregatedPublishLatency99pct;
        result.maxConnectionCount = loadTestResult.connectionCount.stream().max(Double::compareTo).get();
        if (resultConsumer != null) {
            resultConsumer.accept(profile, loadTestResult);
        }
        // TODO validate low throttle / queue / consumer latency
        // TODO a warning if samples are too far off the median
        LOGGER.info(String.format("Result summary for %s consumers / %s partitions latency %,.2f MB/s e2e latency 50: %,.2f median 99: %,.2f 99: %,.2f", pub, load.partitionsPerTopic, avgPublishThrougput / ONE_MB, loadTestResult.aggregatedEndToEndLatency50pct, result.medianEndToEndLatency99pct, loadTestResult.aggregatedEndToEndLatency99pct));
    } catch (Exception e) {
        result.error = new ErrorResult();
        result.error.message = e.getMessage();
        LOGGER.info(String.format("not accepting %s as there were errors during the test", load.name), e);
    }
    return result;
}

Example 3 with TestResult

use of io.openmessaging.benchmark.TestResult in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class OMB method runWorkload.

public OMBWorkloadResult runWorkload(File testDir, OMBDriver driver, List<String> workers, OMBWorkload workload) throws Exception {
    File driverFile = new File(testDir, "driver.yaml");
    File resultFile = new File(testDir, String.format("result_%s.json", workload.getName().replaceAll(" ", "_")));
    Files.writeString(driverFile.toPath(), MAPPER.writeValueAsString(driver));
    LOGGER.info("Wrote driver to {}", driverFile.getAbsolutePath());
    workload.validate();
    try (Worker worker = workers.isEmpty() ? new LocalWorker() : new DistributedWorkersEnsemble(workers);
        WorkloadGenerator generator = new WorkloadGenerator(driver.name, workload, worker)) {
        LOGGER.info("--------------- WORKLOAD: {} --- DRIVER: {} ---------------", workload.name, driver.name);
        worker.initializeDriver(driverFile);
        TestResult result = generator.run();
        try {
            worker.stopAll();
        } catch (IOException e) {
        }
        LOGGER.info("Writing test result into {}", resultFile.getAbsolutePath());
        WRITER.writeValue(resultFile, result);
    } catch (Exception e) {
        LOGGER.error("Failed to run the workload '{}' for driver '{}'", workload.name, driverFile.getAbsolutePath(), e);
        throw e;
    }
    TestMetadataCapture.getInstance().storeOmbData(ombCluster, workload, driver, this);
    return new OMBWorkloadResult(resultFile, createTestResult(resultFile));
}