Examples with StreamGenerator org.apache.druid.testing.utils.StreamGenerator used on opensource projects

Example 1 with StreamGenerator

use of org.apache.druid.testing.utils.StreamGenerator in project druid by druid-io.

the class AbstractStreamIndexingTest method testIndexWithLosingNodeHelper.

private void testIndexWithLosingNodeHelper(Runnable restartRunnable, Runnable waitForReadyRunnable, @Nullable Boolean transactionEnabled) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating one third of the data (before restarting)
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before restart
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Restart Druid process
        LOG.info("Restarting Druid process");
        restartRunnable.run();
        LOG.info("Restarted Druid process");
        // Start generating one third of the data (while restarting)
        int secondsToGenerateSecondRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateSecondRound;
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateSecondRound, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Wait for Druid process to be available
        LOG.info("Waiting for Druid process to be available");
        waitForReadyRunnable.run();
        LOG.info("Druid process is now available");
        // Start generating remaining data (after restarting)
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound + secondsToGenerateSecondRound));
        // Verify supervisor is healthy
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor ingested all data
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}

Example 2 with StreamGenerator

use of org.apache.druid.testing.utils.StreamGenerator in project druid by druid-io.

the class AbstractStreamIndexingTest method testIndexWithStreamReshardHelper.

private void testIndexWithStreamReshardHelper(@Nullable Boolean transactionEnabled, int newShardCount) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating one third of the data (before resharding)
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before resahrding
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Reshard the supervisor by split from STREAM_SHARD_COUNT to newShardCount and waits until the resharding starts
        streamAdminClient.updatePartitionCount(generatedTestConfig.getStreamName(), newShardCount, true);
        // Start generating one third of the data (while resharding)
        int secondsToGenerateSecondRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateSecondRound;
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateSecondRound, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Wait for stream to finish resharding
        ITRetryUtil.retryUntil(() -> streamAdminClient.isStreamActive(generatedTestConfig.getStreamName()), true, 10000, 30, "Waiting for stream to finish resharding");
        ITRetryUtil.retryUntil(() -> streamAdminClient.verfiyPartitionCountUpdated(generatedTestConfig.getStreamName(), STREAM_SHARD_COUNT, newShardCount), true, 10000, 30, "Waiting for stream to finish resharding");
        // Start generating remaining data (after resharding)
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound + secondsToGenerateSecondRound));
        // Verify supervisor is healthy after resahrding
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor can catch up with the stream
        verifyIngestedData(generatedTestConfig, numWritten);
    }
    // Verify that event thrown away count was not incremented by the reshard
    List<TaskResponseObject> completedTasks = indexer.getCompleteTasksForDataSource(generatedTestConfig.getFullDatasourceName());
    for (TaskResponseObject task : completedTasks) {
        try {
            RowIngestionMetersTotals stats = indexer.getTaskStats(task.getId());
            Assert.assertEquals(0L, stats.getThrownAway());
        } catch (Exception e) {
            // Failed task may not have a task stats report. We can ignore it as the task did not consume any data
            if (!task.getStatus().isFailure()) {
                throw e;
            }
        }
    }
}

Example 3 with StreamGenerator

use of org.apache.druid.testing.utils.StreamGenerator in project druid by druid-io.

the class AbstractStreamIndexingTest method doTestIndexDataStableState.

protected void doTestIndexDataStableState(@Nullable Boolean transactionEnabled, String serializerPath, String parserType, String specPath) throws Exception {
    final EventSerializer serializer = jsonMapper.readValue(getResourceAsStream(serializerPath), EventSerializer.class);
    final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(serializer, EVENTS_PER_SECOND, CYCLE_PADDING_MS);
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(parserType, getResourceAsString(specPath));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start data generator
        final long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, TOTAL_NUMBER_OF_SECOND, FIRST_EVENT_TIME);
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}

Example 4 with StreamGenerator

use of org.apache.druid.testing.utils.StreamGenerator in project druid by druid-io.

the class AbstractStreamIndexingTest method doTestIndexDataWithStartStopSupervisor.

protected void doTestIndexDataWithStartStopSupervisor(@Nullable Boolean transactionEnabled) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating half of the data
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 2;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before suspension
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Suspend the supervisor
        indexer.suspendSupervisor(generatedTestConfig.getSupervisorId());
        // Start generating remainning half of the data
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Resume the supervisor
        indexer.resumeSupervisor(generatedTestConfig.getSupervisorId());
        // Verify supervisor is healthy after suspension
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor can catch up with the stream
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}

Example 5 with StreamGenerator

use of org.apache.druid.testing.utils.StreamGenerator in project druid by druid-io.

the class AbstractStreamIndexingTest method doTestIndexDataWithAutoscaler.

protected void doTestIndexDataWithAutoscaler(@Nullable Boolean transactionEnabled) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating half of the data
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 2;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before suspension
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // wait for autoScaling task numbers from 1 to 2.
        ITRetryUtil.retryUntil(() -> indexer.getRunningTasks().size() == 2, true, 10000, 50, "waiting for autoScaling task numbers from 1 to 2");
        // Start generating remainning half of the data
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Verify that supervisor can catch up with the stream
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}