Examples with StreamEventWriter org.apache.druid.testing.utils.StreamEventWriter used on opensource projects

Example 1 with StreamEventWriter

use of org.apache.druid.testing.utils.StreamEventWriter in project druid by druid-io.

the class AbstractStreamIndexingTest method testIndexWithLosingNodeHelper.

private void testIndexWithLosingNodeHelper(Runnable restartRunnable, Runnable waitForReadyRunnable, @Nullable Boolean transactionEnabled) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating one third of the data (before restarting)
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before restart
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Restart Druid process
        LOG.info("Restarting Druid process");
        restartRunnable.run();
        LOG.info("Restarted Druid process");
        // Start generating one third of the data (while restarting)
        int secondsToGenerateSecondRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateSecondRound;
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateSecondRound, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Wait for Druid process to be available
        LOG.info("Waiting for Druid process to be available");
        waitForReadyRunnable.run();
        LOG.info("Druid process is now available");
        // Start generating remaining data (after restarting)
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound + secondsToGenerateSecondRound));
        // Verify supervisor is healthy
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor ingested all data
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}

Example 2 with StreamEventWriter

use of org.apache.druid.testing.utils.StreamEventWriter in project druid by druid-io.

the class AbstractStreamIndexingTest method testIndexWithStreamReshardHelper.

private void testIndexWithStreamReshardHelper(@Nullable Boolean transactionEnabled, int newShardCount) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating one third of the data (before resharding)
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before resahrding
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Reshard the supervisor by split from STREAM_SHARD_COUNT to newShardCount and waits until the resharding starts
        streamAdminClient.updatePartitionCount(generatedTestConfig.getStreamName(), newShardCount, true);
        // Start generating one third of the data (while resharding)
        int secondsToGenerateSecondRound = TOTAL_NUMBER_OF_SECOND / 3;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateSecondRound;
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateSecondRound, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Wait for stream to finish resharding
        ITRetryUtil.retryUntil(() -> streamAdminClient.isStreamActive(generatedTestConfig.getStreamName()), true, 10000, 30, "Waiting for stream to finish resharding");
        ITRetryUtil.retryUntil(() -> streamAdminClient.verfiyPartitionCountUpdated(generatedTestConfig.getStreamName(), STREAM_SHARD_COUNT, newShardCount), true, 10000, 30, "Waiting for stream to finish resharding");
        // Start generating remaining data (after resharding)
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound + secondsToGenerateSecondRound));
        // Verify supervisor is healthy after resahrding
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor can catch up with the stream
        verifyIngestedData(generatedTestConfig, numWritten);
    }
    // Verify that event thrown away count was not incremented by the reshard
    List<TaskResponseObject> completedTasks = indexer.getCompleteTasksForDataSource(generatedTestConfig.getFullDatasourceName());
    for (TaskResponseObject task : completedTasks) {
        try {
            RowIngestionMetersTotals stats = indexer.getTaskStats(task.getId());
            Assert.assertEquals(0L, stats.getThrownAway());
        } catch (Exception e) {
            // Failed task may not have a task stats report. We can ignore it as the task did not consume any data
            if (!task.getStatus().isFailure()) {
                throw e;
            }
        }
    }
}

Example 3 with StreamEventWriter

use of org.apache.druid.testing.utils.StreamEventWriter in project druid by druid-io.

the class ITAutoCompactionLockContentionTest method testAutoCompactionSkipsLockedIntervals.

@Test(dataProvider = "getParameters")
public void testAutoCompactionSkipsLockedIntervals(boolean transactionEnabled) throws Exception {
    if (shouldSkipTest(transactionEnabled)) {
        return;
    }
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        // Start supervisor
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("supervisorSpec: [%s]", taskSpec);
        // Generate data for minutes 1, 2 and 3
        final Interval minute1 = Intervals.of("2000-01-01T01:01:00Z/2000-01-01T01:02:00Z");
        final long rowsForMinute1 = generateData(minute1, streamEventWriter);
        final Interval minute2 = Intervals.of("2000-01-01T01:02:00Z/2000-01-01T01:03:00Z");
        long rowsForMinute2 = generateData(minute2, streamEventWriter);
        final Interval minute3 = Intervals.of("2000-01-01T01:03:00Z/2000-01-01T01:04:00Z");
        final long rowsForMinute3 = generateData(minute3, streamEventWriter);
        // Wait for data to be ingested for all the minutes
        ensureRowCount(rowsForMinute1 + rowsForMinute2 + rowsForMinute3);
        // Wait for the segments to be loaded and interval locks to be released
        ensureLockedIntervals();
        ensureSegmentsLoaded();
        // 2 segments for each minute, total 6
        ensureSegmentsCount(6);
        // Generate more data for minute2 so that it gets locked
        rowsForMinute2 += generateData(minute2, streamEventWriter);
        ensureLockedIntervals(minute2);
        // Trigger auto compaction
        submitAndVerifyCompactionConfig();
        compactionResource.forceTriggerAutoCompaction();
        // Wait for segments to be loaded
        ensureRowCount(rowsForMinute1 + rowsForMinute2 + rowsForMinute3);
        ensureLockedIntervals();
        ensureSegmentsLoaded();
        // Verify that minute1 and minute3 have been compacted
        ensureCompactionTaskCount(2);
        verifyCompactedIntervals(minute1, minute3);
        // Trigger auto compaction again
        compactionResource.forceTriggerAutoCompaction();
        // Verify that all the segments are now compacted
        ensureCompactionTaskCount(3);
        ensureSegmentsLoaded();
        verifyCompactedIntervals(minute1, minute2, minute3);
        ensureSegmentsCount(3);
    }
}

Example 4 with StreamEventWriter

use of org.apache.druid.testing.utils.StreamEventWriter in project druid by druid-io.

the class AbstractStreamIndexingTest method doTestIndexDataStableState.

protected void doTestIndexDataStableState(@Nullable Boolean transactionEnabled, String serializerPath, String parserType, String specPath) throws Exception {
    final EventSerializer serializer = jsonMapper.readValue(getResourceAsStream(serializerPath), EventSerializer.class);
    final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(serializer, EVENTS_PER_SECOND, CYCLE_PADDING_MS);
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(parserType, getResourceAsString(specPath));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start data generator
        final long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, TOTAL_NUMBER_OF_SECOND, FIRST_EVENT_TIME);
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}

Example 5 with StreamEventWriter

use of org.apache.druid.testing.utils.StreamEventWriter in project druid by druid-io.

the class AbstractStreamIndexingTest method doTestIndexDataWithStartStopSupervisor.

protected void doTestIndexDataWithStartStopSupervisor(@Nullable Boolean transactionEnabled) throws Exception {
    final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig(INPUT_FORMAT, getResourceAsString(JSON_INPUT_FORMAT_PATH));
    try (final Closeable closer = createResourceCloser(generatedTestConfig);
        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)) {
        final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform().apply(getResourceAsString(SUPERVISOR_SPEC_TEMPLATE_PATH));
        LOG.info("supervisorSpec: [%s]\n", taskSpec);
        // Start supervisor
        generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
        LOG.info("Submitted supervisor");
        // Start generating half of the data
        int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND;
        int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 2;
        secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound;
        final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator(new JsonEventSerializer(jsonMapper), EVENTS_PER_SECOND, CYCLE_PADDING_MS);
        long numWritten = streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateFirstRound, FIRST_EVENT_TIME);
        // Verify supervisor is healthy before suspension
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Suspend the supervisor
        indexer.suspendSupervisor(generatedTestConfig.getSupervisorId());
        // Start generating remainning half of the data
        numWritten += streamGenerator.run(generatedTestConfig.getStreamName(), streamEventWriter, secondsToGenerateRemaining, FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound));
        // Resume the supervisor
        indexer.resumeSupervisor(generatedTestConfig.getSupervisorId());
        // Verify supervisor is healthy after suspension
        ITRetryUtil.retryUntil(() -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy");
        // Verify that supervisor can catch up with the stream
        verifyIngestedData(generatedTestConfig, numWritten);
    }
}