Examples with RowIngestionMeters org.apache.druid.segment.incremental.RowIngestionMeters used on opensource projects

Example 1 with RowIngestionMeters

use of org.apache.druid.segment.incremental.RowIngestionMeters in project druid by druid-io.

the class ClosedSegmentsSinksBatchAppenderatorTest method testVerifyRowIngestionMetrics.

@Test
public void testVerifyRowIngestionMetrics() throws Exception {
    final RowIngestionMeters rowIngestionMeters = new SimpleRowIngestionMeters();
    try (final ClosedSegmensSinksBatchAppenderatorTester tester = new ClosedSegmensSinksBatchAppenderatorTester(5, 10000L, null, false, rowIngestionMeters)) {
        final Appenderator appenderator = tester.getAppenderator();
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), createInputRow("2000", "foo", "invalid_met"), null);
        appenderator.add(IDENTIFIERS.get(0), createInputRow("2000", "foo", 1), null);
        Assert.assertEquals(1, rowIngestionMeters.getProcessed());
        Assert.assertEquals(1, rowIngestionMeters.getProcessedWithError());
        Assert.assertEquals(0, rowIngestionMeters.getUnparseable());
        Assert.assertEquals(0, rowIngestionMeters.getThrownAway());
    }
}

Example 2 with RowIngestionMeters

use of org.apache.druid.segment.incremental.RowIngestionMeters in project druid by druid-io.

the class ClosedSegmentsSinksBatchAppenderatorTest method testCloseContract.

@Test(timeout = 5000L)
public void testCloseContract() throws Exception {
    final RowIngestionMeters rowIngestionMeters = new SimpleRowIngestionMeters();
    try (final ClosedSegmensSinksBatchAppenderatorTester tester = new ClosedSegmensSinksBatchAppenderatorTester(1, 50000L, null, false, rowIngestionMeters)) {
        final Appenderator appenderator = tester.getAppenderator();
        appenderator.startJob();
        // each one of these adds will trigger a persist since maxRowsInMemory is set to one above
        appenderator.add(IDENTIFIERS.get(0), createInputRow("2000", "bar", 1), null);
        appenderator.add(IDENTIFIERS.get(0), createInputRow("2000", "bar2", 1), null);
        // push only a single segment
        ListenableFuture<SegmentsAndCommitMetadata> firstFuture = appenderator.push(Collections.singletonList(IDENTIFIERS.get(0)), null, false);
        // push remaining segments:
        appenderator.add(IDENTIFIERS.get(1), createInputRow("2000", "bar3", 1), null);
        ListenableFuture<SegmentsAndCommitMetadata> secondFuture = appenderator.push(Collections.singletonList(IDENTIFIERS.get(1)), null, false);
        // close should wait for all pushes and persists to end:
        appenderator.close();
        Assert.assertTrue(!firstFuture.isCancelled());
        Assert.assertTrue(!secondFuture.isCancelled());
        Assert.assertTrue(firstFuture.isDone());
        Assert.assertTrue(secondFuture.isDone());
        final SegmentsAndCommitMetadata segmentsAndCommitMetadataForFirstFuture = firstFuture.get();
        final SegmentsAndCommitMetadata segmentsAndCommitMetadataForSecondFuture = secondFuture.get();
        // all segments must have been pushed:
        Assert.assertEquals(segmentsAndCommitMetadataForFirstFuture.getSegments().size(), 1);
        Assert.assertEquals(segmentsAndCommitMetadataForSecondFuture.getSegments().size(), 1);
    }
}

Example 3 with RowIngestionMeters

use of org.apache.druid.segment.incremental.RowIngestionMeters in project druid by druid-io.

the class StreamAppenderatorTest method testVerifyRowIngestionMetrics.

@Test
public void testVerifyRowIngestionMetrics() throws Exception {
    final RowIngestionMeters rowIngestionMeters = new SimpleRowIngestionMeters();
    try (final StreamAppenderatorTester tester = new StreamAppenderatorTester(5, 10000L, null, false, rowIngestionMeters)) {
        final Appenderator appenderator = tester.getAppenderator();
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", "invalid_met"), Committers.nilSupplier());
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 1), Committers.nilSupplier());
        Assert.assertEquals(1, rowIngestionMeters.getProcessed());
        Assert.assertEquals(1, rowIngestionMeters.getProcessedWithError());
        Assert.assertEquals(0, rowIngestionMeters.getUnparseable());
        Assert.assertEquals(0, rowIngestionMeters.getThrownAway());
    }
}

Example 4 with RowIngestionMeters

use of org.apache.druid.segment.incremental.RowIngestionMeters in project druid by druid-io.

the class PartialSegmentGenerateTask method generateSegments.

private List<DataSegment> generateSegments(final TaskToolbox toolbox, final ParallelIndexSupervisorTaskClient taskClient, final InputSource inputSource, final File tmpDir) throws IOException, InterruptedException, ExecutionException, TimeoutException {
    final DataSchema dataSchema = ingestionSchema.getDataSchema();
    final FireDepartment fireDepartmentForMetrics = new FireDepartment(dataSchema, new RealtimeIOConfig(null, null), null);
    final FireDepartmentMetrics fireDepartmentMetrics = fireDepartmentForMetrics.getMetrics();
    final RowIngestionMeters buildSegmentsMeters = toolbox.getRowIngestionMetersFactory().createRowIngestionMeters();
    toolbox.addMonitor(new RealtimeMetricsMonitor(Collections.singletonList(fireDepartmentForMetrics), Collections.singletonMap(DruidMetrics.TASK_ID, new String[] { getId() })));
    final ParallelIndexTuningConfig tuningConfig = ingestionSchema.getTuningConfig();
    final PartitionsSpec partitionsSpec = tuningConfig.getGivenOrDefaultPartitionsSpec();
    final long pushTimeout = tuningConfig.getPushTimeout();
    final SegmentAllocatorForBatch segmentAllocator = createSegmentAllocator(toolbox, taskClient);
    final SequenceNameFunction sequenceNameFunction = segmentAllocator.getSequenceNameFunction();
    final ParseExceptionHandler parseExceptionHandler = new ParseExceptionHandler(buildSegmentsMeters, tuningConfig.isLogParseExceptions(), tuningConfig.getMaxParseExceptions(), tuningConfig.getMaxSavedParseExceptions());
    final boolean useMaxMemoryEstimates = getContextValue(Tasks.USE_MAX_MEMORY_ESTIMATES, Tasks.DEFAULT_USE_MAX_MEMORY_ESTIMATES);
    final Appenderator appenderator = BatchAppenderators.newAppenderator(getId(), toolbox.getAppenderatorsManager(), fireDepartmentMetrics, toolbox, dataSchema, tuningConfig, new ShuffleDataSegmentPusher(supervisorTaskId, getId(), toolbox.getIntermediaryDataManager()), buildSegmentsMeters, parseExceptionHandler, useMaxMemoryEstimates);
    boolean exceptionOccurred = false;
    try (final BatchAppenderatorDriver driver = BatchAppenderators.newDriver(appenderator, toolbox, segmentAllocator)) {
        driver.startJob();
        final SegmentsAndCommitMetadata pushed = InputSourceProcessor.process(dataSchema, driver, partitionsSpec, inputSource, inputSource.needsFormat() ? ParallelIndexSupervisorTask.getInputFormat(ingestionSchema) : null, tmpDir, sequenceNameFunction, inputRowIteratorBuilder, buildSegmentsMeters, parseExceptionHandler, pushTimeout);
        return pushed.getSegments();
    } catch (Exception e) {
        exceptionOccurred = true;
        throw e;
    } finally {
        if (exceptionOccurred) {
            appenderator.closeNow();
        } else {
            appenderator.close();
        }
    }
}

Example 5 with RowIngestionMeters

use of org.apache.druid.segment.incremental.RowIngestionMeters in project druid by druid-io.

the class StreamChunkParserTest method parseEmptyEndOfShard.

@Test
public void parseEmptyEndOfShard() throws IOException {
    final TrackingJsonInputFormat inputFormat = new TrackingJsonInputFormat(JSONPathSpec.DEFAULT, Collections.emptyMap());
    RowIngestionMeters mockRowIngestionMeters = Mockito.mock(RowIngestionMeters.class);
    final StreamChunkParser<ByteEntity> chunkParser = new StreamChunkParser<>(null, inputFormat, new InputRowSchema(TIMESTAMP_SPEC, DimensionsSpec.EMPTY, ColumnsFilter.all()), TransformSpec.NONE, temporaryFolder.newFolder(), row -> true, mockRowIngestionMeters, parseExceptionHandler);
    List<InputRow> parsedRows = chunkParser.parse(ImmutableList.of(), true);
    Assert.assertEquals(0, parsedRows.size());
    Mockito.verifyNoInteractions(mockRowIngestionMeters);
}