Examples with Committer org.apache.druid.data.input.Committer used on opensource projects

Example 21 with Committer

use of org.apache.druid.data.input.Committer in project druid by druid-io.

the class StreamAppenderatorTest method testMaxBytesInMemoryWithSkipBytesInMemoryOverheadCheckConfig.

@Test
public void testMaxBytesInMemoryWithSkipBytesInMemoryOverheadCheckConfig() throws Exception {
    try (final StreamAppenderatorTester tester = new StreamAppenderatorTester(100, 1024, null, true, new SimpleRowIngestionMeters(), true)) {
        final Appenderator appenderator = tester.getAppenderator();
        final AtomicInteger eventCount = new AtomicInteger(0);
        final Supplier<Committer> committerSupplier = () -> {
            final Object metadata = ImmutableMap.of(eventCount, eventCount.get());
            return new Committer() {

                @Override
                public Object getMetadata() {
                    return metadata;
                }

                @Override
                public void run() {
                // Do nothing
                }
            };
        };
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 1), committerSupplier);
        // expectedSizeInBytes = 44(map overhead) + 28 (TimeAndDims overhead) + 56 (aggregator metrics) + 54 (dimsKeySize) = 182 + 1 byte when null handling is enabled
        int nullHandlingOverhead = NullHandling.sqlCompatible() ? 1 : 0;
        Assert.assertEquals(182 + nullHandlingOverhead, ((StreamAppenderator) appenderator).getBytesInMemory(IDENTIFIERS.get(0)));
        appenderator.add(IDENTIFIERS.get(1), ir("2000", "bar", 1), committerSupplier);
        Assert.assertEquals(182 + nullHandlingOverhead, ((StreamAppenderator) appenderator).getBytesInMemory(IDENTIFIERS.get(1)));
        appenderator.close();
        Assert.assertEquals(0, ((StreamAppenderator) appenderator).getRowsInMemory());
    }
}

Example 22 with Committer

use of org.apache.druid.data.input.Committer in project druid by druid-io.

the class StreamAppenderatorTest method testTaskDoesNotFailAsExceededMemoryWithSkipBytesInMemoryOverheadCheckConfig.

@Test
public void testTaskDoesNotFailAsExceededMemoryWithSkipBytesInMemoryOverheadCheckConfig() throws Exception {
    try (final StreamAppenderatorTester tester = new StreamAppenderatorTester(100, 10, null, true, new SimpleRowIngestionMeters(), true)) {
        final Appenderator appenderator = tester.getAppenderator();
        final AtomicInteger eventCount = new AtomicInteger(0);
        final Supplier<Committer> committerSupplier = () -> {
            final Object metadata = ImmutableMap.of(eventCount, eventCount.get());
            return new Committer() {

                @Override
                public Object getMetadata() {
                    return metadata;
                }

                @Override
                public void run() {
                // Do nothing
                }
            };
        };
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 1), committerSupplier);
        // Expected 0 since we persisted after the add
        Assert.assertEquals(0, ((StreamAppenderator) appenderator).getBytesCurrentlyInMemory());
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 1), committerSupplier);
        // Expected 0 since we persisted after the add
        Assert.assertEquals(0, ((StreamAppenderator) appenderator).getBytesCurrentlyInMemory());
    }
}