Examples with RatioErrorThreshold com.datastax.oss.dsbulk.workflow.api.error.RatioErrorThreshold used on opensource projects

Example 1 with RatioErrorThreshold

use of com.datastax.oss.dsbulk.workflow.api.error.RatioErrorThreshold in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_sample_size_is_met_and_percentage_exceeded.

@Test
void should_stop_when_sample_size_is_met_and_percentage_exceeded() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forRatio(0.01f, 100), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<ReadResult> stmts = Flux.just(failedReadResult1);
    try {
        stmts.repeat(101).transform(logManager.newTotalItemsCounter()).transform(logManager.newFailedReadsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 1%.");
        Assertions.assertThat(((RatioErrorThreshold) e.getThreshold()).getMaxErrorRatio()).isEqualTo(0.01f);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("unload-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    assertThat(lines.stream().filter(l -> l.contains("BulkExecutionException")).count()).isEqualTo(100);
}

Example 2 with RatioErrorThreshold

use of com.datastax.oss.dsbulk.workflow.api.error.RatioErrorThreshold in project dsbulk by datastax.

the class LogSettingsTest method should_accept_maxErrors_as_percentage.

@Test()
void should_accept_maxErrors_as_percentage() throws IOException {
    Config config = TestConfigUtils.createTestConfig("dsbulk.log", "maxErrors", "20%");
    LogSettings settings = new LogSettings(config, executionId);
    settings.init();
    ErrorThreshold threshold = settings.errorThreshold;
    assertThat(threshold).isInstanceOf(RatioErrorThreshold.class);
    assertThat(((RatioErrorThreshold) threshold).getMaxErrorRatio()).isEqualTo(0.2f);
    // min sample is fixed and cannot be changed by the user currently
    assertThat(((RatioErrorThreshold) threshold).getMinSample()).isEqualTo(100);
}