Examples with TooManyErrorsException com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException used on opensource projects

Example 6 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_at_first_error_when_max_errors_is_zero.

@Test
void should_stop_at_first_error_when_max_errors_is_zero() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(0), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<BatchableStatement<?>> stmts = Flux.just(unmappableStmt1);
    try {
        stmts.transform(logManager.newUnmappableStatementsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 0.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(0);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("mapping.bad");
    Path errors = logManager.getOperationDirectory().resolve("mapping-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(1);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Position: 1").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).containsOnlyOnce("java.lang.RuntimeException: error 1");
}

Example 7 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_sample_size_is_met_and_percentage_exceeded.

@Test
void should_stop_when_sample_size_is_met_and_percentage_exceeded() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forRatio(0.01f, 100), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<ReadResult> stmts = Flux.just(failedReadResult1);
    try {
        stmts.repeat(101).transform(logManager.newTotalItemsCounter()).transform(logManager.newFailedReadsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 1%.");
        Assertions.assertThat(((RatioErrorThreshold) e.getThreshold()).getMaxErrorRatio()).isEqualTo(0.01f);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("unload-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    assertThat(lines.stream().filter(l -> l.contains("BulkExecutionException")).count()).isEqualTo(100);
}

Example 8 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_result_mapping_errors_reached.

@Test
void should_stop_when_max_result_mapping_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<Record> stmts = Flux.just(rowRecord1, rowRecord2, rowRecord3);
    try {
        stmts.transform(logManager.newUnmappableRecordsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("mapping-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).doesNotContain("Source: ").doesNotContain("Resource: ").doesNotContain("Position: ").contains("SELECT 1").containsOnlyOnce("c1: 1").containsOnlyOnce("java.lang.RuntimeException: error 1").contains("SELECT 2").containsOnlyOnce("c1: 2").containsOnlyOnce("java.lang.RuntimeException: error 2").doesNotContain("c3: 3");
}

Example 9 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_read_errors_reached.

@Test
void should_stop_when_max_read_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<ReadResult> stmts = Flux.just(failedReadResult1, failedReadResult2, failedReadResult3);
    try {
        stmts.transform(logManager.newFailedReadsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("unload-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).doesNotContain("Resource: ").doesNotContain("Position: ").doesNotContain("Source: ").contains("SELECT 1").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: SELECT 1 (error 1)").contains("SELECT 2").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: SELECT 2 (error 2)");
}