Examples with TooManyErrorsException com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException used on opensource projects

Example 1 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_write_errors_reached_and_statements_batched.

@Test
void should_stop_when_max_write_errors_reached_and_statements_batched() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(1), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<WriteResult> stmts = Flux.just(batchWriteResult);
    try {
        stmts.transform(logManager.newFailedWritesHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 1.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(1);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("load.bad");
    Path errors = logManager.getOperationDirectory().resolve("load-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(3);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(badLines.get(1)).isEqualTo(source2.trim());
    assertThat(badLines.get(2)).isEqualTo(source3.trim());
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1.toString()).containsOnlyOnce("Resource: " + resource2.toString()).containsOnlyOnce("Resource: " + resource3.toString()).containsOnlyOnce("Position: 1").containsOnlyOnce("Position: 2").containsOnlyOnce("Position: 3").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source2)).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source3)).contains("INSERT 1").contains("INSERT 2").contains("INSERT 3").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed").contains("error batch");
    List<String> positionLines = Files.readAllLines(positions, UTF_8);
    assertThat(positionLines).contains("file:///file1.csv:1").contains("file:///file2.csv:2").contains("file:///file3.csv:3");
}

Example 2 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_record_mapping_errors_reached.

@Test
void should_stop_when_max_record_mapping_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<BatchableStatement<?>> stmts = Flux.just(unmappableStmt1, unmappableStmt2, unmappableStmt3);
    try {
        stmts.transform(logManager.newUnmappableStatementsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        Assertions.assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("mapping.bad");
    Path errors = logManager.getOperationDirectory().resolve("mapping-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(3);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(badLines.get(1)).isEqualTo(source2.trim());
    assertThat(badLines.get(2)).isEqualTo(source3.trim());
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).containsOnlyOnce("java.lang.RuntimeException: error 1").containsOnlyOnce("Resource: " + resource2).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source2)).containsOnlyOnce("java.lang.RuntimeException: error 2").containsOnlyOnce("Resource: " + resource3).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source3)).containsOnlyOnce("java.lang.RuntimeException: error 3");
    List<String> positionLines = Files.readAllLines(positions, UTF_8);
    assertThat(positionLines).contains("file:///file1.csv:1").contains("file:///file2.csv:2").contains("file:///file3.csv:3");
}

Example 3 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_cas_errors_reached.

@Test
void should_stop_when_max_cas_errors_reached() throws Exception {
    BatchStatement casBatch = BatchStatement.newInstance(DefaultBatchType.UNLOGGED, mockBulkBoundStatement(1, source1, resource1), mockBulkBoundStatement(2, source2, resource2), mockBulkBoundStatement(3, source3, resource3));
    Row row1 = mockRow(1);
    Row row2 = mockRow(2);
    Row row3 = mockRow(3);
    AsyncResultSet rs = mock(AsyncResultSet.class);
    when(rs.wasApplied()).thenReturn(false);
    when(rs.currentPage()).thenReturn(Lists.newArrayList(row1, row2, row3));
    ExecutionInfo executionInfo = mock(ExecutionInfo.class);
    when(rs.getExecutionInfo()).thenReturn(executionInfo);
    DefaultWriteResult casBatchWriteResult = new DefaultWriteResult(casBatch, rs);
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<WriteResult> stmts = Flux.just(casBatchWriteResult);
    try {
        stmts.transform(logManager.newFailedWritesHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("paxos.bad");
    Path errors = logManager.getOperationDirectory().resolve("paxos-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(3);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(badLines.get(1)).isEqualTo(source2.trim());
    assertThat(badLines.get(2)).isEqualTo(source3.trim());
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Failed conditional updates: ").containsOnlyOnce("INSERT INTO 1").contains("c1: 1").containsOnlyOnce("INSERT INTO 2").contains("c1: 2").containsOnlyOnce("INSERT INTO 3").contains("c1: 3");
}

Example 4 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_connector_errors_reached.

@Test
void should_stop_when_max_connector_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<Record> records = Flux.just(csvRecord1, csvRecord2, csvRecord3);
    try {
        records.transform(logManager.newFailedRecordsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("connector.bad");
    Path errors = logManager.getOperationDirectory().resolve("connector-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Position: 1").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).containsOnlyOnce("java.lang.RuntimeException: error 1").containsOnlyOnce("Resource: " + resource2).containsOnlyOnce("Position: 2").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source2)).containsOnlyOnce("java.lang.RuntimeException: error 2").containsOnlyOnce("Resource: " + resource3).containsOnlyOnce("Position: 3").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source3)).containsOnlyOnce("java.lang.RuntimeException: error 3");
}

Example 5 with TooManyErrorsException

use of com.datastax.oss.dsbulk.workflow.api.error.TooManyErrorsException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_write_errors_reached.

@Test
void should_stop_when_max_write_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<WriteResult> stmts = Flux.just(failedWriteResult1, failedWriteResult2, failedWriteResult3);
    try {
        stmts.transform(logManager.newFailedWritesHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("load.bad");
    Path errors = logManager.getOperationDirectory().resolve("load-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(3);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(badLines.get(1)).isEqualTo(source2.trim());
    assertThat(badLines.get(2)).isEqualTo(source3.trim());
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Position: 1").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).contains("INSERT 1").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: INSERT 1 (error 1)").containsOnlyOnce("Resource: " + resource2).containsOnlyOnce("Position: 2").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source2)).contains("INSERT 2").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: INSERT 2 (error 2)").containsOnlyOnce("Resource: " + resource3).containsOnlyOnce("Position: 3").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source3)).contains("INSERT 3").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: INSERT 3 (error 3)");
    List<String> positionLines = Files.readAllLines(positions, UTF_8);
    assertThat(positionLines).contains("file:///file1.csv:1").contains("file:///file2.csv:2").contains("file:///file3.csv:3");
}