Examples with DefaultWriteResult com.datastax.oss.dsbulk.executor.api.result.DefaultWriteResult used on opensource projects

Example 1 with DefaultWriteResult

use of com.datastax.oss.dsbulk.executor.api.result.DefaultWriteResult in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_cas_errors_reached.

@Test
void should_stop_when_max_cas_errors_reached() throws Exception {
    BatchStatement casBatch = BatchStatement.newInstance(DefaultBatchType.UNLOGGED, mockBulkBoundStatement(1, source1, resource1), mockBulkBoundStatement(2, source2, resource2), mockBulkBoundStatement(3, source3, resource3));
    Row row1 = mockRow(1);
    Row row2 = mockRow(2);
    Row row3 = mockRow(3);
    AsyncResultSet rs = mock(AsyncResultSet.class);
    when(rs.wasApplied()).thenReturn(false);
    when(rs.currentPage()).thenReturn(Lists.newArrayList(row1, row2, row3));
    ExecutionInfo executionInfo = mock(ExecutionInfo.class);
    when(rs.getExecutionInfo()).thenReturn(executionInfo);
    DefaultWriteResult casBatchWriteResult = new DefaultWriteResult(casBatch, rs);
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<WriteResult> stmts = Flux.just(casBatchWriteResult);
    try {
        stmts.transform(logManager.newFailedWritesHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("paxos.bad");
    Path errors = logManager.getOperationDirectory().resolve("paxos-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(3);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(badLines.get(1)).isEqualTo(source2.trim());
    assertThat(badLines.get(2)).isEqualTo(source3.trim());
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Failed conditional updates: ").containsOnlyOnce("INSERT INTO 1").contains("c1: 1").containsOnlyOnce("INSERT INTO 2").contains("c1: 2").containsOnlyOnce("INSERT INTO 3").contains("c1: 3");
}

Example 2 with DefaultWriteResult

use of com.datastax.oss.dsbulk.executor.api.result.DefaultWriteResult in project dsbulk by datastax.

the class LogManagerTest method should_log_query_warnings_when_writing.

@Test
void should_log_query_warnings_when_writing(@LogCapture(value = LogManager.class, level = WARN) LogInterceptor logs) throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(100), ErrorThreshold.forAbsoluteValue(1), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    ExecutionInfo info1 = mock(ExecutionInfo.class);
    when(info1.getWarnings()).thenReturn(ImmutableList.of("warning1", "warning2"));
    AsyncResultSet rs1 = mock(AsyncResultSet.class);
    when(rs1.getExecutionInfo()).thenReturn(info1);
    ExecutionInfo info2 = mock(ExecutionInfo.class);
    when(info2.getWarnings()).thenReturn(ImmutableList.of("warning3"));
    AsyncResultSet rs2 = mock(AsyncResultSet.class);
    when(rs2.getExecutionInfo()).thenReturn(info2);
    Flux.just(new DefaultWriteResult(SimpleStatement.newInstance("SELECT 1"), rs1), new DefaultWriteResult(SimpleStatement.newInstance("SELECT 2"), rs2)).transform(logManager.newQueryWarningsHandler()).blockLast();
    logManager.close();
    assertThat(logs).hasMessageContaining("Query generated server-side warning: warning1").doesNotHaveMessageContaining("warning2").doesNotHaveMessageContaining("warning3").hasMessageContaining("The maximum number of logged query warnings has been exceeded (1); " + "subsequent warnings will not be logged.");
}

Example 3 with DefaultWriteResult

use of com.datastax.oss.dsbulk.executor.api.result.DefaultWriteResult in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_unrecoverable_error_writing.

@Test
void should_stop_when_unrecoverable_error_writing() throws Exception {
    Path outputDir = Files.createTempDirectory("test4");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(1000), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    DefaultWriteResult result = new DefaultWriteResult(new BulkExecutionException(new DriverExecutionException(new IllegalArgumentException("error 1")), new MappedBoundStatement(csvRecord1, mockBoundStatement("INSERT 1"))));
    Flux<WriteResult> stmts = Flux.just(result);
    try {
        stmts.transform(logManager.newFailedWritesHandler()).blockLast();
        fail("Expecting DriverExecutionException to be thrown");
    } catch (DriverExecutionException e) {
        assertThat(e.getCause()).isInstanceOf(IllegalArgumentException.class).hasMessage("error 1");
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("load.bad");
    Path errors = logManager.getOperationDirectory().resolve("load-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    List<String> badLines = Files.readAllLines(bad, UTF_8);
    assertThat(badLines).hasSize(1);
    assertThat(badLines.get(0)).isEqualTo(source1.trim());
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).contains("Position: 1").contains("INSERT 1").contains("error 1").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: INSERT 1");
    List<String> positionLines = Files.readAllLines(positions, UTF_8);
    assertThat(positionLines).contains("file:///file1.csv:1");
}

Example 4 with DefaultWriteResult

use of com.datastax.oss.dsbulk.executor.api.result.DefaultWriteResult in project dsbulk by datastax.

the class LogManagerTest method setUp.

@BeforeEach
void setUp() throws Exception {
    session = mockSession();
    resource1 = new URI("file:///file1.csv");
    resource2 = new URI("file:///file2.csv");
    resource3 = new URI("file:///file3.csv");
    csvRecord1 = new DefaultErrorRecord(source1, resource1, 1, new RuntimeException("error 1"));
    csvRecord2 = new DefaultErrorRecord(source2, resource2, 2, new RuntimeException("error 2"));
    csvRecord3 = new DefaultErrorRecord(source3, resource3, 3, new RuntimeException("error 3"));
    unmappableStmt1 = new UnmappableStatement(csvRecord1, new RuntimeException("error 1"));
    unmappableStmt2 = new UnmappableStatement(csvRecord2, new RuntimeException("error 2"));
    unmappableStmt3 = new UnmappableStatement(csvRecord3, new RuntimeException("error 3"));
    failedWriteResult1 = new DefaultWriteResult(new BulkExecutionException(new DriverTimeoutException("error 1"), new MappedBoundStatement(csvRecord1, mockBoundStatement("INSERT 1"))));
    failedWriteResult2 = new DefaultWriteResult(new BulkExecutionException(new DriverTimeoutException("error 2"), new MappedBoundStatement(csvRecord2, mockBoundStatement("INSERT 2"))));
    failedWriteResult3 = new DefaultWriteResult(new BulkExecutionException(new DriverTimeoutException("error 3"), new MappedBoundStatement(csvRecord3, mockBoundStatement("INSERT 3"))));
    failedReadResult1 = new DefaultReadResult(new BulkExecutionException(new DriverTimeoutException("error 1"), mockBoundStatement("SELECT 1")));
    failedReadResult2 = new DefaultReadResult(new BulkExecutionException(new DriverTimeoutException("error 2"), mockBoundStatement("SELECT 2")));
    failedReadResult3 = new DefaultReadResult(new BulkExecutionException(new DriverTimeoutException("error 3"), mockBoundStatement("SELECT 3")));
    BatchStatement batch = BatchStatement.newInstance(DefaultBatchType.UNLOGGED, new MappedBoundStatement(csvRecord1, mockBoundStatement("INSERT 1", "foo", 42)), new MappedBoundStatement(csvRecord2, mockBoundStatement("INSERT 2", "bar", 43)), new MappedBoundStatement(csvRecord3, mockBoundStatement("INSERT 3", "qix", 44)));
    batchWriteResult = new DefaultWriteResult(new BulkExecutionException(new DriverTimeoutException("error batch"), batch));
    ExecutionInfo info = mock(ExecutionInfo.class);
    row1 = mockRow(1);
    Row row2 = mockRow(2);
    Row row3 = mockRow(3);
    Statement<?> stmt1 = SimpleStatement.newInstance("SELECT 1");
    Statement<?> stmt2 = SimpleStatement.newInstance("SELECT 2");
    Statement<?> stmt3 = SimpleStatement.newInstance("SELECT 3");
    successfulReadResult1 = new DefaultReadResult(stmt1, info, row1);
    ReadResult successfulReadResult2 = new DefaultReadResult(stmt2, info, row2);
    ReadResult successfulReadResult3 = new DefaultReadResult(stmt3, info, row3);
    rowRecord1 = new DefaultErrorRecord(successfulReadResult1, tableResource, 1, new RuntimeException("error 1"));
    rowRecord2 = new DefaultErrorRecord(successfulReadResult2, tableResource, 2, new RuntimeException("error 2"));
    rowRecord3 = new DefaultErrorRecord(successfulReadResult3, tableResource, 3, new RuntimeException("error 3"));
}