Examples with BulkExecutionException com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException used on opensource projects

Example 1 with BulkExecutionException

use of com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException in project dsbulk by datastax.

the class MetricsCollectingExecutionListenerTest method should_collect_metrics.

@Test
void should_collect_metrics() {
    MetricsCollectingExecutionListener listener = new MetricsCollectingExecutionListener();
    ExecutionContext global = new TestExecutionContext();
    ExecutionContext local1 = new TestExecutionContext();
    ExecutionContext local2 = new TestExecutionContext();
    ExecutionContext local3 = new TestExecutionContext();
    ExecutionContext local4 = new TestExecutionContext();
    listener.onExecutionStarted(successfulRead, global);
    listener.onExecutionStarted(failedRead, global);
    listener.onExecutionStarted(successfulWrite, global);
    listener.onExecutionStarted(failedWrite, global);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(0);
    assertThat(listener.getBytesSentMeter().get().getCount()).isEqualTo(0);
    assertThat(listener.getBytesReceivedMeter().get().getCount()).isEqualTo(0);
    listener.onReadRequestStarted(successfulRead, local1);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(1);
    listener.onReadRequestStarted(failedRead, local2);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(2);
    listener.onWriteRequestStarted(successfulWrite, local3);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(3);
    assertThat(listener.getBytesSentMeter().get().getCount()).isEqualTo(8);
    listener.onWriteRequestStarted(failedWrite, local4);
    assertThat(listener.getBytesSentMeter().get().getCount()).isEqualTo(16);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(4);
    listener.onReadRequestSuccessful(successfulRead, local1);
    // simulate 3 rows received
    listener.onRowReceived(row, local1);
    assertThat(listener.getBytesReceivedMeter().get().getCount()).isEqualTo(4);
    listener.onRowReceived(row, local1);
    assertThat(listener.getBytesReceivedMeter().get().getCount()).isEqualTo(8);
    listener.onRowReceived(row, local1);
    assertThat(listener.getBytesReceivedMeter().get().getCount()).isEqualTo(12);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(3);
    listener.onReadRequestFailed(failedRead, new RuntimeException(), local2);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(2);
    listener.onWriteRequestSuccessful(successfulWrite, local3);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(1);
    listener.onWriteRequestFailed(failedWrite, new RuntimeException(), local4);
    assertThat(listener.getInFlightRequestsCounter().getCount()).isEqualTo(0);
    listener.onExecutionSuccessful(successfulRead, global);
    listener.onExecutionFailed(new BulkExecutionException(new RuntimeException(), failedRead), global);
    listener.onExecutionSuccessful(successfulWrite, global);
    listener.onExecutionFailed(new BulkExecutionException(new RuntimeException(), failedWrite), global);
    // 3 successful reads
    // 1 failed read
    // 2 successful writes
    // 2 failed writes
    assertThat(listener.getTotalStatementsTimer().getCount()).isEqualTo(4);
    assertThat(listener.getFailedStatementsCounter().getCount()).isEqualTo(2);
    assertThat(listener.getSuccessfulStatementsCounter().getCount()).isEqualTo(2);
    assertThat(listener.getTotalReadsWritesTimer().getCount()).isEqualTo(8);
    assertThat(listener.getFailedReadsWritesCounter().getCount()).isEqualTo(3);
    assertThat(listener.getSuccessfulReadsWritesCounter().getCount()).isEqualTo(5);
    assertThat(listener.getTotalWritesTimer().getCount()).isEqualTo(4);
    assertThat(listener.getFailedWritesCounter().getCount()).isEqualTo(2);
    assertThat(listener.getSuccessfulWritesCounter().getCount()).isEqualTo(2);
    assertThat(listener.getTotalReadsTimer().getCount()).isEqualTo(4);
    assertThat(listener.getFailedReadsCounter().getCount()).isEqualTo(1);
    assertThat(listener.getSuccessfulReadsCounter().getCount()).isEqualTo(3);
}

Example 2 with BulkExecutionException

use of com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException in project dsbulk by datastax.

the class BulkExecutorITBase method verifyReads.

protected void verifyReads(int expectedSuccessful, int expectedFailed, Iterable<ReadResult> actual) {
    AtomicInteger i = new AtomicInteger();
    long actualSuccessful = Flux.fromIterable(actual).filter(Result::isSuccess).map(result -> result.getRow().orElseThrow(AssertionError::new)).map(row -> row.getInt("pk")).sort().doOnNext(pk -> {
        assertThat(pk).isEqualTo(i.get());
        i.getAndIncrement();
    }).count().blockOptional().orElse(0L);
    assertThat(actualSuccessful).isEqualTo(expectedSuccessful);
    long actualFailed = Flux.fromIterable(actual).filter(r -> !r.isSuccess()).doOnNext(r -> {
        assertThat(r.getRow().isPresent()).isFalse();
        assertThat(r.getError().isPresent()).isTrue();
        BulkExecutionException error = r.getError().get();
        verifyException(error);
    }).count().blockOptional().orElse(0L);
    assertThat(actualFailed).isEqualTo(expectedFailed);
}

Example 3 with BulkExecutionException

use of com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_unrecoverable_error_reading.

@Test
void should_stop_when_unrecoverable_error_reading() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    DefaultReadResult result = new DefaultReadResult(new BulkExecutionException(new DriverExecutionException(new IllegalArgumentException("error 1")), mockBoundStatement("SELECT 1")));
    Flux<ReadResult> stmts = Flux.just(result);
    try {
        stmts.transform(logManager.newFailedReadsHandler()).blockLast();
        fail("Expecting DriverExecutionException to be thrown");
    } catch (DriverExecutionException e) {
        assertThat(e.getCause()).isInstanceOf(IllegalArgumentException.class).hasMessage("error 1");
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("unload-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).doesNotContain("Resource: ").doesNotContain("Source: ").doesNotContain("Position: ").contains("SELECT 1").contains("error 1").containsOnlyOnce("com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException: Statement execution failed: SELECT 1");
}

Example 4 with BulkExecutionException

use of com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_sample_size_is_met_and_percentage_exceeded.

@Test
void should_stop_when_sample_size_is_met_and_percentage_exceeded() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forRatio(0.01f, 100), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<ReadResult> stmts = Flux.just(failedReadResult1);
    try {
        stmts.repeat(101).transform(logManager.newTotalItemsCounter()).transform(logManager.newFailedReadsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 1%.");
        Assertions.assertThat(((RatioErrorThreshold) e.getThreshold()).getMaxErrorRatio()).isEqualTo(0.01f);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("unload-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    assertThat(lines.stream().filter(l -> l.contains("BulkExecutionException")).count()).isEqualTo(100);
}

Example 5 with BulkExecutionException

use of com.datastax.oss.dsbulk.executor.api.exception.BulkExecutionException in project dsbulk by datastax.

the class MetricsCollectingExecutionListenerTest method should_not_collect_throughput_metrics.

@Test
void should_not_collect_throughput_metrics() {
    MetricsCollectingExecutionListener listener = new MetricsCollectingExecutionListener(new MetricRegistry(), ProtocolVersion.DEFAULT, CodecRegistry.DEFAULT, false);
    assertThat(listener.getBytesSentMeter()).isNotPresent();
    assertThat(listener.getBytesReceivedMeter()).isNotPresent();
    // 3 successful reads
    // 1 failed read
    // 2 successful writes
    // 2 failed writes
    ExecutionContext global = new TestExecutionContext();
    ExecutionContext local1 = new TestExecutionContext();
    ExecutionContext local2 = new TestExecutionContext();
    ExecutionContext local3 = new TestExecutionContext();
    ExecutionContext local4 = new TestExecutionContext();
    listener.onExecutionStarted(successfulRead, global);
    listener.onExecutionStarted(failedRead, global);
    listener.onExecutionStarted(successfulWrite, global);
    listener.onExecutionStarted(failedWrite, global);
    listener.onReadRequestStarted(successfulRead, local1);
    listener.onReadRequestStarted(failedRead, local2);
    listener.onWriteRequestStarted(successfulWrite, local3);
    listener.onWriteRequestStarted(failedWrite, local4);
    listener.onReadRequestSuccessful(successfulRead, local1);
    listener.onRowReceived(row, local1);
    listener.onRowReceived(row, local1);
    listener.onRowReceived(row, local1);
    listener.onReadRequestFailed(failedRead, new RuntimeException(), local2);
    listener.onWriteRequestSuccessful(successfulWrite, local3);
    listener.onWriteRequestFailed(failedWrite, new RuntimeException(), local4);
    listener.onExecutionSuccessful(successfulRead, global);
    listener.onExecutionFailed(new BulkExecutionException(new RuntimeException(), failedRead), global);
    listener.onExecutionSuccessful(successfulWrite, global);
    listener.onExecutionFailed(new BulkExecutionException(new RuntimeException(), failedWrite), global);
    assertThat(listener.getBytesSentMeter()).isNotPresent();
    assertThat(listener.getBytesReceivedMeter()).isNotPresent();
}