Examples with Record com.datastax.oss.dsbulk.connectors.api.Record used on opensource projects

Example 26 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method cas_load_with_errors.

/**
 * Test for CAS failures (DAT-384).
 */
@Test
void cas_load_with_errors() {
    session.execute("DROP TABLE IF EXISTS test_cas");
    session.execute("CREATE TABLE test_cas (pk int, cc int, v int, PRIMARY KEY (pk, cc))");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 1, 1)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 2, 2)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 3, 3)");
    // two failed CAS records will cause the entire batch to fail
    // will fail
    Record record1Failed = RecordUtils.mappedCSV("pk", "1", "cc", "1", "v", "1");
    // will fail
    Record record2Failed = RecordUtils.mappedCSV("pk", "1", "cc", "2", "v", "2");
    Record record3NotApplied = // will not be applied
    RecordUtils.mappedCSV("pk", "1", "cc", "4", "v", "4");
    MockConnector.mockReads(record1Failed, record2Failed, record3NotApplied);
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.query");
    args.add("INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_COMPLETED_WITH_ERRORS);
    Path bad = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos.bad")).orElse(null);
    assertThat(bad).exists();
    assertThat(FileUtils.readAllLines(bad)).containsExactly(record1Failed.getSource().toString(), record2Failed.getSource().toString(), record3NotApplied.getSource().toString());
    Path errors = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos-errors.log")).orElse(null);
    assertThat(errors).exists();
    assertThat(FileUtils.readAllLines(errors).collect(Collectors.joining("\n"))).contains(String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 1\n" + "    v: 1", record1Failed.getResource(), record1Failed.getPosition(), record1Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 2\n" + "    v: 2", record2Failed.getResource(), record2Failed.getPosition(), record2Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 4\n" + "    v: 4", record3NotApplied.getResource(), record3NotApplied.getPosition(), record3NotApplied.getSource()), "Failed conditional updates:", "\"[applied]\": false\npk: 1\ncc: 1\nv: 1", "\"[applied]\": false\npk: 1\ncc: 2\nv: 2");
    List<Row> rows = session.execute("SELECT v FROM test_cas WHERE pk = 1").all();
    assertThat(rows).hasSize(3);
    assertThat(rows.get(0).getInt(0)).isEqualTo(1);
    assertThat(rows.get(1).getInt(0)).isEqualTo(2);
    assertThat(rows.get(2).getInt(0)).isEqualTo(3);
}

Example 27 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class SearchEndToEndCCMIT method normal_unload_of_search_enabled_table.

/**
 * Test for DAT-365: regular unload of a search-enabled table should not contain the solr_query
 * column.
 */
@Test
void normal_unload_of_search_enabled_table() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search2 (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search2 WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 2, 'qix')");
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute("SELECT v FROM test_search2 WHERE solr_query = '{\"q\": \"v:foo\"}'").all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("test_search2");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(records).hasSize(3).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    }, Index.atIndex(0)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("1");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("bar");
    }, Index.atIndex(1)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("2");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("qix");
    }, Index.atIndex(2));
}

Example 28 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class UnloadWorkflow method manyWriters.

private Flux<Record> manyWriters() {
    // writeConcurrency and readConcurrency are >= 0.5C here
    int actualConcurrency = Math.min(readConcurrency, writeConcurrency);
    int numThreads = Math.min(numCores * 2, actualConcurrency);
    Scheduler scheduler = Schedulers.newParallel(numThreads, new DefaultThreadFactory("workflow"));
    schedulers.add(scheduler);
    return Flux.fromIterable(readStatements).flatMap(results -> {
        Flux<Record> records = Flux.from(executor.readReactive(results)).publishOn(scheduler, 500).transform(queryWarningsHandler).transform(totalItemsMonitor).transform(totalItemsCounter).transform(failedReadResultsMonitor).transform(failedReadsHandler).map(readResultMapper::map).transform(failedRecordsMonitor).transform(unmappableRecordsHandler);
        if (actualConcurrency == writeConcurrency) {
            records = records.transform(writer);
        } else {
            // If the actual concurrency is lesser than the connector's desired write
            // concurrency, we need to give the connector a chance to switch writers
            // frequently so that it can really redirect records to all the final destinations
            // (to that many files on disk for example). If the connector is correctly
            // implemented, each window will be redirected to a different destination
            // in a round-robin fashion.
            records = records.window(500).flatMap(window -> window.transform(writer), 1, 500);
        }
        return records.transform(failedRecordsMonitor).transform(failedRecordsHandler);
    }, actualConcurrency, 500);
}

Example 29 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class UnloadWorkflow method execute.

@Override
public boolean execute() {
    LOGGER.debug("{} started.", this);
    metricsManager.start();
    Flux<Record> flux;
    if (writeConcurrency == 1) {
        flux = oneWriter();
    } else if (writeConcurrency < numCores / 2 || readConcurrency < numCores / 2) {
        flux = fewWriters();
    } else {
        flux = manyWriters();
    }
    Stopwatch timer = Stopwatch.createStarted();
    flux.then().flux().transform(terminationHandler).blockLast();
    timer.stop();
    int totalErrors = logManager.getTotalErrors();
    metricsManager.stop(timer.elapsed(), totalErrors == 0);
    Duration elapsed = DurationUtils.round(timer.elapsed(), TimeUnit.SECONDS);
    String elapsedStr = elapsed.isZero() ? "less than one second" : DurationUtils.formatDuration(elapsed);
    if (totalErrors == 0) {
        LOGGER.info("{} completed successfully in {}.", this, elapsedStr);
    } else {
        LOGGER.warn("{} completed with {} errors in {}.", this, totalErrors, elapsedStr);
    }
    return totalErrors == 0;
}

Example 30 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class MappedStatementPrinter method appendRecord.

default void appendRecord(MappedStatement statement, StatementWriter out) {
    Record record = statement.getRecord();
    out.newLine().indent().append("Resource: ").append(String.valueOf(record.getResource())).newLine().indent().append("Position: ").append(String.valueOf(record.getPosition()));
    if (record.getSource() != null) {
        out.newLine().indent().append("Source: ").append(LogManagerUtils.formatSource(record));
    }
}