Examples with Record com.datastax.oss.dsbulk.connectors.api.Record used on opensource projects

Example 21 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class JsonConnectorTest method should_read_single_file_multi_doc.

@ParameterizedTest(name = "[{index}] read multi doc file {0} with compression {1} (sources: {2})")
@MethodSource
@DisplayName("Should read multidoc file with given compression")
void should_read_single_file_multi_doc(String fileName, String compression, boolean retainRecordSources) throws Exception {
    JsonConnector connector = new JsonConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.json", "url", url("/" + fileName), "parserFeatures", "{ALLOW_COMMENTS:true}", "deserializationFeatures", "{USE_BIG_DECIMAL_FOR_FLOATS : false}", "compression", quoteJson(compression));
    connector.configure(settings, true, retainRecordSources);
    connector.init();
    assertThat(connector.readConcurrency()).isOne();
    List<Record> actual = Flux.merge(connector.read()).collectList().block();
    verifyRecords(actual, retainRecordSources, rawURL("/" + fileName).toURI());
    connector.close();
}

Example 22 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class JsonConnectorTest method should_read_single_empty_file_single_doc.

@Test
void should_read_single_empty_file_single_doc() throws Exception {
    JsonConnector connector = new JsonConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.json", "url", url("/empty.json"), "parserFeatures", "{ALLOW_COMMENTS:true}", "mode", "SINGLE_DOCUMENT");
    connector.configure(settings, true, true);
    connector.init();
    assertThat(connector.readConcurrency()).isOne();
    // should complete with 0 records.
    List<Record> actual = Flux.merge(connector.read()).collectList().block();
    assertThat(actual).hasSize(0);
    connector.close();
}

Example 23 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_connector_errors_reached.

@Test
void should_stop_when_max_connector_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<Record> records = Flux.just(csvRecord1, csvRecord2, csvRecord3);
    try {
        records.transform(logManager.newFailedRecordsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path bad = logManager.getOperationDirectory().resolve("connector.bad");
    Path errors = logManager.getOperationDirectory().resolve("connector-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(bad.toFile()).exists();
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(bad, errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).containsOnlyOnce("Resource: " + resource1).containsOnlyOnce("Position: 1").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source1)).containsOnlyOnce("java.lang.RuntimeException: error 1").containsOnlyOnce("Resource: " + resource2).containsOnlyOnce("Position: 2").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source2)).containsOnlyOnce("java.lang.RuntimeException: error 2").containsOnlyOnce("Resource: " + resource3).containsOnlyOnce("Position: 3").containsOnlyOnce("Source: " + LogManagerUtils.formatSingleLine(source3)).containsOnlyOnce("java.lang.RuntimeException: error 3");
}

Example 24 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class MetricsManagerTest method should_increment_records.

@Test
void should_increment_records(@LogCapture(value = MetricsManager.class, level = INFO) LogInterceptor logs, @StreamCapture(STDERR) StreamInterceptor stderr) {
    try (MetricsManager manager = new MetricsManager(new MetricRegistry(), false, "test", Executors.newSingleThreadScheduledExecutor(), SECONDS, MILLISECONDS, -1, -1, true, false, false, true, null, null, LogSettings.Verbosity.normal, Duration.ofSeconds(5), false, protocolVersion, codecRegistry, RowType.REGULAR)) {
        manager.init();
        manager.start();
        Flux<Record> records = Flux.just(record1, record2, record3);
        records.transform(manager.newTotalItemsMonitor()).transform(manager.newFailedItemsMonitor()).blockLast();
        manager.stop(Duration.ofSeconds(123), true);
        MetricRegistry registry = (MetricRegistry) ReflectionUtils.getInternalState(manager, "registry");
        assertThat(registry.counter("records/total").getCount()).isEqualTo(3);
        assertThat(registry.counter("records/failed").getCount()).isEqualTo(1);
        assertThat(logs.getLoggedEvents()).isEmpty();
        assertThat(stderr.getStreamLinesPlain()).anySatisfy(line -> assertThat(line).startsWith("    3 |      1 |"));
    }
}

Example 25 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class PrometheusEndToEndSimulacronIT method newConnectorDelegate.

@NonNull
private Connector newConnectorDelegate() {
    return new Connector() {

        @Override
        public int readConcurrency() {
            return 1;
        }

        @Override
        public boolean supports(@NonNull ConnectorFeature feature) {
            return feature == CommonConnectorFeature.INDEXED_RECORDS;
        }

        @NonNull
        @Override
        public Publisher<Publisher<Record>> read() {
            AtomicInteger counter = new AtomicInteger();
            AtomicBoolean running = new AtomicBoolean(true);
            return Flux.just(Flux.generate(sink -> {
                int i = counter.getAndAdd(1);
                if (i == 0) {
                    startTimer(running);
                }
                if (running.get()) {
                    Record record = RecordUtils.indexedCSV("pk", "pk" + 1, "cc", "cc" + 1, "v", "v" + 1);
                    sink.next(record);
                } else {
                    sink.complete();
                }
            }));
        }

        @NonNull
        @Override
        public RecordMetadata getRecordMetadata() {
            return (fieldType, cqlType) -> GenericType.STRING;
        }

        @NonNull
        @Override
        public Function<Publisher<Record>, Publisher<Record>> write() {
            throw new UnsupportedOperationException();
        }

        @Override
        public int writeConcurrency() {
            throw new UnsupportedOperationException();
        }
    };
}