Search in sources :

Example 1 with StringUtils.quoteJson

use of com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method full_load_query_warnings.

/**
 * Test for DAT-451.
 */
@Test
void full_load_query_warnings() throws Exception {
    assumeTrue(ccm.getCassandraVersion().compareTo(V3) >= 0, "Query warnings are only present in C* >= 3.0");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("json");
    args.add("--log.maxQueryWarnings");
    args.add("1");
    args.add("--connector.json.url");
    args.add(StringUtils.quoteJson(JsonUtils.JSON_RECORDS));
    args.add("--batch.mode");
    args.add("REPLICA_SET");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("ip_by_country");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(500, "SELECT * FROM ip_by_country");
    validatePositionsFile(JsonUtils.JSON_RECORDS, 500);
    /*
    Unlogged batch covering N partitions detected against table [ks1.ip_by_country].
    You should use a logged batch for atomicity, or asynchronous writes for performance.
    DSE 6.0+:
    Unlogged batch covering 20 partitions detected against table {ks1.ip_by_country}.
    You should use a logged batch for atomicity, or asynchronous writes for performance.
     */
    assertThat(logs).hasMessageContaining("Query generated server-side warning").hasMessageMatching("Unlogged batch covering \\d+ partitions detected").hasMessageContaining(session.getKeyspace().get().asCql(true) + ".ip_by_country").hasMessageContaining("The maximum number of logged query warnings has been exceeded (1); " + "subsequent warnings will not be logged.");
    assertThat(stderr.getStreamLinesPlain()).anySatisfy(line -> assertThat(line).contains("Query generated server-side warning")).anySatisfy(line -> assertThat(line).containsPattern("Unlogged batch covering \\d+ partitions detected")).anySatisfy(line -> assertThat(line).contains(session.getKeyspace().get().asCql(true) + ".ip_by_country")).anySatisfy(line -> assertThat(line).contains("The maximum number of logged query warnings has been exceeded (1); " + "subsequent warnings will not be logged."));
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) CSVConnectorEndToEndCCMIT.assertComplexRows(com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.assertComplexRows) UNNECESSARY(java.math.RoundingMode.UNNECESSARY) CqlIdentifier(com.datastax.oss.driver.api.core.CqlIdentifier) STATUS_COMPLETED_WITH_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS) EndToEndUtils.validateExceptionsLog(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog) AfterAll(org.junit.jupiter.api.AfterAll) Matcher(java.util.regex.Matcher) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Assertions(org.assertj.core.api.Assertions) Tag(org.junit.jupiter.api.Tag) Path(java.nio.file.Path) IP_BY_COUNTRY_MAPPING_NAMED(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_NAMED) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) StringUtils(com.datastax.oss.dsbulk.tests.utils.StringUtils) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) List(java.util.List) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) CSVConnectorEndToEndCCMIT.checkNumbersWritten(com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.checkNumbersWritten) Pattern(java.util.regex.Pattern) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) EndToEndUtils.validateNumberOfBadRecords(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) EndToEndUtils.validatePositionsFile(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Version(com.datastax.oss.driver.api.core.Version) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.createIpByCountryTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) OverflowStrategy(com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy) Row(com.datastax.oss.driver.api.core.cql.Row) FileUtils(com.datastax.oss.dsbulk.tests.utils.FileUtils) Files(java.nio.file.Files) StringUtils.quoteJson(com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson) EndToEndUtils.createWithSpacesTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable) IOException(java.io.IOException) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CSVConnectorEndToEndCCMIT.checkTemporalsWritten(com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.checkTemporalsWritten) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) Bytes(com.datastax.oss.protocol.internal.util.Bytes) AfterEach(org.junit.jupiter.api.AfterEach) EndToEndUtils.validateOutputFiles(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles) JsonUtils(com.datastax.oss.dsbulk.runner.tests.JsonUtils) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) Test(org.junit.jupiter.api.Test)

Aggregations

CqlIdentifier (com.datastax.oss.driver.api.core.CqlIdentifier)1 CqlSession (com.datastax.oss.driver.api.core.CqlSession)1 Version (com.datastax.oss.driver.api.core.Version)1 Row (com.datastax.oss.driver.api.core.cql.Row)1 OverflowStrategy (com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy)1 DataStaxBulkLoader (com.datastax.oss.dsbulk.runner.DataStaxBulkLoader)1 ExitStatus (com.datastax.oss.dsbulk.runner.ExitStatus)1 STATUS_COMPLETED_WITH_ERRORS (com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS)1 STATUS_OK (com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK)1 CSVConnectorEndToEndCCMIT.assertComplexRows (com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.assertComplexRows)1 CSVConnectorEndToEndCCMIT.checkNumbersWritten (com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.checkNumbersWritten)1 CSVConnectorEndToEndCCMIT.checkTemporalsWritten (com.datastax.oss.dsbulk.runner.ccm.CSVConnectorEndToEndCCMIT.checkTemporalsWritten)1 IP_BY_COUNTRY_MAPPING_NAMED (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_NAMED)1 EndToEndUtils.assertStatus (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus)1 EndToEndUtils.createIpByCountryTable (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable)1 EndToEndUtils.createWithSpacesTable (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable)1 EndToEndUtils.validateExceptionsLog (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog)1 EndToEndUtils.validateNumberOfBadRecords (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords)1 EndToEndUtils.validateOutputFiles (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles)1 EndToEndUtils.validatePositionsFile (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile)1