Search in sources :

Example 1 with STATUS_COMPLETED_WITH_ERRORS

use of com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method cas_load_with_errors.

/**
 * Test for CAS failures (DAT-384).
 */
@Test
void cas_load_with_errors() {
    session.execute("DROP TABLE IF EXISTS test_cas");
    session.execute("CREATE TABLE test_cas (pk int, cc int, v int, PRIMARY KEY (pk, cc))");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 1, 1)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 2, 2)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 3, 3)");
    // two failed CAS records will cause the entire batch to fail
    // will fail
    Record record1Failed = RecordUtils.mappedCSV("pk", "1", "cc", "1", "v", "1");
    // will fail
    Record record2Failed = RecordUtils.mappedCSV("pk", "1", "cc", "2", "v", "2");
    Record record3NotApplied = // will not be applied
    RecordUtils.mappedCSV("pk", "1", "cc", "4", "v", "4");
    MockConnector.mockReads(record1Failed, record2Failed, record3NotApplied);
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.query");
    args.add("INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_COMPLETED_WITH_ERRORS);
    Path bad = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos.bad")).orElse(null);
    assertThat(bad).exists();
    assertThat(FileUtils.readAllLines(bad)).containsExactly(record1Failed.getSource().toString(), record2Failed.getSource().toString(), record3NotApplied.getSource().toString());
    Path errors = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos-errors.log")).orElse(null);
    assertThat(errors).exists();
    assertThat(FileUtils.readAllLines(errors).collect(Collectors.joining("\n"))).contains(String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 1\n" + "    v: 1", record1Failed.getResource(), record1Failed.getPosition(), record1Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 2\n" + "    v: 2", record2Failed.getResource(), record2Failed.getPosition(), record2Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 4\n" + "    v: 4", record3NotApplied.getResource(), record3NotApplied.getPosition(), record3NotApplied.getSource()), "Failed conditional updates:", "\"[applied]\": false\npk: 1\ncc: 1\nv: 1", "\"[applied]\": false\npk: 1\ncc: 2\nv: 2");
    List<Row> rows = session.execute("SELECT v FROM test_cas WHERE pk = 1").all();
    assertThat(rows).hasSize(3);
    assertThat(rows.get(0).getInt(0)).isEqualTo(1);
    assertThat(rows.get(1).getInt(0)).isEqualTo(2);
    assertThat(rows.get(2).getInt(0)).isEqualTo(3);
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) Path(java.nio.file.Path) UNNECESSARY(java.math.RoundingMode.UNNECESSARY) STDOUT(com.datastax.oss.dsbulk.tests.logging.StreamType.STDOUT) BeforeEach(org.junit.jupiter.api.BeforeEach) CodecUtils(com.datastax.oss.dsbulk.codecs.api.util.CodecUtils) Strings(com.datastax.oss.driver.shaded.guava.common.base.Strings) InstanceOfAssertFactories(org.assertj.core.api.InstanceOfAssertFactories) ZonedDateTime(java.time.ZonedDateTime) INSERT_INTO_IP_BY_COUNTRY(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.INSERT_INTO_IP_BY_COUNTRY) EndToEndUtils.validateExceptionsLog(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog) InetAddress(java.net.InetAddress) AfterAll(org.junit.jupiter.api.AfterAll) BigDecimal(java.math.BigDecimal) BeforeAll(org.junit.jupiter.api.BeforeAll) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Assumptions.assumeFalse(org.junit.jupiter.api.Assumptions.assumeFalse) Map(java.util.Map) BigInteger(java.math.BigInteger) Tag(org.junit.jupiter.api.Tag) FLOOR(java.math.RoundingMode.FLOOR) Path(java.nio.file.Path) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) Set(java.util.Set) Splitter(com.datastax.oss.driver.shaded.guava.common.base.Splitter) Arguments(org.junit.jupiter.params.provider.Arguments) Stream(java.util.stream.Stream) STATUS_ABORTED_TOO_MANY_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_TOO_MANY_ERRORS) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) MICROSECONDS(java.util.concurrent.TimeUnit.MICROSECONDS) EndToEndUtils.validateNumberOfBadRecords(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords) CQLUtils(com.datastax.oss.dsbulk.tests.utils.CQLUtils) ImmutableSet(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet) STATUS_ABORTED_FATAL_ERROR(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_FATAL_ERROR) ImmutableMap(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap) ArrayList(java.util.ArrayList) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.createIpByCountryTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) Row(com.datastax.oss.driver.api.core.cql.Row) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) Files(java.nio.file.Files) TupleType(com.datastax.oss.driver.api.core.type.TupleType) DataType(com.datastax.oss.driver.api.core.type.DataType) IOException(java.io.IOException) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) UnknownHostException(java.net.UnknownHostException) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) ChronoUnit(java.time.temporal.ChronoUnit) Assumptions(org.junit.jupiter.api.Assumptions) EndToEndUtils.validateOutputFiles(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles) CqlDuration(com.datastax.oss.driver.api.core.data.CqlDuration) URL(java.net.URL) CsvUtils(com.datastax.oss.dsbulk.runner.tests.CsvUtils) MockConnector(com.datastax.oss.dsbulk.runner.tests.MockConnector) ByteBuffer(java.nio.ByteBuffer) STATUS_COMPLETED_WITH_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS) RecordUtils(com.datastax.oss.dsbulk.runner.tests.RecordUtils) Lists(com.datastax.oss.driver.shaded.guava.common.collect.Lists) Uuids(com.datastax.oss.driver.api.core.uuid.Uuids) LocalTime(java.time.LocalTime) URI(java.net.URI) Record(com.datastax.oss.dsbulk.connectors.api.Record) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) RoundingMode(java.math.RoundingMode) MethodSource(org.junit.jupiter.params.provider.MethodSource) CodecRegistry(com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry) OperationDirectory(com.datastax.oss.dsbulk.workflow.api.log.OperationDirectory) IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE) TupleValue(com.datastax.oss.driver.api.core.data.TupleValue) LineNumberReader(java.io.LineNumberReader) UUID(java.util.UUID) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) UdtValue(com.datastax.oss.driver.api.core.data.UdtValue) Objects(java.util.Objects) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) DataTypes(com.datastax.oss.driver.api.core.type.DataTypes) List(java.util.List) Type(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) LocalDate(java.time.LocalDate) TypeCodec(com.datastax.oss.driver.api.core.type.codec.TypeCodec) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) CsvSource(org.junit.jupiter.params.provider.CsvSource) EndToEndUtils.validatePositionsFile(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile) HashMap(java.util.HashMap) Version(com.datastax.oss.driver.api.core.Version) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) ImmutableList(com.google.common.collect.ImmutableList) CompressedIOUtils(com.datastax.oss.dsbulk.io.CompressedIOUtils) OverflowStrategy(com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy) FileUtils(com.datastax.oss.dsbulk.tests.utils.FileUtils) EndToEndUtils.createIpByCountryCaseSensitiveTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryCaseSensitiveTable) StringUtils.quoteJson(com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson) UTF_8(java.nio.charset.StandardCharsets.UTF_8) EndToEndUtils.createWithSpacesTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable) OSS(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type.OSS) DefaultProtocolVersion(com.datastax.oss.driver.api.core.DefaultProtocolVersion) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType) IP_BY_COUNTRY_MAPPING_INDEXED(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_INDEXED) EPOCH(java.time.Instant.EPOCH) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) Record(com.datastax.oss.dsbulk.connectors.api.Record) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) Row(com.datastax.oss.driver.api.core.cql.Row) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

CqlSession (com.datastax.oss.driver.api.core.CqlSession)1 DefaultProtocolVersion (com.datastax.oss.driver.api.core.DefaultProtocolVersion)1 Version (com.datastax.oss.driver.api.core.Version)1 ResultSet (com.datastax.oss.driver.api.core.cql.ResultSet)1 Row (com.datastax.oss.driver.api.core.cql.Row)1 CqlDuration (com.datastax.oss.driver.api.core.data.CqlDuration)1 TupleValue (com.datastax.oss.driver.api.core.data.TupleValue)1 UdtValue (com.datastax.oss.driver.api.core.data.UdtValue)1 DataType (com.datastax.oss.driver.api.core.type.DataType)1 DataTypes (com.datastax.oss.driver.api.core.type.DataTypes)1 TupleType (com.datastax.oss.driver.api.core.type.TupleType)1 UserDefinedType (com.datastax.oss.driver.api.core.type.UserDefinedType)1 TypeCodec (com.datastax.oss.driver.api.core.type.codec.TypeCodec)1 CodecRegistry (com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry)1 Uuids (com.datastax.oss.driver.api.core.uuid.Uuids)1 Splitter (com.datastax.oss.driver.shaded.guava.common.base.Splitter)1 Strings (com.datastax.oss.driver.shaded.guava.common.base.Strings)1 ImmutableMap (com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap)1 ImmutableSet (com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet)1 Lists (com.datastax.oss.driver.shaded.guava.common.collect.Lists)1