Search in sources :

Example 1 with Type

use of com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method unload_load_preserving_ttl_and_timestamp.

@SuppressWarnings("unused")
Stream<Arguments> unload_load_preserving_ttl_and_timestamp() throws UnknownHostException {
    session.execute("CREATE TYPE IF NOT EXISTS preserve_ttl_timestamp_udt (f1 int, f2 text)");
    TupleType tupleType = DataTypes.tupleOf(DataTypes.INT, DataTypes.TEXT);
    UserDefinedType udt = session.getKeyspace().flatMap(ks -> session.getMetadata().getKeyspace(ks)).flatMap(ks -> ks.getUserDefinedType("preserve_ttl_timestamp_udt")).orElseThrow(IllegalStateException::new);
    return Stream.of(Arguments.of(DataTypes.ASCII, "abc", "def", "abc", "def"), Arguments.of(DataTypes.BIGINT, 123L, 456L, "123", "456"), Arguments.of(DataTypes.BLOB, ByteBuffer.wrap(new byte[] { 1, 2 }), ByteBuffer.wrap(new byte[] { 3, 4 }), "AQI=", "AwQ="), Arguments.of(DataTypes.BOOLEAN, true, false, "1", "0"), // Arguments.of(DataTypes.COUNTER, v1, v2, csv1, csv2),
    Arguments.of(DataTypes.DECIMAL, new BigDecimal("12.34"), new BigDecimal("56.78"), "12.34", "56.78"), Arguments.of(DataTypes.DOUBLE, 12.34, 56.78, "12.34", "56.78"), Arguments.of(DataTypes.FLOAT, 12.34f, 56.78f, "12.34", "56.78"), Arguments.of(DataTypes.INT, 12, 34, "12", "34"), Arguments.of(DataTypes.TIMESTAMP, Instant.parse("2021-01-11T11:01:01Z"), Instant.parse("2022-02-22T22:02:02Z"), "2021-01-11T11:01:01Z", "2022-02-22T22:02:02Z"), Arguments.of(DataTypes.UUID, Uuids.startOf(123456789), Uuids.endOf(123456789), Uuids.startOf(123456789).toString(), Uuids.endOf(123456789).toString()), Arguments.of(DataTypes.VARINT, new BigInteger("1234"), new BigInteger("5678"), "1234", "5678"), Arguments.of(DataTypes.TIMEUUID, Uuids.startOf(123456789), Uuids.endOf(123456789), Uuids.startOf(123456789).toString(), Uuids.endOf(123456789).toString()), Arguments.of(DataTypes.INET, InetAddress.getByAddress(new byte[] { 1, 2, 3, 4 }), InetAddress.getByAddress(new byte[] { 5, 6, 7, 8 }), "1.2.3.4", "5.6.7.8"), Arguments.of(DataTypes.DATE, LocalDate.of(2021, 1, 11), LocalDate.of(2022, 2, 22), "2021-01-11", "2022-02-22"), Arguments.of(DataTypes.TEXT, "abc", "def", "abc", "def"), Arguments.of(DataTypes.TIME, LocalTime.of(1, 2, 3), LocalTime.of(4, 5, 6), "01:02:03", "04:05:06"), Arguments.of(DataTypes.SMALLINT, (short) 12, (short) 34, "12", "34"), Arguments.of(DataTypes.TINYINT, (byte) 12, (byte) 34, "12", "34"), Arguments.of(DataTypes.DURATION, CqlDuration.newInstance(1, 2, 34), CqlDuration.newInstance(5, 6, 78), "1mo2d34ns", "5mo6d78ns"), Arguments.of(tupleType, tupleType.newValue(12, "ab"), tupleType.newValue(34, "cd"), "[12,\\\"ab\\\"]", "[34,\\\"cd\\\"]"), Arguments.of(// only frozen UDTs work
    udt.copy(true), udt.newValue(12, "ab"), udt.newValue(34, "cd"), "{\\\"f1\\\":12,\\\"f2\\\":\\\"ab\\\"}", "{\\\"f1\\\":34,\\\"f2\\\":\\\"cd\\\"}"));
}
Also used : UNNECESSARY(java.math.RoundingMode.UNNECESSARY) STDOUT(com.datastax.oss.dsbulk.tests.logging.StreamType.STDOUT) BeforeEach(org.junit.jupiter.api.BeforeEach) CodecUtils(com.datastax.oss.dsbulk.codecs.api.util.CodecUtils) Strings(com.datastax.oss.driver.shaded.guava.common.base.Strings) InstanceOfAssertFactories(org.assertj.core.api.InstanceOfAssertFactories) ZonedDateTime(java.time.ZonedDateTime) INSERT_INTO_IP_BY_COUNTRY(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.INSERT_INTO_IP_BY_COUNTRY) EndToEndUtils.validateExceptionsLog(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog) InetAddress(java.net.InetAddress) AfterAll(org.junit.jupiter.api.AfterAll) BigDecimal(java.math.BigDecimal) BeforeAll(org.junit.jupiter.api.BeforeAll) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Assumptions.assumeFalse(org.junit.jupiter.api.Assumptions.assumeFalse) Map(java.util.Map) BigInteger(java.math.BigInteger) Tag(org.junit.jupiter.api.Tag) FLOOR(java.math.RoundingMode.FLOOR) Path(java.nio.file.Path) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) Set(java.util.Set) Splitter(com.datastax.oss.driver.shaded.guava.common.base.Splitter) Arguments(org.junit.jupiter.params.provider.Arguments) Stream(java.util.stream.Stream) STATUS_ABORTED_TOO_MANY_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_TOO_MANY_ERRORS) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) MICROSECONDS(java.util.concurrent.TimeUnit.MICROSECONDS) EndToEndUtils.validateNumberOfBadRecords(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords) CQLUtils(com.datastax.oss.dsbulk.tests.utils.CQLUtils) ImmutableSet(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet) STATUS_ABORTED_FATAL_ERROR(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_FATAL_ERROR) ImmutableMap(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap) ArrayList(java.util.ArrayList) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.createIpByCountryTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) Row(com.datastax.oss.driver.api.core.cql.Row) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) Files(java.nio.file.Files) TupleType(com.datastax.oss.driver.api.core.type.TupleType) DataType(com.datastax.oss.driver.api.core.type.DataType) IOException(java.io.IOException) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) UnknownHostException(java.net.UnknownHostException) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) ChronoUnit(java.time.temporal.ChronoUnit) Assumptions(org.junit.jupiter.api.Assumptions) EndToEndUtils.validateOutputFiles(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles) CqlDuration(com.datastax.oss.driver.api.core.data.CqlDuration) URL(java.net.URL) CsvUtils(com.datastax.oss.dsbulk.runner.tests.CsvUtils) MockConnector(com.datastax.oss.dsbulk.runner.tests.MockConnector) ByteBuffer(java.nio.ByteBuffer) STATUS_COMPLETED_WITH_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS) RecordUtils(com.datastax.oss.dsbulk.runner.tests.RecordUtils) Lists(com.datastax.oss.driver.shaded.guava.common.collect.Lists) Uuids(com.datastax.oss.driver.api.core.uuid.Uuids) LocalTime(java.time.LocalTime) URI(java.net.URI) Record(com.datastax.oss.dsbulk.connectors.api.Record) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) RoundingMode(java.math.RoundingMode) MethodSource(org.junit.jupiter.params.provider.MethodSource) CodecRegistry(com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry) OperationDirectory(com.datastax.oss.dsbulk.workflow.api.log.OperationDirectory) IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE) TupleValue(com.datastax.oss.driver.api.core.data.TupleValue) LineNumberReader(java.io.LineNumberReader) UUID(java.util.UUID) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) UdtValue(com.datastax.oss.driver.api.core.data.UdtValue) Objects(java.util.Objects) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) DataTypes(com.datastax.oss.driver.api.core.type.DataTypes) List(java.util.List) Type(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) LocalDate(java.time.LocalDate) TypeCodec(com.datastax.oss.driver.api.core.type.codec.TypeCodec) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) CsvSource(org.junit.jupiter.params.provider.CsvSource) EndToEndUtils.validatePositionsFile(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile) HashMap(java.util.HashMap) Version(com.datastax.oss.driver.api.core.Version) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) ImmutableList(com.google.common.collect.ImmutableList) CompressedIOUtils(com.datastax.oss.dsbulk.io.CompressedIOUtils) OverflowStrategy(com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy) FileUtils(com.datastax.oss.dsbulk.tests.utils.FileUtils) EndToEndUtils.createIpByCountryCaseSensitiveTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryCaseSensitiveTable) StringUtils.quoteJson(com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson) UTF_8(java.nio.charset.StandardCharsets.UTF_8) EndToEndUtils.createWithSpacesTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable) OSS(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type.OSS) DefaultProtocolVersion(com.datastax.oss.driver.api.core.DefaultProtocolVersion) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType) IP_BY_COUNTRY_MAPPING_INDEXED(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_INDEXED) EPOCH(java.time.Instant.EPOCH) InputStream(java.io.InputStream) TupleType(com.datastax.oss.driver.api.core.type.TupleType) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType) BigInteger(java.math.BigInteger) BigDecimal(java.math.BigDecimal)

Example 2 with Type

use of com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method unload_load_preserving_ttl_and_timestamp_unsupported_types.

@SuppressWarnings("unused")
Stream<Arguments> unload_load_preserving_ttl_and_timestamp_unsupported_types() {
    session.execute("CREATE TYPE IF NOT EXISTS preserve_ttl_timestamp_udt (f1 int, f2 text)");
    UserDefinedType udt = session.getKeyspace().flatMap(ks -> session.getMetadata().getKeyspace(ks)).flatMap(ks -> ks.getUserDefinedType("preserve_ttl_timestamp_udt")).orElseThrow(IllegalStateException::new);
    return Stream.of(Arguments.of(DataTypes.listOf(DataTypes.INT), ImmutableList.of(12, 34)), Arguments.of(DataTypes.setOf(DataTypes.INT), ImmutableSet.of(12, 34)), Arguments.of(DataTypes.mapOf(DataTypes.INT, DataTypes.TEXT), ImmutableMap.of(12, "ab", 34, "cd")), Arguments.of(DataTypes.frozenListOf(DataTypes.INT), ImmutableList.of(12, 34)), Arguments.of(DataTypes.frozenSetOf(DataTypes.INT), ImmutableSet.of(12, 34)), Arguments.of(DataTypes.frozenMapOf(DataTypes.INT, DataTypes.TEXT), ImmutableMap.of(12, "ab", 34, "cd")), // non-frozen UDT
    Arguments.of(udt, udt.newValue(12, "ab")));
}
Also used : UNNECESSARY(java.math.RoundingMode.UNNECESSARY) STDOUT(com.datastax.oss.dsbulk.tests.logging.StreamType.STDOUT) BeforeEach(org.junit.jupiter.api.BeforeEach) CodecUtils(com.datastax.oss.dsbulk.codecs.api.util.CodecUtils) Strings(com.datastax.oss.driver.shaded.guava.common.base.Strings) InstanceOfAssertFactories(org.assertj.core.api.InstanceOfAssertFactories) ZonedDateTime(java.time.ZonedDateTime) INSERT_INTO_IP_BY_COUNTRY(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.INSERT_INTO_IP_BY_COUNTRY) EndToEndUtils.validateExceptionsLog(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog) InetAddress(java.net.InetAddress) AfterAll(org.junit.jupiter.api.AfterAll) BigDecimal(java.math.BigDecimal) BeforeAll(org.junit.jupiter.api.BeforeAll) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Assumptions.assumeFalse(org.junit.jupiter.api.Assumptions.assumeFalse) Map(java.util.Map) BigInteger(java.math.BigInteger) Tag(org.junit.jupiter.api.Tag) FLOOR(java.math.RoundingMode.FLOOR) Path(java.nio.file.Path) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) Set(java.util.Set) Splitter(com.datastax.oss.driver.shaded.guava.common.base.Splitter) Arguments(org.junit.jupiter.params.provider.Arguments) Stream(java.util.stream.Stream) STATUS_ABORTED_TOO_MANY_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_TOO_MANY_ERRORS) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) MICROSECONDS(java.util.concurrent.TimeUnit.MICROSECONDS) EndToEndUtils.validateNumberOfBadRecords(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords) CQLUtils(com.datastax.oss.dsbulk.tests.utils.CQLUtils) ImmutableSet(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet) STATUS_ABORTED_FATAL_ERROR(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_FATAL_ERROR) ImmutableMap(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap) ArrayList(java.util.ArrayList) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.createIpByCountryTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) Row(com.datastax.oss.driver.api.core.cql.Row) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) Files(java.nio.file.Files) TupleType(com.datastax.oss.driver.api.core.type.TupleType) DataType(com.datastax.oss.driver.api.core.type.DataType) IOException(java.io.IOException) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) UnknownHostException(java.net.UnknownHostException) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) ChronoUnit(java.time.temporal.ChronoUnit) Assumptions(org.junit.jupiter.api.Assumptions) EndToEndUtils.validateOutputFiles(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles) CqlDuration(com.datastax.oss.driver.api.core.data.CqlDuration) URL(java.net.URL) CsvUtils(com.datastax.oss.dsbulk.runner.tests.CsvUtils) MockConnector(com.datastax.oss.dsbulk.runner.tests.MockConnector) ByteBuffer(java.nio.ByteBuffer) STATUS_COMPLETED_WITH_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS) RecordUtils(com.datastax.oss.dsbulk.runner.tests.RecordUtils) Lists(com.datastax.oss.driver.shaded.guava.common.collect.Lists) Uuids(com.datastax.oss.driver.api.core.uuid.Uuids) LocalTime(java.time.LocalTime) URI(java.net.URI) Record(com.datastax.oss.dsbulk.connectors.api.Record) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) RoundingMode(java.math.RoundingMode) MethodSource(org.junit.jupiter.params.provider.MethodSource) CodecRegistry(com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry) OperationDirectory(com.datastax.oss.dsbulk.workflow.api.log.OperationDirectory) IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE) TupleValue(com.datastax.oss.driver.api.core.data.TupleValue) LineNumberReader(java.io.LineNumberReader) UUID(java.util.UUID) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) UdtValue(com.datastax.oss.driver.api.core.data.UdtValue) Objects(java.util.Objects) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) DataTypes(com.datastax.oss.driver.api.core.type.DataTypes) List(java.util.List) Type(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) LocalDate(java.time.LocalDate) TypeCodec(com.datastax.oss.driver.api.core.type.codec.TypeCodec) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) CsvSource(org.junit.jupiter.params.provider.CsvSource) EndToEndUtils.validatePositionsFile(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile) HashMap(java.util.HashMap) Version(com.datastax.oss.driver.api.core.Version) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) ImmutableList(com.google.common.collect.ImmutableList) CompressedIOUtils(com.datastax.oss.dsbulk.io.CompressedIOUtils) OverflowStrategy(com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy) FileUtils(com.datastax.oss.dsbulk.tests.utils.FileUtils) EndToEndUtils.createIpByCountryCaseSensitiveTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryCaseSensitiveTable) StringUtils.quoteJson(com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson) UTF_8(java.nio.charset.StandardCharsets.UTF_8) EndToEndUtils.createWithSpacesTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable) OSS(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type.OSS) DefaultProtocolVersion(com.datastax.oss.driver.api.core.DefaultProtocolVersion) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType) IP_BY_COUNTRY_MAPPING_INDEXED(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_INDEXED) EPOCH(java.time.Instant.EPOCH) InputStream(java.io.InputStream) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType)

Aggregations

CqlSession (com.datastax.oss.driver.api.core.CqlSession)2 DefaultProtocolVersion (com.datastax.oss.driver.api.core.DefaultProtocolVersion)2 Version (com.datastax.oss.driver.api.core.Version)2 ResultSet (com.datastax.oss.driver.api.core.cql.ResultSet)2 Row (com.datastax.oss.driver.api.core.cql.Row)2 CqlDuration (com.datastax.oss.driver.api.core.data.CqlDuration)2 TupleValue (com.datastax.oss.driver.api.core.data.TupleValue)2 UdtValue (com.datastax.oss.driver.api.core.data.UdtValue)2 DataType (com.datastax.oss.driver.api.core.type.DataType)2 DataTypes (com.datastax.oss.driver.api.core.type.DataTypes)2 TupleType (com.datastax.oss.driver.api.core.type.TupleType)2 UserDefinedType (com.datastax.oss.driver.api.core.type.UserDefinedType)2 TypeCodec (com.datastax.oss.driver.api.core.type.codec.TypeCodec)2 CodecRegistry (com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry)2 Uuids (com.datastax.oss.driver.api.core.uuid.Uuids)2 Splitter (com.datastax.oss.driver.shaded.guava.common.base.Splitter)2 Strings (com.datastax.oss.driver.shaded.guava.common.base.Strings)2 ImmutableMap (com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap)2 ImmutableSet (com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet)2 Lists (com.datastax.oss.driver.shaded.guava.common.collect.Lists)2