Examples with RecordMapper com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper used on opensource projects

Example 6 with RecordMapper

use of com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper in project dsbulk by datastax.

the class LoadWorkflow method init.

@Override
public void init() throws Exception {
    settingsManager.init("LOAD", true, SchemaGenerationStrategy.MAP_AND_WRITE);
    executionId = settingsManager.getExecutionId();
    LogSettings logSettings = settingsManager.getLogSettings();
    logSettings.init();
    ConnectorSettings connectorSettings = settingsManager.getConnectorSettings();
    connectorSettings.init(logSettings.isSources());
    connector = connectorSettings.getConnector();
    connector.init();
    DriverSettings driverSettings = settingsManager.getDriverSettings();
    SchemaSettings schemaSettings = settingsManager.getSchemaSettings();
    BatchSettings batchSettings = settingsManager.getBatchSettings();
    ExecutorSettings executorSettings = settingsManager.getExecutorSettings();
    CodecSettings codecSettings = settingsManager.getCodecSettings();
    MonitoringSettings monitoringSettings = settingsManager.getMonitoringSettings();
    engineSettings = settingsManager.getEngineSettings();
    driverSettings.init(true);
    logSettings.logEffectiveSettings(settingsManager.getEffectiveBulkLoaderConfig(), driverSettings.getDriverConfig());
    monitoringSettings.init();
    codecSettings.init();
    executorSettings.init();
    engineSettings.init();
    ConvertingCodecFactory codecFactory = codecSettings.createCodecFactory(schemaSettings.isAllowExtraFields(), schemaSettings.isAllowMissingFields());
    session = driverSettings.newSession(executionId, codecFactory.getCodecRegistry(), monitoringSettings.getRegistry());
    ClusterInformationUtils.printDebugInfoAboutCluster(session);
    schemaSettings.init(session, codecFactory, connector.supports(CommonConnectorFeature.INDEXED_RECORDS), connector.supports(CommonConnectorFeature.MAPPED_RECORDS));
    logManager = logSettings.newLogManager(session, true);
    logManager.init();
    batchSettings.init();
    batchingEnabled = batchSettings.isBatchingEnabled();
    batchBufferSize = batchSettings.getBufferSize();
    RecordMapper recordMapper;
    try {
        recordMapper = schemaSettings.createRecordMapper(session, connector.getRecordMetadata(), batchingEnabled);
    } catch (NestedBatchException e) {
        LOGGER.warn(e.getMessage());
        batchingEnabled = false;
        recordMapper = schemaSettings.createRecordMapper(session, connector.getRecordMetadata(), false);
    }
    mapper = recordMapper::map;
    if (batchingEnabled) {
        batcher = batchSettings.newStatementBatcher(session)::batchByGroupingKey;
    }
    metricsManager = monitoringSettings.newMetricsManager(true, batchingEnabled, logManager.getOperationDirectory(), logSettings.getVerbosity(), session.getContext().getProtocolVersion(), session.getContext().getCodecRegistry(), schemaSettings.getRowType());
    metricsManager.init();
    executor = executorSettings.newWriteExecutor(session, metricsManager.getExecutionListener());
    dryRun = engineSettings.isDryRun();
    if (dryRun) {
        LOGGER.info("Dry-run mode enabled.");
    }
    closed.set(false);
    totalItemsMonitor = metricsManager.newTotalItemsMonitor();
    failedRecordsMonitor = metricsManager.newFailedItemsMonitor();
    failedStatementsMonitor = metricsManager.newFailedItemsMonitor();
    batcherMonitor = metricsManager.newBatcherMonitor();
    totalItemsCounter = logManager.newTotalItemsCounter();
    failedRecordsHandler = logManager.newFailedRecordsHandler();
    unmappableStatementsHandler = logManager.newUnmappableStatementsHandler();
    queryWarningsHandler = logManager.newQueryWarningsHandler();
    failedWritesHandler = logManager.newFailedWritesHandler();
    resultPositionsHndler = logManager.newResultPositionsHandler();
    terminationHandler = logManager.newTerminationHandler();
    numCores = Runtime.getRuntime().availableProcessors();
    if (connector.readConcurrency() < 1) {
        throw new IllegalArgumentException("Invalid read concurrency: " + connector.readConcurrency());
    }
    readConcurrency = connector.readConcurrency();
    hasManyReaders = readConcurrency >= Math.max(4, numCores / 4);
    LOGGER.debug("Using read concurrency: {}", readConcurrency);
    writeConcurrency = engineSettings.getMaxConcurrentQueries().orElseGet(this::determineWriteConcurrency);
    LOGGER.debug("Using write concurrency: {} (user-supplied: {})", writeConcurrency, engineSettings.getMaxConcurrentQueries().isPresent());
}

Example 7 with RecordMapper

use of com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper in project dsbulk by datastax.

the class SchemaSettingsTest method should_use_default_writetime_var_name.

@Test
void should_use_default_writetime_var_name() {
    Config config = TestConfigUtils.createTestConfig("dsbulk.schema", "keyspace", "ks", "table", "t1", "mapping", "\" *=*, f1 = writetime(*) \"");
    SchemaSettings settings = new SchemaSettings(config, MAP_AND_WRITE);
    settings.init(session, codecFactory, false, true);
    RecordMapper mapper = settings.createRecordMapper(session, recordMetadata, false);
    DefaultMapping mapping = (DefaultMapping) getInternalState(mapper, "mapping");
    assertThat(mapping).isNotNull();
    @SuppressWarnings("unchecked") Set<CQLWord> writeTimeVariables = (Set<CQLWord>) getInternalState(mapping, "writeTimeVariables");
    assertThat(writeTimeVariables).containsOnly(CQLWord.fromInternal("writetime(*)"));
}

Example 8 with RecordMapper

use of com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper in project dsbulk by datastax.

the class SchemaSettingsTest method should_detect_writetime_var_in_query.

@Test
void should_detect_writetime_var_in_query() {
    ColumnDefinitions definitions = mockColumnDefinitions(mockColumnDefinition("c1", DataTypes.TEXT), mockColumnDefinition("c2", DataTypes.TEXT), mockColumnDefinition("c3", DataTypes.TEXT));
    when(ps.getVariableDefinitions()).thenReturn(definitions);
    when(table.getColumn(CqlIdentifier.fromInternal("c2"))).thenReturn(Optional.of(col2));
    Config config = TestConfigUtils.createTestConfig("dsbulk.schema", "query", "\"INSERT INTO ks.t1 (c1,c2) VALUES (:c1, :c2) USING TIMESTAMP :c3\"", "mapping", "\" f1 = c1 , f2 = c2 , f3 = c3 \" ");
    SchemaSettings settings = new SchemaSettings(config, MAP_AND_WRITE);
    settings.init(session, codecFactory, false, true);
    RecordMapper mapper = settings.createRecordMapper(session, recordMetadata, false);
    DefaultMapping mapping = (DefaultMapping) getInternalState(mapper, "mapping");
    assertThat(mapping).isNotNull();
    @SuppressWarnings("unchecked") Set<CQLWord> writeTimeVariables = (Set<CQLWord>) getInternalState(mapping, "writeTimeVariables");
    assertThat(writeTimeVariables).containsOnly(CQLWord.fromInternal(C3.asInternal()));
}

Example 9 with RecordMapper

use of com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper in project dsbulk by datastax.

the class SchemaSettingsTest method should_detect_positional_writetime_var_in_query.

@Test
void should_detect_positional_writetime_var_in_query() {
    ColumnDefinitions definitions = mockColumnDefinitions(mockColumnDefinition("c1", DataTypes.TEXT), mockColumnDefinition("c2", DataTypes.TEXT), mockColumnDefinition("\"This is a quoted \\\" variable name\"", DataTypes.TEXT));
    when(ps.getVariableDefinitions()).thenReturn(definitions);
    when(table.getColumn(CqlIdentifier.fromInternal("c2"))).thenReturn(Optional.of(col2));
    Config config = TestConfigUtils.createTestConfig("dsbulk.schema", "query", "\"INSERT INTO ks.t1 (c1,c2) VALUES (?, ?) USING TTL 123 AND tImEsTaMp ?\"");
    SchemaSettings settings = new SchemaSettings(config, MAP_AND_WRITE);
    settings.init(session, codecFactory, false, true);
    RecordMapper mapper = settings.createRecordMapper(session, recordMetadata, false);
    DefaultMapping mapping = (DefaultMapping) getInternalState(mapper, "mapping");
    assertThat(mapping).isNotNull();
    @SuppressWarnings("unchecked") Set<CQLWord> writeTimeVariables = (Set<CQLWord>) getInternalState(mapping, "writeTimeVariables");
    assertThat(writeTimeVariables).containsOnly(INTERNAL_TIMESTAMP_VARNAME);
}

Example 10 with RecordMapper

use of com.datastax.oss.dsbulk.workflow.commons.schema.RecordMapper in project dsbulk by datastax.

the class SchemaSettingsTest method should_warn_when_null_to_unset_true_and_protocol_version_lesser_than_4.

@Test
void should_warn_when_null_to_unset_true_and_protocol_version_lesser_than_4() {
    when(context.getProtocolVersion()).thenReturn(V3);
    Config config = TestConfigUtils.createTestConfig("dsbulk.schema", "nullToUnset", true, "keyspace", "ks", "table", "t1");
    SchemaSettings settings = new SchemaSettings(config, MAP_AND_WRITE);
    settings.init(session, codecFactory, false, true);
    RecordMapper mapper = settings.createRecordMapper(session, recordMetadata, false);
    assertThat(mapper).isNotNull();
    assertThat((Boolean) getInternalState(mapper, NULL_TO_UNSET)).isFalse();
    assertThat(logs).hasMessageContaining(String.format("Protocol version in use (%s) does not support unset bound variables; " + "forcing schema.nullToUnset to false", V3));
}