Examples with ChunkingType com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig.ChunkingType used on opensource projects

Example 1 with ChunkingType

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig.ChunkingType in project databus by linkedin.

the class OracleTxlogEventReader method createQueryStatement.

private PreparedStatement createQueryStatement(Connection conn, OracleTriggerMonitoredSourceInfo source, long sinceScn, int currentFetchSize, boolean useChunking) throws SQLException {
    boolean debugEnabled = _log.isDebugEnabled();
    String eventQuery = null;
    ChunkingType type = _chunkingType;
    if (!useChunking || (!type.isChunkingEnabled())) {
        eventQuery = _eventQueriesBySource.get(source.getSourceId());
    } else {
        if (type == ChunkingType.SCN_CHUNKING)
            eventQuery = _eventChunkedScnQueriesBySource.get(source.getSourceId());
        else
            eventQuery = _eventChunkedTxnQueriesBySource.get(source.getSourceId());
    }
    if (debugEnabled)
        _log.debug("source[" + source.getEventView() + "]: " + eventQuery + "; skipInfinityScn=" + source.isSkipInfinityScn() + " ; sinceScn=" + sinceScn);
    PreparedStatement pStmt = conn.prepareStatement(eventQuery);
    if (!useChunking || (!type.isChunkingEnabled())) {
        pStmt.setFetchSize(currentFetchSize);
        pStmt.setLong(1, sinceScn);
        if (!source.isSkipInfinityScn())
            pStmt.setLong(2, sinceScn);
    } else {
        int i = 1;
        pStmt.setLong(i++, sinceScn);
        pStmt.setLong(i++, sinceScn);
        if (ChunkingType.TXN_CHUNKING == type) {
            pStmt.setLong(i++, _txnsPerChunk);
        } else {
            long untilScn = sinceScn + _scnChunkSize;
            _log.info("SCN chunking mode, next target SCN is: " + untilScn);
            pStmt.setLong(i++, untilScn);
        }
    }
    return pStmt;
}

Example 2 with ChunkingType

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig.ChunkingType in project databus by linkedin.

the class PhysicalSourceConfig method build.

@Override
public PhysicalSourceStaticConfig build() throws InvalidConfigException {
    checkForNulls();
    // check config options for chained relays
    if (_largestEventSizeInBytes >= _largestWindowSizeInBytes) {
        throw new InvalidConfigException("Invalid relay config: largestEventSizeInBytes has to be lesser than largestWindowSizeInBytes:" + " largestEventSizeInBytes=" + _largestEventSizeInBytes + " largestWindowSizeInBytes=" + _largestWindowSizeInBytes);
    }
    LogicalSourceStaticConfig[] sourcesStaticConfigs = new LogicalSourceStaticConfig[_sources.size()];
    for (int i = 0; i < _sources.size(); ++i) {
        sourcesStaticConfigs[i] = _sources.get(i).build();
    }
    ChunkingType chunkingType = ChunkingType.valueOf(_chunkingType);
    return new PhysicalSourceStaticConfig(_name, _id, _uri, _resourceKey, sourcesStaticConfigs, _role, _slowSourceQueryThreshold, _restartScnOffset, _retries.build(), chunkingType, _txnsPerChunk, _scnChunkSize, _chunkedScnThreshold, _maxScnDelayMs, _eventRatePerSec, _maxThrottleDurationInSecs, isDbusEventBufferSet() ? _dbusEventBuffer.build() : null, _largestEventSizeInBytes, _largestWindowSizeInBytes, _errorOnMissingFields, _xmlVersion, _xmlEncoding, _replBitSetter.build());
}

Example 3 with ChunkingType

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig.ChunkingType in project databus by linkedin.

the class TestGoldenGateEventProducer method buildPssc.

private PhysicalSourceStaticConfig buildPssc(long rate, long throttleDuration) throws InvalidConfigException {
    String name = "anet";
    short id = 505, partition = 0;
    String uri = "gg:///mnt/dbext/x1";
    String resourceKey = "test";
    String partitionFunction = "constant:1";
    boolean skipInfinityScn = false;
    String queryHints = null;
    LogicalSourceStaticConfig[] sources = new LogicalSourceStaticConfig[1];
    LogicalSourceStaticConfig lssc = new LogicalSourceStaticConfig(id, name, uri, partitionFunction, partition, skipInfinityScn, queryHints, queryHints, queryHints);
    sources[0] = lssc;
    String role = "MASTER";
    long slowSourceQueryThreshold = 0L;
    long restartScnOffset = 0L;
    BackoffTimerStaticConfigBuilder bsc = new BackoffTimerStaticConfigBuilder();
    ChunkingType ct = ChunkingType.NO_CHUNKING;
    long txnsPerChunk = 0L;
    long scnChunkSize = 0L;
    long chunkedScnThreshold = 0L;
    long maxScnDelayMs = 0L;
    long eventRatePerSec = rate;
    long eventRateThrottleDuration = throttleDuration;
    int largestEventSizeInBytes = 10240;
    long largestWindowSizeInBytes = 10240;
    DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
    cfgBuilder.setMaxSize(10 * 1024 * 1024);
    cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
    cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
    DbusEventBuffer.StaticConfig dbusEventBuffer = cfgBuilder.build();
    boolean errorOnMissingFields = true;
    String xmlVersion = "1.0";
    String xmlEncoding = "";
    String fieldName = "";
    String regex = "";
    ReplicationBitSetterStaticConfig replicationBitSetter = new ReplicationBitSetterStaticConfig(ReplicationBitSetterStaticConfig.SourceType.NONE, fieldName, regex, MissingValueBehavior.STOP_WITH_ERROR);
    PhysicalSourceStaticConfig pssc = new PhysicalSourceStaticConfig(name, id, uri, resourceKey, sources, role, slowSourceQueryThreshold, restartScnOffset, bsc.build(), ct, txnsPerChunk, scnChunkSize, chunkedScnThreshold, maxScnDelayMs, eventRatePerSec, eventRateThrottleDuration, dbusEventBuffer, largestEventSizeInBytes, largestWindowSizeInBytes, errorOnMissingFields, xmlVersion, xmlEncoding, replicationBitSetter);
    return pssc;
}