Examples with ReplicationBitSetterStaticConfig com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig used on opensource projects

Example 1 with ReplicationBitSetterStaticConfig

use of com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig in project databus by linkedin.

the class ReplicationBitSetterConfig method build.

@Override
public ReplicationBitSetterStaticConfig build() throws InvalidConfigException {
    SourceType type = null;
    try {
        type = SourceType.valueOf(_sourceType);
    } catch (IllegalArgumentException iae) {
        throw new InvalidConfigException("Source Types should be one of (" + Arrays.asList(SourceType.values()) + ") but is (" + _sourceType + ")");
    }
    MissingValueBehavior missingValueForDelete = null;
    try {
        missingValueForDelete = MissingValueBehavior.valueOf(_missingValueBehavior);
    } catch (IllegalArgumentException iae) {
        throw new InvalidConfigException("Missing Value For Delete Behavior should be one of (" + Arrays.asList(MissingValueBehavior.values()) + ") but is (" + _missingValueBehavior + ")");
    }
    return new ReplicationBitSetterStaticConfig(type, _fieldName, _remoteUpdateValueRegex, missingValueForDelete);
}

Example 2 with ReplicationBitSetterStaticConfig

use of com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method buildPssc.

private PhysicalSourceStaticConfig buildPssc(long rate, long throttleDuration) throws InvalidConfigException {
    String name = "anet";
    short id = 505, partition = 0;
    String uri = "gg:///mnt/dbext/x1";
    String resourceKey = "test";
    String partitionFunction = "constant:1";
    boolean skipInfinityScn = false;
    String queryHints = null;
    LogicalSourceStaticConfig[] sources = new LogicalSourceStaticConfig[1];
    LogicalSourceStaticConfig lssc = new LogicalSourceStaticConfig(id, name, uri, partitionFunction, partition, skipInfinityScn, queryHints, queryHints, queryHints);
    sources[0] = lssc;
    String role = "MASTER";
    long slowSourceQueryThreshold = 0L;
    long restartScnOffset = 0L;
    BackoffTimerStaticConfigBuilder bsc = new BackoffTimerStaticConfigBuilder();
    ChunkingType ct = ChunkingType.NO_CHUNKING;
    long txnsPerChunk = 0L;
    long scnChunkSize = 0L;
    long chunkedScnThreshold = 0L;
    long maxScnDelayMs = 0L;
    long eventRatePerSec = rate;
    long eventRateThrottleDuration = throttleDuration;
    int largestEventSizeInBytes = 10240;
    long largestWindowSizeInBytes = 10240;
    DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
    cfgBuilder.setMaxSize(10 * 1024 * 1024);
    cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
    cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
    DbusEventBuffer.StaticConfig dbusEventBuffer = cfgBuilder.build();
    boolean errorOnMissingFields = true;
    String xmlVersion = "1.0";
    String xmlEncoding = "";
    String fieldName = "";
    String regex = "";
    ReplicationBitSetterStaticConfig replicationBitSetter = new ReplicationBitSetterStaticConfig(ReplicationBitSetterStaticConfig.SourceType.NONE, fieldName, regex, MissingValueBehavior.STOP_WITH_ERROR);
    PhysicalSourceStaticConfig pssc = new PhysicalSourceStaticConfig(name, id, uri, resourceKey, sources, role, slowSourceQueryThreshold, restartScnOffset, bsc.build(), ct, txnsPerChunk, scnChunkSize, chunkedScnThreshold, maxScnDelayMs, eventRatePerSec, eventRateThrottleDuration, dbusEventBuffer, largestEventSizeInBytes, largestWindowSizeInBytes, errorOnMissingFields, xmlVersion, xmlEncoding, replicationBitSetter);
    return pssc;
}

Example 3 with ReplicationBitSetterStaticConfig

use of com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig in project databus by linkedin.

the class ColumnsState method generateAvroRecord.

/**
 * Convert the eventFields read from the xml to an avro record
 * @param eventFields hashmap <database field, value>
 * @return
 */
private GenericRecord generateAvroRecord(HashMap<String, ColumnState.EventField> eventFields, ReplicationBitSetterStaticConfig replicationBitConfig, Pattern replicationValuePattern) throws Exception {
    GenericRecord record = new GenericData.Record(getCurrentSchema());
    List<Schema.Field> fields = getCurrentSchema().getFields();
    String pkFieldName = SchemaHelper.getMetaField(getCurrentSchema(), "pk");
    if (pkFieldName == null)
        throw new DatabusException("No primary key specified in the schema");
    PrimaryKey pk = new PrimaryKey(pkFieldName);
    for (Schema.Field field : fields) {
        if (field.schema().getType() == Schema.Type.ARRAY) {
            throw new DatabusException("The gg parser cannot handle ARRAY datatypes. Found in field: " + field);
        } else {
            // The current database field being processed (field is avro field name  and databaseFieldName is oracle field name (one to one mapping)
            String databaseFieldName = SchemaHelper.getMetaField(field, "dbFieldName");
            // Check if it's ok for this field to be null
            checkNullSafety(eventFields, pk, field, databaseFieldName, replicationBitConfig);
            // Insert the field into the generic record
            String fieldValue = insertFieldIntoRecord(eventFields, record, pkFieldName, pk, field, databaseFieldName);
            // Set the replication flag if this is a replicated event
            if (replicationBitConfig.getSourceType() == ReplicationBitSetterStaticConfig.SourceType.COLUMN && isReplicationField(databaseFieldName, replicationBitConfig)) {
                setReplicated(StateMachineHelper.verifyReplicationStatus(replicationValuePattern, fieldValue, replicationBitConfig.getMissingValueBehavior()));
            }
        }
    }
    // prettyPrint(record);     //TODO remove this -> only for debugging
    return record;
}

Example 4 with ReplicationBitSetterStaticConfig

use of com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig in project databus by linkedin.

the class ColumnsState method checkNullSafety.

/**
 * Checks if the given field can be null.
 * 1. Primary keys cannot be null
 * 2. Replication fields cannot be null
 * 3. If the errorOnMissing field config is null, then this method will throw an exception
 * @param eventFields Map containing dbFieldName => fieldValue
 * @param pk The primary key object stored as an class object
 * @param field The current field being processed(Avro)
 * @param databaseFieldName Field being processed(oracle name)
 * @throws DatabusException
 */
private void checkNullSafety(HashMap<String, ColumnState.EventField> eventFields, PrimaryKey pk, Schema.Field field, String databaseFieldName, ReplicationBitSetterStaticConfig replicationBitConfig) throws DatabusException {
    if (eventFields.get(databaseFieldName) == null) {
        LOG.error("Missing field " + databaseFieldName + " in event from the xml trail for table " + _currentTable);
        if (// Are we ok to accept null fields ?
        !_errorOnMissingFields) {
            // We cannot tolerate empty primary key fields, so we'll throw exception if key is null
            if (pk.isPartOfPrimaryKey(field))
                throw new DatabusException("Skip errors on missing DB Fields is true, but cannot proceed because primary key not found: " + field.name());
            // We also need the replication field, it's not optional if MissingValueBehavior == STOP_WITH_ERROR
            if (replicationBitConfig.getSourceType() == ReplicationBitSetterStaticConfig.SourceType.COLUMN && isReplicationField(databaseFieldName, replicationBitConfig) && replicationBitConfig.getMissingValueBehavior() == MissingValueBehavior.STOP_WITH_ERROR) {
                throw new DatabusException("Skip errors on missing DB Fields is true, but the replication field is missing, this is mandatory, cannot proceed with  " + field.name() + " field missing");
            }
            setSeenMissingFields(true);
            // If not primary key, we create fake hash entry with a null eventfield entry
            ColumnState.EventField emptyEventField = new ColumnState.EventField(false, null, true);
            eventFields.put(databaseFieldName, emptyEventField);
        } else
            throw new DatabusException("Unable to find a required field " + databaseFieldName + " in the xml trail file");
    }
}