Examples with PartitionFunction com.linkedin.databus2.producers.PartitionFunction used on opensource projects

Example 6 with PartitionFunction

use of com.linkedin.databus2.producers.PartitionFunction in project databus by linkedin.

the class GGEventGenerationFactory method buildPartitionFunction.

/**
   * Given a logical source config, create a partition function.
   *
   * @param sourceConfig
   * @return the partition function
   * @throws InvalidConfigException
   */
public static PartitionFunction buildPartitionFunction(LogicalSourceStaticConfig sourceConfig) throws InvalidConfigException {
    String partitionFunction = sourceConfig.getPartitionFunction();
    if (partitionFunction.startsWith("constant:")) {
        try {
            String numberPart = partitionFunction.substring("constant:".length()).trim();
            short constantPartitionNumber = Short.valueOf(numberPart);
            return new ConstantPartitionFunction(constantPartitionNumber);
        } catch (Exception ex) {
            // to parse the partition number.
            throw new InvalidConfigException("Invalid partition configuration (" + partitionFunction + "). " + "Could not parse the constant partition number.");
        }
    } else {
        throw new InvalidConfigException("Invalid partition configuration (" + partitionFunction + ").");
    }
}

Example 7 with PartitionFunction

use of com.linkedin.databus2.producers.PartitionFunction in project databus by linkedin.

the class TestGoldenGateEventProducer method buildPssc.

private PhysicalSourceStaticConfig buildPssc(long rate, long throttleDuration) throws InvalidConfigException {
    String name = "anet";
    short id = 505, partition = 0;
    String uri = "gg:///mnt/dbext/x1";
    String resourceKey = "test";
    String partitionFunction = "constant:1";
    boolean skipInfinityScn = false;
    String queryHints = null;
    LogicalSourceStaticConfig[] sources = new LogicalSourceStaticConfig[1];
    LogicalSourceStaticConfig lssc = new LogicalSourceStaticConfig(id, name, uri, partitionFunction, partition, skipInfinityScn, queryHints, queryHints, queryHints);
    sources[0] = lssc;
    String role = "MASTER";
    long slowSourceQueryThreshold = 0L;
    long restartScnOffset = 0L;
    BackoffTimerStaticConfigBuilder bsc = new BackoffTimerStaticConfigBuilder();
    ChunkingType ct = ChunkingType.NO_CHUNKING;
    long txnsPerChunk = 0L;
    long scnChunkSize = 0L;
    long chunkedScnThreshold = 0L;
    long maxScnDelayMs = 0L;
    long eventRatePerSec = rate;
    long eventRateThrottleDuration = throttleDuration;
    int largestEventSizeInBytes = 10240;
    long largestWindowSizeInBytes = 10240;
    DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
    cfgBuilder.setMaxSize(10 * 1024 * 1024);
    cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
    cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
    DbusEventBuffer.StaticConfig dbusEventBuffer = cfgBuilder.build();
    boolean errorOnMissingFields = true;
    String xmlVersion = "1.0";
    String xmlEncoding = "";
    String fieldName = "";
    String regex = "";
    ReplicationBitSetterStaticConfig replicationBitSetter = new ReplicationBitSetterStaticConfig(ReplicationBitSetterStaticConfig.SourceType.NONE, fieldName, regex, MissingValueBehavior.STOP_WITH_ERROR);
    PhysicalSourceStaticConfig pssc = new PhysicalSourceStaticConfig(name, id, uri, resourceKey, sources, role, slowSourceQueryThreshold, restartScnOffset, bsc.build(), ct, txnsPerChunk, scnChunkSize, chunkedScnThreshold, maxScnDelayMs, eventRatePerSec, eventRateThrottleDuration, dbusEventBuffer, largestEventSizeInBytes, largestWindowSizeInBytes, errorOnMissingFields, xmlVersion, xmlEncoding, replicationBitSetter);
    return pssc;
}

Example 8 with PartitionFunction

use of com.linkedin.databus2.producers.PartitionFunction in project databus by linkedin.

the class TestGoldenGateEventProducer method buildSimplePssc.

private PhysicalSourceStaticConfig buildSimplePssc(short[] sourceIds, String[] sourceNames, String uri) throws InvalidConfigException {
    String partitionFunction = "constant:1";
    String pPartName = "part1";
    short pPartId = 0;
    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for (int i = 0; i < sourceIds.length; i++) {
        LogicalSourceConfig lc = new LogicalSourceConfig();
        lc.setId(sourceIds[i]);
        lc.setName(sourceNames[i]);
        lc.setPartitionFunction(partitionFunction);
        // this format is expected by GG
        lc.setUri(pPartName + "." + sourceNames[i]);
        pc.addSource(lc);
    }
    return pc.build();
}

Example 9 with PartitionFunction

use of com.linkedin.databus2.producers.PartitionFunction in project databus by linkedin.

the class OracleEventProducerFactory method buildPartitionFunction.

public PartitionFunction buildPartitionFunction(LogicalSourceStaticConfig sourceConfig) throws InvalidConfigException {
    String partitionFunction = sourceConfig.getPartitionFunction();
    if (partitionFunction.startsWith("constant:")) {
        try {
            String numberPart = partitionFunction.substring("constant:".length()).trim();
            short constantPartitionNumber = Short.valueOf(numberPart);
            return new ConstantPartitionFunction(constantPartitionNumber);
        } catch (Exception ex) {
            // Could be a NumberFormatException, IndexOutOfBoundsException or other exception when trying to parse the partition number.
            throw new InvalidConfigException("Invalid partition configuration (" + partitionFunction + "). " + "Could not parse the constant partition number.");
        }
    } else {
        throw new InvalidConfigException("Invalid partition configuration (" + partitionFunction + ").");
    }
}