Examples with ImporterConfig org.voltdb.importer.ImporterConfig used on opensource projects

Example 1 with ImporterConfig

use of org.voltdb.importer.ImporterConfig in project voltdb by VoltDB.

the class KafkaTopicTest method main.

/**
     * @param args the command line arguments
     */
public static void main(String[] args) {
    Properties props = new Properties();
    if (args.length < 1) {
        System.out.println("testkafkaimporter: path-to-properties-file - The file should have brokers and topics properties at minimum.");
        System.exit(1);
    }
    String filename = args[0];
    try (InputStream is = new FileInputStream(new File(filename))) {
        props.load(is);
    } catch (Exception ex) {
        ex.printStackTrace();
    }
    String partition = props.getProperty("partition");
    if (args.length > 2) {
        partition = args[2];
        if (partition != null && partition.equals("all")) {
            //Process all partitions.
            partition = null;
        }
    }
    String format = props.getProperty("format");
    if (format == null || format.length() <= 0)
        format = "csv";
    System.out.println("Properties are: " + props);
    props.put("procedure", "fake");
    KafkaStreamImporterFactory factory = new KafkaStreamImporterFactory();
    VoltCSVFormatterFactory ffactory = new VoltCSVFormatterFactory();
    ffactory.create(format, props);
    FormatterBuilder fb = new FormatterBuilder(format, props);
    fb.setFormatterFactory(ffactory);
    Map<URI, ImporterConfig> mmap = factory.createImporterConfigurations(props, fb);
    System.out.println("Number of Partitions for topic are: " + mmap.size() + " Requested partition: " + partition);
    CountDownLatch cdl = new CountDownLatch(mmap.size());
    for (URI uri : mmap.keySet()) {
        if (partition != null && !uri.toString().endsWith(partition)) {
            cdl.countDown();
            continue;
        }
        KafkaTopicPartitionImporter importer = new KafkaTopicPartitionImporter((KafkaStreamImporterConfig) mmap.get(uri));
        Runner runner = new Runner(importer, cdl);
        runner.start();
    }
    try {
        cdl.await();
    } catch (InterruptedException ex) {
        ex.printStackTrace();
    }
    System.out.println("Exiting.");
}

Example 2 with ImporterConfig

use of org.voltdb.importer.ImporterConfig in project voltdb by VoltDB.

the class Log4jSocketImporterFactory method createImporterConfigurations.

@Override
public Map<URI, ImporterConfig> createImporterConfigurations(Properties props, FormatterBuilder formatterBuilder) {
    System.out.println("***Creating log4j importer for " + props);
    ImporterConfig config = new Log4jSocketImporterConfig(props);
    return ImmutableMap.of(config.getResourceID(), config);
}

Example 3 with ImporterConfig

use of org.voltdb.importer.ImporterConfig in project voltdb by VoltDB.

the class KinesisStreamImporterConfig method createConfigEntries.

public static Map<URI, ImporterConfig> createConfigEntries(Properties props, FormatterBuilder formatterBuilder) {
    Map<URI, ImporterConfig> configs = new HashMap<>();
    String appName = getProperty(props, "app.name", "");
    String streamName = getProperty(props, "stream.name", "");
    String region = getProperty(props, "region", "");
    String procedure = getProperty(props, "procedure", "");
    String secretKey = getProperty(props, "secret.key", "");
    String accessKey = getProperty(props, "access.key", "");
    long readInterval = getPropertyAsLong(props, "idle.time.between.reads", 1000);
    long maxReadBatchSize = getPropertyAsLong(props, "max.read.batch.size", 10000);
    long taskBackoffTimeMillis = getPropertyAsLong(props, "task.backoff.time.millis", 500);
    List<Shard> shards = discoverShards(region, streamName, accessKey, secretKey, appName);
    if (shards == null || shards.isEmpty()) {
        LOGGER.warn(String.format("Kinesis stream %s or regions %s are not configured.", streamName, region));
        return configs;
    }
    // build URI per stream, per shard and per application
    int shardCnt = 0;
    for (Shard shard : shards) {
        StringBuilder builder = new StringBuilder(128);
        builder.append("kinesis://").append(region).append("/").append(streamName).append("/").append("shard-").append(shardCnt++).append("/").append(appName);
        URI uri = URI.create(builder.toString());
        ImporterConfig config = new KinesisStreamImporterConfig(appName, region, streamName, procedure, secretKey, accessKey, readInterval, maxReadBatchSize, uri, taskBackoffTimeMillis, formatterBuilder);
        configs.put(uri, config);
    }
    return configs;
}