Examples with PipeManager com.airbnb.spinaltap.common.pipe.PipeManager used on opensource projects

Example 1 with PipeManager

use of com.airbnb.spinaltap.common.pipe.PipeManager in project SpinalTap by airbnb.

the class SpinalTapStandaloneApp method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        log.error("Usage: SpinalTapStandaloneApp <config.yaml>");
        System.exit(1);
    }
    ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
    SpinalTapStandaloneConfiguration config = objectMapper.readValue(new File(args[0]), SpinalTapStandaloneConfiguration.class);
    MySQLPipeFactory mySQLPipeFactory = new MySQLPipeFactory(config.getMysqlUser(), config.getMysqlPassword(), config.getMysqlServerId(), () -> new KafkaDestinationBuilder<>(config.getKafkaProducerConfig()), config.getMysqlSchemaStoreConfig(), new TaggedMetricRegistry());
    CuratorFramework zkClient = CuratorFrameworkFactory.builder().namespace(config.getZkNamespace()).connectString(config.getZkConnectionString()).retryPolicy(new ExponentialBackoffRetry(100, 3)).build();
    ZookeeperRepositoryFactory zkRepositoryFactory = new ZookeeperRepositoryFactory(zkClient);
    zkClient.start();
    PipeManager pipeManager = new PipeManager();
    for (MysqlConfiguration mysqlSourceConfig : config.getMysqlSources()) {
        String sourceName = mysqlSourceConfig.getName();
        String partitionName = String.format("%s_0", sourceName);
        pipeManager.addPipes(sourceName, partitionName, mySQLPipeFactory.createPipes(mysqlSourceConfig, partitionName, zkRepositoryFactory, 0));
    }
    Runtime.getRuntime().addShutdownHook(new Thread(() -> pipeManager.stop()));
}