Examples with ConfigBuilder org.apache.gobblin.config.ConfigBuilder used on opensource projects

Example 1 with ConfigBuilder

use of org.apache.gobblin.config.ConfigBuilder in project incubator-gobblin by apache.

the class FlowStatusTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ConfigBuilder configBuilder = ConfigBuilder.create();
    JobStatusRetriever jobStatusRetriever = new TestJobStatusRetriever();
    final FlowStatusGenerator flowStatusGenerator = FlowStatusGenerator.builder().jobStatusRetriever(jobStatusRetriever).build();
    Injector injector = Guice.createInjector(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(FlowStatusGenerator.class).annotatedWith(Names.named(FlowStatusResource.FLOW_STATUS_GENERATOR_INJECT_NAME)).toInstance(flowStatusGenerator);
        }
    });
    _server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(FlowStatusResource.class)).injector(injector).build();
    _server.startAsync();
    _server.awaitRunning();
    _client = new FlowStatusClient(String.format("http://localhost:%s/", _server.getPort()));
    messageJoiner = Joiner.on(FlowStatusResource.MESSAGE_SEPARATOR);
}

Example 2 with ConfigBuilder

use of org.apache.gobblin.config.ConfigBuilder in project incubator-gobblin by apache.

the class ZkDatasetStateStoreTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ConfigBuilder configBuilder = ConfigBuilder.create();
    testingServer = new TestingServer(-1);
    zkJobStateStore = new ZkStateStore<>(testingServer.getConnectString(), "/STATE_STORE/TEST", false, JobState.class);
    configBuilder.addPrimitive(ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_KEY, testingServer.getConnectString());
    configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, "/STATE_STORE/TEST2");
    ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class);
    DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("zk").newInstance();
    zkDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build());
    // clear data that may have been left behind by a prior test run
    zkJobStateStore.delete(TEST_JOB_NAME);
    zkDatasetStateStore.delete(TEST_JOB_NAME);
    zkJobStateStore.delete(TEST_JOB_NAME2);
    zkDatasetStateStore.delete(TEST_JOB_NAME2);
}

Example 3 with ConfigBuilder

use of org.apache.gobblin.config.ConfigBuilder in project incubator-gobblin by apache.

the class MysqlDatasetStateStoreTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
    String jdbcUrl = testMetastoreDatabase.getJdbcUrl();
    ConfigBuilder configBuilder = ConfigBuilder.create();
    BasicDataSource mySqlDs = new BasicDataSource();
    mySqlDs.setDriverClassName(ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER);
    mySqlDs.setDefaultAutoCommit(false);
    mySqlDs.setUrl(jdbcUrl);
    mySqlDs.setUsername(TEST_USER);
    mySqlDs.setPassword(TEST_PASSWORD);
    dbJobStateStore = new MysqlStateStore<>(mySqlDs, TEST_STATE_STORE, false, JobState.class);
    configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, jdbcUrl);
    configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, TEST_USER);
    configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, TEST_PASSWORD);
    ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class);
    DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("mysql").newInstance();
    dbDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build());
    // clear data that may have been left behind by a prior test run
    dbJobStateStore.delete(TEST_JOB_NAME);
    dbDatasetStateStore.delete(TEST_JOB_NAME);
    dbJobStateStore.delete(TEST_JOB_NAME2);
    dbDatasetStateStore.delete(TEST_JOB_NAME2);
}

Example 4 with ConfigBuilder

use of org.apache.gobblin.config.ConfigBuilder in project incubator-gobblin by apache.

the class FlowConfigTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ConfigBuilder configBuilder = ConfigBuilder.create();
    _testDirectory = Files.createTempDir();
    configBuilder.addPrimitive(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY, _testDirectory.getAbsolutePath()).addPrimitive(ConfigurationKeys.SPECSTORE_FS_DIR_KEY, TEST_SPEC_STORE_DIR);
    cleanUpDir(TEST_SPEC_STORE_DIR);
    Config config = configBuilder.build();
    final FlowCatalog flowCatalog = new FlowCatalog(config);
    flowCatalog.startAsync();
    flowCatalog.awaitRunning();
    Injector injector = Guice.createInjector(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(FlowCatalog.class).annotatedWith(Names.named("flowCatalog")).toInstance(flowCatalog);
            // indicate that we are in unit testing since the resource is being blocked until flow catalog changes have
            // been made
            binder.bindConstant().annotatedWith(Names.named("readyToUse")).to(Boolean.TRUE);
        }
    });
    _server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(FlowConfigsResource.class)).injector(injector).build();
    _server.startAsync();
    _server.awaitRunning();
    _client = new FlowConfigClient(String.format("http://localhost:%s/", _server.getPort()));
}

Example 5 with ConfigBuilder

use of org.apache.gobblin.config.ConfigBuilder in project incubator-gobblin by apache.

the class FlowConfigsResource method createFlowSpecForConfig.

/**
 * Build a {@link FlowSpec} from a {@link FlowConfig}
 * @param flowConfig flow configuration
 * @return {@link FlowSpec} created with attributes from flowConfig
 */
private FlowSpec createFlowSpecForConfig(FlowConfig flowConfig) {
    ConfigBuilder configBuilder = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowConfig.getId().getFlowGroup()).addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowConfig.getId().getFlowName());
    if (flowConfig.hasSchedule()) {
        Schedule schedule = flowConfig.getSchedule();
        configBuilder.addPrimitive(ConfigurationKeys.JOB_SCHEDULE_KEY, schedule.getCronSchedule());
        configBuilder.addPrimitive(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, schedule.isRunImmediately());
    }
    Config config = configBuilder.build();
    Config configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties()));
    URI templateURI = null;
    try {
        templateURI = new URI(flowConfig.getTemplateUris());
    } catch (URISyntaxException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e);
    }
    return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build();
}