Examples with FlowCatalog org.apache.gobblin.runtime.spec_catalog.FlowCatalog used on opensource projects

Example 1 with FlowCatalog

use of org.apache.gobblin.runtime.spec_catalog.FlowCatalog in project incubator-gobblin by apache.

the class GitConfigMonitorTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(TEST_DIR);
    // Create a bare repository
    RepositoryCache.FileKey fileKey = RepositoryCache.FileKey.exact(remoteDir, FS.DETECTED);
    this.remoteRepo = fileKey.open(false);
    this.remoteRepo.create(true);
    this.gitForPush = Git.cloneRepository().setURI(this.remoteRepo.getDirectory().getAbsolutePath()).setDirectory(cloneDir).call();
    // push an empty commit as a base for detecting changes
    this.gitForPush.commit().setMessage("First commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.config = ConfigBuilder.create().addPrimitive(ConfigurationKeys.GIT_CONFIG_MONITOR_REPO_URI, this.remoteRepo.getDirectory().getAbsolutePath()).addPrimitive(ConfigurationKeys.GIT_CONFIG_MONITOR_REPO_DIR, TEST_DIR + "/jobConfig").addPrimitive(ConfigurationKeys.FLOWSPEC_STORE_DIR_KEY, TEST_DIR + "flowCatalog").addPrimitive(ConfigurationKeys.GIT_CONFIG_MONITOR_POLLING_INTERVAL, 5).build();
    this.flowCatalog = new FlowCatalog(config);
    this.flowCatalog.startAsync().awaitRunning();
    this.gitConfigMonitor = new GitConfigMonitor(this.config, this.flowCatalog);
    this.gitConfigMonitor.setActive(true);
}

Example 2 with FlowCatalog

use of org.apache.gobblin.runtime.spec_catalog.FlowCatalog in project incubator-gobblin by apache.

the class OrchestratorTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(TOPOLOGY_SPEC_STORE_DIR);
    cleanUpDir(FLOW_SPEC_STORE_DIR);
    Properties orchestratorProperties = new Properties();
    Properties topologyProperties = new Properties();
    topologyProperties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    Properties flowProperties = new Properties();
    flowProperties.put("specStore.fs.dir", FLOW_SPEC_STORE_DIR);
    this.serviceLauncher = new ServiceBasedAppLauncher(orchestratorProperties, "OrchestratorCatalogTest");
    this.topologyCatalog = new TopologyCatalog(ConfigUtils.propertiesToConfig(topologyProperties), Optional.of(logger));
    this.serviceLauncher.addService(topologyCatalog);
    this.flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(flowProperties), Optional.of(logger));
    this.serviceLauncher.addService(flowCatalog);
    this.orchestrator = new Orchestrator(ConfigUtils.propertiesToConfig(orchestratorProperties), Optional.of(this.topologyCatalog), Optional.of(logger));
    this.topologyCatalog.addListener(orchestrator);
    this.flowCatalog.addListener(orchestrator);
    // Start application
    this.serviceLauncher.start();
    // Create Spec to play with
    this.topologySpec = initTopologySpec();
    this.flowSpec = initFlowSpec();
}

Example 3 with FlowCatalog

use of org.apache.gobblin.runtime.spec_catalog.FlowCatalog in project incubator-gobblin by apache.

the class FlowCatalogTest method setup.

@BeforeClass
public void setup() throws Exception {
    File specStoreDir = new File(SPEC_STORE_DIR);
    if (specStoreDir.exists()) {
        FileUtils.deleteDirectory(specStoreDir);
    }
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    this.serviceLauncher = new ServiceBasedAppLauncher(properties, "FlowCatalogTest");
    this.flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties), Optional.of(logger));
    this.serviceLauncher.addService(flowCatalog);
    // Start Catalog
    this.serviceLauncher.start();
    // Create Spec to play with
    this.flowSpec = initFlowSpec();
}

Example 4 with FlowCatalog

use of org.apache.gobblin.runtime.spec_catalog.FlowCatalog in project incubator-gobblin by apache.

the class FlowConfigTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ConfigBuilder configBuilder = ConfigBuilder.create();
    _testDirectory = Files.createTempDir();
    configBuilder.addPrimitive(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY, _testDirectory.getAbsolutePath()).addPrimitive(ConfigurationKeys.SPECSTORE_FS_DIR_KEY, TEST_SPEC_STORE_DIR);
    cleanUpDir(TEST_SPEC_STORE_DIR);
    Config config = configBuilder.build();
    final FlowCatalog flowCatalog = new FlowCatalog(config);
    flowCatalog.startAsync();
    flowCatalog.awaitRunning();
    Injector injector = Guice.createInjector(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(FlowCatalog.class).annotatedWith(Names.named("flowCatalog")).toInstance(flowCatalog);
            // indicate that we are in unit testing since the resource is being blocked until flow catalog changes have
            // been made
            binder.bindConstant().annotatedWith(Names.named("readyToUse")).to(Boolean.TRUE);
        }
    });
    _server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(FlowConfigsResource.class)).injector(injector).build();
    _server.startAsync();
    _server.awaitRunning();
    _client = new FlowConfigClient(String.format("http://localhost:%s/", _server.getPort()));
}