Examples with ITestMetastoreDatabase org.apache.gobblin.metastore.testing.ITestMetastoreDatabase used on opensource projects

Example 1 with ITestMetastoreDatabase

use of org.apache.gobblin.metastore.testing.ITestMetastoreDatabase in project incubator-gobblin by apache.

the class TestMysqlJobCatalog method setUp.

/**
 * create a new DB/`JobCatalog` for each test, so they're completely independent
 */
@BeforeMethod
public void setUp() throws Exception {
    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
    Config config = ConfigBuilder.create().addPrimitive(ConfigurationKeys.METRICS_ENABLED_KEY, "true").addPrimitive(MysqlJobCatalog.DB_CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl()).addPrimitive(MysqlJobCatalog.DB_CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_USER_KEY, USER).addPrimitive(MysqlJobCatalog.DB_CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, PASSWORD).addPrimitive(MysqlJobCatalog.DB_CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TABLE).build();
    this.cat = new MysqlJobCatalog(config);
}

Example 2 with ITestMetastoreDatabase

use of org.apache.gobblin.metastore.testing.ITestMetastoreDatabase in project incubator-gobblin by apache.

the class MysqlBaseSpecStoreTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
    // prefix keys to demonstrate disambiguation mechanism used to side-step intentially-sabatoged non-prefixed, 'fallback'
    Config config = ConfigBuilder.create().addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, " SABATOGE! !" + testDb.getJdbcUrl()).addPrimitive(MysqlBaseSpecStore.CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl()).addPrimitive(MysqlBaseSpecStore.CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_USER_KEY, USER).addPrimitive(MysqlBaseSpecStore.CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, PASSWORD).addPrimitive(MysqlBaseSpecStore.CONFIG_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TABLE).build();
    this.specStore = new MysqlBaseSpecStore(config, new JavaSpecSerDe());
    topoSpec1 = new TopologySpec.Builder(this.uri1).withConfig(ConfigBuilder.create().addPrimitive("key", "value").addPrimitive("key3", "value3").addPrimitive("config.with.dot", "value4").build()).withDescription("Test1").withVersion("Test version").withSpecExecutor(MockedSpecExecutor.createDummySpecExecutor(new URI("execA"))).build();
    topoSpec2 = new TopologySpec.Builder(this.uri2).withConfig(ConfigBuilder.create().addPrimitive("converter", "value1,value2,value3").addPrimitive("key3", "value3").build()).withDescription("Test2").withVersion("Test version 2").withSpecExecutor(MockedSpecExecutor.createDummySpecExecutor(new URI("execB"))).build();
}

Example 3 with ITestMetastoreDatabase

use of org.apache.gobblin.metastore.testing.ITestMetastoreDatabase in project incubator-gobblin by apache.

the class MysqlSpecStoreTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
    Config config = ConfigBuilder.create().addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl()).addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, USER).addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, PASSWORD).addPrimitive(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TABLE).build();
    this.specStore = new MysqlSpecStore(config, new TestSpecSerDe());
    this.oldSpecStore = new OldSpecStore(config, new TestSpecSerDe());
    flowSpec1 = FlowSpec.builder(this.uri1).withConfig(ConfigBuilder.create().addPrimitive("key", "value").addPrimitive("key3", "value3").addPrimitive("config.with.dot", "value4").addPrimitive(FLOW_SOURCE_IDENTIFIER_KEY, "source").addPrimitive(FLOW_DESTINATION_IDENTIFIER_KEY, "destination").addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "fg1").addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "fn1").build()).withDescription("Test flow spec").withVersion("Test version").build();
    flowSpec2 = FlowSpec.builder(this.uri2).withConfig(ConfigBuilder.create().addPrimitive("converter", "value1,value2,value3").addPrimitive("key3", "value3").addPrimitive(FLOW_SOURCE_IDENTIFIER_KEY, "source").addPrimitive(FLOW_DESTINATION_IDENTIFIER_KEY, "destination").addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "fg2").addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "fn2").build()).withDescription("Test flow spec 2").withVersion("Test version 2").build();
    flowSpec3 = FlowSpec.builder(this.uri3).withConfig(ConfigBuilder.create().addPrimitive("key3", "value3").addPrimitive(FLOW_SOURCE_IDENTIFIER_KEY, "source").addPrimitive(FLOW_DESTINATION_IDENTIFIER_KEY, "destination").addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "fg3").addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "fn3").build()).withDescription("Test flow spec 3").withVersion("Test version 3").build();
    flowSpec4 = FlowSpec.builder(this.uri4).withConfig(ConfigBuilder.create().addPrimitive("key4", "value4").addPrimitive(FLOW_SOURCE_IDENTIFIER_KEY, "source").addPrimitive(FLOW_DESTINATION_IDENTIFIER_KEY, "destination").addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "fg4").addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "fn4").addPrimitive(ConfigurationKeys.FLOW_OWNING_GROUP_KEY, "owningGroup4").build()).withDescription("Test flow spec 4").withVersion("Test version 4").build();
}

Example 4 with ITestMetastoreDatabase

use of org.apache.gobblin.metastore.testing.ITestMetastoreDatabase in project incubator-gobblin by apache.

the class GobblinServiceRedirectTest method setup.

@BeforeClass
public void setup() throws Exception {
    port1 = Integer.toString(new PortUtils.ServerSocketPortLocator().random());
    port2 = Integer.toString(new PortUtils.ServerSocketPortLocator().random());
    BASE_PATH1.deleteOnExit();
    BASE_PATH2.deleteOnExit();
    // Use a random ZK port
    this.testingZKServer = new TestingServer(-1);
    logger.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());
    HelixUtils.createGobblinHelixCluster(testingZKServer.getConnectString(), TEST_HELIX_CLUSTER_NAME);
    ITestMetastoreDatabase testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
    Properties commonServiceCoreProperties = new Properties();
    mysql = new MySQLContainer("mysql:" + TestServiceDatabaseConfig.MysqlVersion);
    mysql.start();
    commonServiceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_URL_KEY, mysql.getJdbcUrl());
    commonServiceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_USERNAME, mysql.getUsername());
    commonServiceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_PASSWORD, mysql.getPassword());
    commonServiceCoreProperties.put(ServiceConfigKeys.ZK_CONNECTION_STRING_KEY, testingZKServer.getConnectString());
    commonServiceCoreProperties.put(ServiceConfigKeys.HELIX_CLUSTER_NAME_KEY, TEST_HELIX_CLUSTER_NAME);
    commonServiceCoreProperties.put(ServiceConfigKeys.HELIX_INSTANCE_NAME_KEY, "GaaS_" + UUID.randomUUID().toString());
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY, TEST_GOBBLIN_EXECUTOR_NAME);
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".description", "StandaloneTestExecutor");
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".version", "1");
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".uri", "gobblinExecutor");
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".specExecutorInstance", "org.gobblin.service.InMemorySpecExecutor");
    commonServiceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".specExecInstance.capabilities", TEST_SOURCE_NAME + ":" + TEST_SINK_NAME);
    commonServiceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_USER_KEY, "testUser");
    commonServiceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "testPassword");
    commonServiceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_URL_KEY, testMetastoreDatabase.getJdbcUrl());
    commonServiceCoreProperties.put("zookeeper.connect", testingZKServer.getConnectString());
    commonServiceCoreProperties.put(ConfigurationKeys.STATE_STORE_FACTORY_CLASS_KEY, MysqlJobStatusStateStoreFactory.class.getName());
    commonServiceCoreProperties.put(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_STATUS_MONITOR_ENABLED_KEY, false);
    commonServiceCoreProperties.put(ServiceConfigKeys.FORCE_LEADER, true);
    commonServiceCoreProperties.put(ServiceConfigKeys.SERVICE_URL_PREFIX, PREFIX);
    commonServiceCoreProperties.put(ServiceConfigKeys.SERVICE_NAME, SERVICE_NAME);
    node1ServiceCoreProperties = new Properties();
    node1ServiceCoreProperties.putAll(commonServiceCoreProperties);
    node1ServiceCoreProperties.put(ConfigurationKeys.TOPOLOGYSPEC_STORE_DIR_KEY, NODE_1_TOPOLOGY_SPEC_STORE_DIR);
    node1ServiceCoreProperties.put(FlowCatalog.FLOWSPEC_STORE_DIR_KEY, NODE_1_FLOW_SPEC_STORE_DIR);
    node1ServiceCoreProperties.put(FsJobStatusRetriever.CONF_PREFIX + "." + ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, NODE_1_JOB_STATUS_STATE_STORE_DIR);
    node1ServiceCoreProperties.put(QUARTZ_INSTANCE_NAME, "RedirectQuartzScheduler1");
    node1ServiceCoreProperties.put(QUARTZ_THREAD_POOL_COUNT, 3);
    node1ServiceCoreProperties.put(ServiceConfigKeys.SERVICE_PORT, port1);
    node2ServiceCoreProperties = new Properties();
    node2ServiceCoreProperties.putAll(commonServiceCoreProperties);
    node2ServiceCoreProperties.put(ConfigurationKeys.TOPOLOGYSPEC_STORE_DIR_KEY, NODE_2_TOPOLOGY_SPEC_STORE_DIR);
    node2ServiceCoreProperties.put(FlowCatalog.FLOWSPEC_STORE_DIR_KEY, NODE_2_FLOW_SPEC_STORE_DIR);
    node2ServiceCoreProperties.put(FsJobStatusRetriever.CONF_PREFIX + "." + ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, NODE_2_JOB_STATUS_STATE_STORE_DIR);
    node2ServiceCoreProperties.put(QUARTZ_INSTANCE_NAME, "RedirectQuartzScheduler2");
    node2ServiceCoreProperties.put(QUARTZ_THREAD_POOL_COUNT, 3);
    node2ServiceCoreProperties.put(ServiceConfigKeys.SERVICE_PORT, port2);
    // Start Node 1
    this.node1GobblinServiceManager = GobblinServiceManager.create("RedirectCoreService1", "1", ConfigUtils.propertiesToConfig(node1ServiceCoreProperties), new Path(NODE_1_SERVICE_WORK_DIR));
    this.node1GobblinServiceManager.start();
    // Start Node 2
    this.node2GobblinServiceManager = GobblinServiceManager.create("RedirectCoreService2", "2", ConfigUtils.propertiesToConfig(node2ServiceCoreProperties), new Path(NODE_2_SERVICE_WORK_DIR));
    this.node2GobblinServiceManager.start();
    // Initialize Node 1 Client
    Map<String, String> transportClientProperties = Maps.newHashMap();
    transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT, "10000");
    this.node1FlowConfigClient = new FlowConfigClient(String.format("http://localhost:%s/", this.node1GobblinServiceManager.restliServer.getPort()), transportClientProperties);
    // Initialize Node 2 Client
    this.node2FlowConfigClient = new FlowConfigClient(String.format("http://localhost:%s/", this.node2GobblinServiceManager.restliServer.getPort()), transportClientProperties);
}

Example 5 with ITestMetastoreDatabase

use of org.apache.gobblin.metastore.testing.ITestMetastoreDatabase in project incubator-gobblin by apache.

the class GobblinServiceManagerTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(SERVICE_WORK_DIR);
    cleanUpDir(SPEC_STORE_PARENT_DIR);
    mysql = new MySQLContainer("mysql:" + TestServiceDatabaseConfig.MysqlVersion);
    mysql.start();
    serviceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_URL_KEY, mysql.getJdbcUrl());
    serviceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_USERNAME, mysql.getUsername());
    serviceCoreProperties.put(ServiceConfigKeys.SERVICE_DB_PASSWORD, mysql.getPassword());
    ITestMetastoreDatabase testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
    testingServer = new TestingServer(true);
    flowProperties.put("param1", "value1");
    flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME);
    flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME);
    serviceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_USER_KEY, "testUser");
    serviceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "testPassword");
    serviceCoreProperties.put(ConfigurationKeys.STATE_STORE_DB_URL_KEY, testMetastoreDatabase.getJdbcUrl());
    serviceCoreProperties.put("zookeeper.connect", testingServer.getConnectString());
    serviceCoreProperties.put(ConfigurationKeys.STATE_STORE_FACTORY_CLASS_KEY, MysqlJobStatusStateStoreFactory.class.getName());
    serviceCoreProperties.put(ConfigurationKeys.TOPOLOGYSPEC_STORE_DIR_KEY, TOPOLOGY_SPEC_STORE_DIR);
    serviceCoreProperties.put(FlowCatalog.FLOWSPEC_STORE_DIR_KEY, FLOW_SPEC_STORE_DIR);
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY, TEST_GOBBLIN_EXECUTOR_NAME);
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".description", "StandaloneTestExecutor");
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".version", FlowSpec.Builder.DEFAULT_VERSION);
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".uri", "gobblinExecutor");
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".specExecutorInstance", "org.apache.gobblin.service.InMemorySpecExecutor");
    serviceCoreProperties.put(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + TEST_GOBBLIN_EXECUTOR_NAME + ".specExecInstance.capabilities", TEST_SOURCE_NAME + ":" + TEST_SINK_NAME);
    serviceCoreProperties.put(ServiceConfigKeys.GOBBLIN_SERVICE_GIT_CONFIG_MONITOR_ENABLED_KEY, true);
    serviceCoreProperties.put(GitConfigMonitor.GIT_CONFIG_MONITOR_PREFIX + "." + ConfigurationKeys.GIT_MONITOR_REPO_URI, GIT_REMOTE_REPO_DIR);
    serviceCoreProperties.put(GitConfigMonitor.GIT_CONFIG_MONITOR_PREFIX + "." + ConfigurationKeys.GIT_MONITOR_REPO_DIR, GIT_LOCAL_REPO_DIR);
    serviceCoreProperties.put(GitConfigMonitor.GIT_CONFIG_MONITOR_PREFIX + "." + ConfigurationKeys.GIT_MONITOR_POLLING_INTERVAL, 5);
    serviceCoreProperties.put(FsJobStatusRetriever.CONF_PREFIX + "." + ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, JOB_STATUS_STATE_STORE_DIR);
    serviceCoreProperties.put(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_STATUS_MONITOR_ENABLED_KEY, false);
    serviceCoreProperties.put(ServiceConfigKeys.GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY, MockedSpecCompiler.class.getCanonicalName());
    transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT, "10000");
    // Create a bare repository
    RepositoryCache.FileKey fileKey = RepositoryCache.FileKey.exact(new File(GIT_REMOTE_REPO_DIR), FS.DETECTED);
    fileKey.open(false).create(true);
    this.gitForPush = Git.cloneRepository().setURI(GIT_REMOTE_REPO_DIR).setDirectory(new File(GIT_CLONE_DIR)).call();
    // push an empty commit as a base for detecting changes
    this.gitForPush.commit().setMessage("First commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(new RefSpec("master")).call();
    this.gobblinServiceManager = GobblinServiceManager.create("CoreService", "1", ConfigUtils.propertiesToConfig(serviceCoreProperties), new Path(SERVICE_WORK_DIR));
    this.gobblinServiceManager.start();
    this.flowConfigClient = new FlowConfigV2Client(String.format("http://127.0.0.1:%s/", this.gobblinServiceManager.getRestLiServerListeningURI().getPort()), transportClientProperties);
}