Examples with GobblinScopeTypes org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes used on opensource projects

Example 6 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class SingleTask method run.

public void run() throws IOException, InterruptedException {
    List<WorkUnit> workUnits = getWorkUnits();
    JobState jobState = getJobState();
    Config jobConfig = getConfigFromJobState(jobState);
    _logger.debug("SingleTask.run: jobId {} workUnitFilePath {} jobStateFilePath {} jobState {} jobConfig {}", _jobId, _workUnitFilePath, _jobStateFilePath, jobState, jobConfig);
    try (SharedResourcesBroker<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(jobConfig, GobblinScopeTypes.GLOBAL.defaultScopeInstance())) {
        SharedResourcesBroker<GobblinScopeTypes> jobBroker = getJobBroker(jobState, globalBroker);
        _taskattempt = _taskAttemptBuilder.build(workUnits.iterator(), _jobId, jobState, jobBroker);
        _taskattempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.CommitPolicy.IMMEDIATE);
    }
}

Example 7 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class AutoscopedFactoryTest method testAutoscopedResourcesOnlyClosedInCorrectScope.

@Test
public void testAutoscopedResourcesOnlyClosedInCorrectScope() throws Exception {
    Config config = ConfigFactory.empty();
    config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB);
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build();
    SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build();
    SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")).withAdditionalParentBroker(containerBroker).build();
    TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"));
    // since object autoscopes at job level, it should not be closed if we close the task broker
    taskBroker.close();
    Assert.assertFalse(autoscopedResource.isClosed());
    // however, when closing job broker, resource should be closed
    jobBroker.close();
    Assert.assertTrue(autoscopedResource.isClosed());
}

Example 8 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testScoping.

@Test
public void testScoping() throws Exception {
    // Correct creation behavior
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build();
    Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getType(), GobblinScopeTypes.INSTANCE);
    Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getClass(), GobblinScopeInstance.class);
    Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE), GobblinScopeTypes.INSTANCE.defaultScopeInstance());
    Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getType(), GobblinScopeTypes.JOB);
    Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getClass(), JobScopeInstance.class);
    Assert.assertEquals(((JobScopeInstance) jobBroker.getScope(GobblinScopeTypes.JOB)).getJobId(), "job123");
    try {
        jobBroker.getScope(GobblinScopeTypes.TASK);
        Assert.fail();
    } catch (NoSuchScopeException nsse) {
    // should throw no scope exception
    }
}

Example 9 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testRedirect.

@Test
public void testRedirect() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build();
    // create a shared resource
    TestFactory.SharedResource resource = jobBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB);
    Assert.assertEquals(resource.getKey(), "myKey");
    Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource);
    Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource);
}

Example 10 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testScopedView.

@Test
public void testScopedView() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build();
    SharedResourcesBrokerImpl<GobblinScopeTypes> instanceView = jobBroker.getScopedView(GobblinScopeTypes.INSTANCE);
    Assert.assertEquals(instanceView.selfScope().getType(), GobblinScopeTypes.INSTANCE);
    TestFactory.SharedResource resource = instanceView.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"));
    TestFactory.SharedResource resource2 = jobBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.INSTANCE);
    Assert.assertEquals(resource, resource2);
    try {
        instanceView.newSubscopedBuilder(new JobScopeInstance("otherJob", "job234"));
        Assert.fail();
    } catch (UnsupportedOperationException exc) {
    // Expected
    }
}