Examples with SchedulerStateManagerAdaptor org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor used on opensource projects

Example 31 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class HealthManagerTest method testInitialize.

@Test
public void testInitialize() throws Exception {
    String topologyName = "testTopology";
    Config config = Config.newBuilder().put(Key.SCHEDULER_IS_SERVICE, true).put(Key.TOPOLOGY_NAME, topologyName).put(Key.ENVIRON, "environ").put(Key.CLUSTER, "cluster").build();
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    SchedulerLocation schedulerLocation = SchedulerLocation.newBuilder().setTopologyName(topologyName).setHttpEndpoint("http://127.0.0.1").build();
    when(adaptor.getSchedulerLocation(anyString())).thenReturn(schedulerLocation);
    HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
    AbstractModule baseModule = HealthManager.buildBaseModule("127.0.0.1", TrackerMetricsProvider.class.getName(), publishingMetrics);
    HealthManager healthManager = new HealthManager(config, baseModule);
    Map<String, Object> policy = new HashMap<>();
    policy.put(PolicyConfigKey.HEALTH_POLICY_CLASS.key(), TestPolicy.class.getName());
    policy.put("test-config", "test-value");
    String[] policyIds = { "policy" };
    HealthPolicyConfigReader policyConfigProvider = mock(HealthPolicyConfigReader.class);
    when(policyConfigProvider.getPolicyIds()).thenReturn(Arrays.asList(policyIds));
    when(policyConfigProvider.getPolicyConfig("policy")).thenReturn(policy);
    HealthManager spyHealthMgr = spy(healthManager);
    doReturn(adaptor).when(spyHealthMgr).createStateMgrAdaptor();
    doReturn(policyConfigProvider).when(spyHealthMgr).createPolicyConfigReader();
    spyHealthMgr.initialize();
    List<IHealthPolicy> healthPolicies = spyHealthMgr.getHealthPolicies();
    assertEquals(1, healthPolicies.size());
    TestPolicy healthPolicy = (TestPolicy) healthPolicies.iterator().next();
    assertNotNull(healthPolicy.schedulerClient);
    assertEquals(healthPolicy.stateMgrAdaptor, adaptor);
    assertNotNull(healthPolicy.metricsProvider);
    assertEquals(healthPolicy.config.getConfig("test-config"), "test-value");
}

Example 32 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class PackingPlanProviderTest method fetchesAndCachesPackingFromStateMgr.

@Test
public void fetchesAndCachesPackingFromStateMgr() {
    PackingPlans.PackingPlan proto = PackingTestUtils.testProtoPackingPlan(topologyName, new RoundRobinPacking());
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    when(adaptor.getPackingPlan(topologyName)).thenReturn(proto);
    PackingPlanProvider provider = new PackingPlanProvider(adaptor, eventManager, topologyName);
    PackingPlan packing = provider.get();
    Assert.assertEquals(1, packing.getContainers().size());
    // once fetched it is cached
    provider.get();
    verify(adaptor, times(1)).getPackingPlan(topologyName);
}

Example 33 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class LauncherUtilsTest method constructsConfigWithTopologyInfo.

@Test
public void constructsConfigWithTopologyInfo() throws Exception {
    TopologyAPI.Topology mockTopology = PowerMockito.mock(TopologyAPI.Topology.class);
    PowerMockito.when(mockTopology.getId()).thenReturn("testTopologyId");
    PowerMockito.when(mockTopology.getName()).thenReturn("testTopologyName");
    SchedulerStateManagerAdaptor mockStMgr = Mockito.mock(SchedulerStateManagerAdaptor.class);
    PowerMockito.spy(TopologyUtils.class);
    PowerMockito.doReturn(456).when(TopologyUtils.class, "getNumContainers", mockTopology);
    Config runtime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(mockTopology)).putAll(LauncherUtils.getInstance().createAdaptorRuntime(mockStMgr)).build();
    Assert.assertEquals("testTopologyId", Runtime.topologyId(runtime));
    Assert.assertEquals("testTopologyName", Runtime.topologyName(runtime));
    Assert.assertEquals(mockTopology, Runtime.topology(runtime));
    Assert.assertEquals(mockStMgr, Runtime.schedulerStateManagerAdaptor(runtime));
    Assert.assertEquals(456 + 1, Runtime.numContainers(runtime).longValue());
}

Example 34 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class AuroraSchedulerTest method testOnSchedule.

/**
 * Tests that we can schedule
 */
@Test
public void testOnSchedule() throws Exception {
    AuroraController controller = Mockito.mock(AuroraController.class);
    doReturn(controller).when(scheduler).getController();
    SchedulerStateManagerAdaptor stateManager = mock(SchedulerStateManagerAdaptor.class);
    Config runtime = Mockito.mock(Config.class);
    when(runtime.get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR)).thenReturn(stateManager);
    when(runtime.getStringValue(Key.TOPOLOGY_NAME)).thenReturn(TOPOLOGY_NAME);
    Config mConfig = Mockito.mock(Config.class);
    PowerMockito.mockStatic(Config.class);
    when(Config.toClusterMode(mConfig)).thenReturn(mConfig);
    when(mConfig.getStringValue(eq(AuroraContext.JOB_TEMPLATE), anyString())).thenReturn(AURORA_PATH);
    scheduler.initialize(mConfig, runtime);
    // Fail to schedule due to null PackingPlan
    Assert.assertFalse(scheduler.onSchedule(null));
    PackingPlan plan = new PackingPlan(PACKING_PLAN_ID, new HashSet<PackingPlan.ContainerPlan>());
    assertTrue(plan.getContainers().isEmpty());
    // Fail to schedule due to PackingPlan is empty
    Assert.assertFalse(scheduler.onSchedule(plan));
    // Construct valid PackingPlan
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(PackingTestUtils.testContainerPlan(CONTAINER_ID));
    PackingPlan validPlan = new PackingPlan(PACKING_PLAN_ID, containers);
    // Failed to create job via controller
    doReturn(false).when(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    doReturn(true).when(stateManager).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
    Assert.assertFalse(scheduler.onSchedule(validPlan));
    Mockito.verify(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    Mockito.verify(stateManager).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
    // Happy path
    doReturn(true).when(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    assertTrue(scheduler.onSchedule(validPlan));
    Mockito.verify(controller, Mockito.times(2)).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    Mockito.verify(stateManager, Mockito.times(2)).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
}

Example 35 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class LaunchRunner method call.

/**
 * Call launcher to launch topology
 *
 * @throws LauncherException
 * @throws PackingException
 * @throws SubmitDryRunResponse
 */
public void call() throws LauncherException, PackingException, SubmitDryRunResponse {
    SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
    TopologyAPI.Topology topology = Runtime.topology(runtime);
    String topologyName = Context.topologyName(config);
    PackingPlan packedPlan = LauncherUtils.getInstance().createPackingPlan(config, runtime);
    if (Context.dryRun(config)) {
        throw new SubmitDryRunResponse(topology, config, packedPlan);
    }
    // initialize the launcher
    launcher.initialize(config, runtime);
    // Set topology def first since we determine whether a topology is running
    // by checking the existence of topology def
    // store the trimmed topology definition into the state manager
    // TODO(rli): log-and-false anti-pattern is too nested on this path. will not refactor
    Boolean result = statemgr.setTopology(trimTopology(topology), topologyName);
    if (result == null || !result) {
        throw new LauncherException(String.format("Failed to set topology definition for topology '%s'", topologyName));
    }
    result = statemgr.setPackingPlan(createPackingPlan(packedPlan), topologyName);
    if (result == null || !result) {
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set packing plan for topology '%s'", topologyName));
    }
    // store the execution state into the state manager
    ExecutionEnvironment.ExecutionState executionState = createExecutionState();
    result = statemgr.setExecutionState(executionState, topologyName);
    if (result == null || !result) {
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set execution state for topology '%s'", topologyName));
    }
    // returning false. In some cases the scheduler needs to have the topology deleted.
    try {
        if (!launcher.launch(packedPlan)) {
            throw new TopologySubmissionException(null);
        }
    } catch (TopologySubmissionException e) {
        // Compile error message to throw.
        final StringBuilder errorMessage = new StringBuilder(String.format("Failed to launch topology '%s'", topologyName));
        if (e.getMessage() != null) {
            errorMessage.append("\n").append(e.getMessage());
        }
        try {
            // Clear state from the Scheduler via RPC.
            Scheduler.KillTopologyRequest killTopologyRequest = Scheduler.KillTopologyRequest.newBuilder().setTopologyName(topologyName).build();
            ISchedulerClient schedulerClient = new SchedulerClientFactory(config, runtime).getSchedulerClient();
            if (!schedulerClient.killTopology(killTopologyRequest)) {
                final String logMessage = String.format("Failed to remove topology '%s' from scheduler after failed submit. " + "Please re-try the kill command.", topologyName);
                errorMessage.append("\n").append(logMessage);
                LOG.log(Level.SEVERE, logMessage);
            }
        // SUPPRESS CHECKSTYLE IllegalCatch
        } catch (Exception ignored) {
            // The above call to clear the Scheduler may fail. This situation can be ignored.
            LOG.log(Level.FINE, String.format("Failure clearing failed topology `%s` from Scheduler during `submit`", topologyName));
        }
        // Clear state from the State Manager.
        statemgr.deleteExecutionState(topologyName);
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(errorMessage.toString());
    }
}