Examples with RemoteJobPlanner org.apache.samza.execution.RemoteJobPlanner used on opensource projects

Example 1 with RemoteJobPlanner

use of org.apache.samza.execution.RemoteJobPlanner in project samza by apache.

the class TestJobCoordinatorLaunchUtil method testRunJobCoordinator.

@Test
public void testRunJobCoordinator() throws Exception {
    String jobCoordinatorFactoryClass = "org.apache.samza.custom.MyJobCoordinatorFactory";
    Config originalConfig = buildOriginalConfig(ImmutableMap.of(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, jobCoordinatorFactoryClass));
    JobConfig fullConfig = new JobConfig(new MapConfig(originalConfig, Collections.singletonMap("isAfterPlanning", "true")));
    Config autoSizingConfig = new MapConfig(Collections.singletonMap(JobConfig.JOB_AUTOSIZING_CONTAINER_COUNT, "10"));
    Config finalConfig = new MapConfig(autoSizingConfig, fullConfig);
    RemoteJobPlanner remoteJobPlanner = mock(RemoteJobPlanner.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    JobCoordinatorFactory jobCoordinatorFactory = mock(JobCoordinatorFactory.class);
    JobCoordinator jobCoordinator = mock(JobCoordinator.class);
    PowerMockito.mockStatic(CoordinatorStreamUtil.class);
    PowerMockito.doNothing().when(CoordinatorStreamUtil.class, "createCoordinatorStream", any());
    PowerMockito.doReturn(new MapConfig()).when(CoordinatorStreamUtil.class, "buildCoordinatorStreamConfig", any());
    PowerMockito.doReturn(autoSizingConfig).when(CoordinatorStreamUtil.class, "readLaunchConfigFromCoordinatorStream", any(), any());
    PowerMockito.whenNew(CoordinatorStreamStore.class).withAnyArguments().thenReturn(coordinatorStreamStore);
    PowerMockito.whenNew(RemoteJobPlanner.class).withAnyArguments().thenReturn(remoteJobPlanner);
    when(remoteJobPlanner.prepareJobs()).thenReturn(Collections.singletonList(fullConfig));
    PowerMockito.mockStatic(ReflectionUtil.class);
    PowerMockito.doReturn(jobCoordinatorFactory).when(ReflectionUtil.class, "getObj", jobCoordinatorFactoryClass, JobCoordinatorFactory.class);
    when(jobCoordinatorFactory.getJobCoordinator(eq("samza-job-coordinator"), eq(finalConfig), any(), eq(coordinatorStreamStore))).thenReturn(jobCoordinator);
    // use a latch to keep track of when shutdown hook was added to know when we should start verifications
    CountDownLatch addShutdownHookLatch = new CountDownLatch(1);
    PowerMockito.spy(JobCoordinatorLaunchUtil.class);
    PowerMockito.doAnswer(invocation -> {
        addShutdownHookLatch.countDown();
        return null;
    }).when(JobCoordinatorLaunchUtil.class, "addShutdownHook", any());
    MetricsReporter metricsReporter = mock(MetricsReporter.class);
    Map<String, MetricsReporter> metricsReporterMap = ImmutableMap.of("reporter", metricsReporter);
    PowerMockito.mockStatic(MetricsReporterLoader.class);
    PowerMockito.doReturn(metricsReporterMap).when(MetricsReporterLoader.class, "getMetricsReporters", new MetricsConfig(finalConfig), "JobCoordinator");
    NoProcessorJobCoordinatorListener jobCoordinatorListener = mock(NoProcessorJobCoordinatorListener.class);
    PowerMockito.whenNew(NoProcessorJobCoordinatorListener.class).withAnyArguments().thenReturn(jobCoordinatorListener);
    Thread runThread = new Thread(() -> JobCoordinatorLaunchUtil.run(new MockStreamApplication(), originalConfig));
    runThread.start();
    // last thing before waiting for shutdown is to add shutdown hook, so do verifications once hook is added
    addShutdownHookLatch.await();
    verifyStatic();
    CoordinatorStreamUtil.createCoordinatorStream(fullConfig);
    verifyStatic();
    CoordinatorStreamUtil.writeConfigToCoordinatorStream(finalConfig, true);
    verifyStatic();
    JobCoordinatorLaunchUtil.addShutdownHook(jobCoordinator);
    InOrder inOrder = Mockito.inOrder(metricsReporter, jobCoordinator);
    inOrder.verify(metricsReporter).register(eq("JobCoordinator"), any());
    inOrder.verify(metricsReporter).start();
    ArgumentCaptor<CountDownLatch> countDownLatchArgumentCaptor = ArgumentCaptor.forClass(CountDownLatch.class);
    verifyNew(NoProcessorJobCoordinatorListener.class).withArguments(countDownLatchArgumentCaptor.capture());
    inOrder.verify(jobCoordinator).setListener(jobCoordinatorListener);
    inOrder.verify(jobCoordinator).start();
    // wait some time and then make sure the run thread is still alive
    Thread.sleep(Duration.ofMillis(500).toMillis());
    assertTrue(runThread.isAlive());
    // trigger the count down latch so that the run thread can exit
    countDownLatchArgumentCaptor.getValue().countDown();
    runThread.join(Duration.ofSeconds(10).toMillis());
    assertFalse(runThread.isAlive());
    verify(metricsReporter).stop();
}

Example 2 with RemoteJobPlanner

use of org.apache.samza.execution.RemoteJobPlanner in project samza by apache.

the class RemoteApplicationRunner method run.

@Override
public void run(ExternalContext externalContext) {
    if (new JobConfig(config).getConfigLoaderFactory().isPresent()) {
        JobRunner runner = new JobRunner(JobPlanner.generateSingleJobConfig(config));
        runner.submit();
        return;
    }
    // TODO SAMZA-2432: Clean this up once SAMZA-2405 is completed when legacy flow is removed.
    try {
        JobPlanner planner = new RemoteJobPlanner(ApplicationDescriptorUtil.getAppDescriptor(app, config));
        List<JobConfig> jobConfigs = planner.prepareJobs();
        if (jobConfigs.isEmpty()) {
            throw new SamzaException("No jobs to run.");
        }
        // 3. submit jobs for remote execution
        jobConfigs.forEach(jobConfig -> {
            LOG.info("Starting job {} with config {}", jobConfig.getName(), jobConfig);
            JobRunner runner = new JobRunner(jobConfig);
            runner.run(true);
        });
    } catch (Throwable t) {
        throw new SamzaException("Failed to run application", t);
    }
}

Example 3 with RemoteJobPlanner

use of org.apache.samza.execution.RemoteJobPlanner in project samza by apache.

the class TestJobCoordinatorLaunchUtil method testRunClusterBasedJobCoordinator.

@Test
public void testRunClusterBasedJobCoordinator() throws Exception {
    Config originalConfig = buildOriginalConfig(ImmutableMap.of());
    JobConfig fullConfig = new JobConfig(new MapConfig(originalConfig, Collections.singletonMap("isAfterPlanning", "true")));
    Config autoSizingConfig = new MapConfig(Collections.singletonMap(JobConfig.JOB_AUTOSIZING_CONTAINER_COUNT, "10"));
    Config finalConfig = new MapConfig(autoSizingConfig, fullConfig);
    RemoteJobPlanner mockJobPlanner = mock(RemoteJobPlanner.class);
    CoordinatorStreamStore mockCoordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ClusterBasedJobCoordinator mockJC = mock(ClusterBasedJobCoordinator.class);
    PowerMockito.mockStatic(CoordinatorStreamUtil.class);
    PowerMockito.doNothing().when(CoordinatorStreamUtil.class, "createCoordinatorStream", any());
    PowerMockito.doReturn(new MapConfig()).when(CoordinatorStreamUtil.class, "buildCoordinatorStreamConfig", any());
    PowerMockito.doReturn(autoSizingConfig).when(CoordinatorStreamUtil.class, "readLaunchConfigFromCoordinatorStream", any(), any());
    PowerMockito.whenNew(CoordinatorStreamStore.class).withAnyArguments().thenReturn(mockCoordinatorStreamStore);
    PowerMockito.whenNew(RemoteJobPlanner.class).withAnyArguments().thenReturn(mockJobPlanner);
    PowerMockito.whenNew(ClusterBasedJobCoordinator.class).withAnyArguments().thenReturn(mockJC);
    when(mockJobPlanner.prepareJobs()).thenReturn(Collections.singletonList(fullConfig));
    JobCoordinatorLaunchUtil.run(new MockStreamApplication(), originalConfig);
    verifyNew(ClusterBasedJobCoordinator.class).withArguments(any(MetricsRegistryMap.class), eq(mockCoordinatorStreamStore), eq(finalConfig));
    verify(mockJC, times(1)).run();
    verifyStatic(times(1));
    CoordinatorStreamUtil.createCoordinatorStream(fullConfig);
    verifyStatic(times(1));
    CoordinatorStreamUtil.writeConfigToCoordinatorStream(finalConfig, true);
}

Example 4 with RemoteJobPlanner

use of org.apache.samza.execution.RemoteJobPlanner in project samza by apache.

the class JobCoordinatorLaunchUtil method run.

/**
 * Run {@link ClusterBasedJobCoordinator} with full job config.
 *
 * @param app SamzaApplication to run.
 * @param config full job config.
 */
@SuppressWarnings("rawtypes")
public static void run(SamzaApplication app, Config config) {
    // Execute planning
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(app, config);
    RemoteJobPlanner planner = new RemoteJobPlanner(appDesc);
    List<JobConfig> jobConfigs = planner.prepareJobs();
    if (jobConfigs.size() != 1) {
        throw new SamzaException("Only support single remote job is supported.");
    }
    Config fullConfig = jobConfigs.get(0);
    // Create coordinator stream if does not exist before fetching launch config from it.
    CoordinatorStreamUtil.createCoordinatorStream(fullConfig);
    MetricsRegistryMap metrics = new MetricsRegistryMap();
    MetadataStore metadataStore = new CoordinatorStreamStore(CoordinatorStreamUtil.buildCoordinatorStreamConfig(fullConfig), metrics);
    // MetadataStore will be closed in ClusterBasedJobCoordinator#onShutDown
    // initialization of MetadataStore can be moved to ClusterBasedJobCoordinator after we clean up
    // ClusterBasedJobCoordinator#createFromMetadataStore
    metadataStore.init();
    // Reads extra launch config from metadata store.
    Config launchConfig = CoordinatorStreamUtil.readLaunchConfigFromCoordinatorStream(fullConfig, metadataStore);
    Config finalConfig = new MapConfig(launchConfig, fullConfig);
    // This needs to be consistent with RemoteApplicationRunner#run where JobRunner#submit to be called instead of JobRunner#run
    CoordinatorStreamUtil.writeConfigToCoordinatorStream(finalConfig, true);
    DiagnosticsUtil.createDiagnosticsStream(finalConfig);
    Optional<String> jobCoordinatorFactoryClassName = new JobCoordinatorConfig(config).getOptionalJobCoordinatorFactoryClassName();
    if (jobCoordinatorFactoryClassName.isPresent()) {
        runJobCoordinator(jobCoordinatorFactoryClassName.get(), metrics, metadataStore, finalConfig);
    } else {
        ClusterBasedJobCoordinator jc = new ClusterBasedJobCoordinator(metrics, metadataStore, finalConfig);
        jc.run();
    }
}