Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 6 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class LocalContainerRunner method run.

@Override
public void run(StreamApplication streamApp) {
    ContainerModel containerModel = jobModel.getContainers().get(containerId);
    Object taskFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, this);
    container = SamzaContainer$.MODULE$.apply(containerModel, config, jobModel.maxChangeLogStreamPartitions, Util.<String, MetricsReporter>javaMapAsScalaMap(new HashMap<>()), taskFactory);
    container.setContainerListener(new SamzaContainerListener() {

        @Override
        public void onContainerStart() {
            log.info("Container Started");
        }

        @Override
        public void onContainerStop(boolean invokedExternally) {
            log.info("Container Stopped");
        }

        @Override
        public void onContainerFailed(Throwable t) {
            log.info("Container Failed");
            containerRunnerException = t;
        }
    });
    startContainerHeartbeatMonitor();
    container.run();
    stopContainerHeartbeatMonitor();
    if (containerRunnerException != null) {
        log.error("Container stopped with Exception. Exiting process now.", containerRunnerException);
        System.exit(1);
    }
}

Example 7 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class ClusterBasedJobCoordinator method run.

/**
 * Starts the JobCoordinator.
 */
public void run() {
    if (!isStarted.compareAndSet(false, true)) {
        LOG.warn("Attempting to start an already started job coordinator. ");
        return;
    }
    // set up JmxServer (if jmx is enabled)
    if (isJmxEnabled) {
        jmxServer = new JmxServer();
        state.jmxUrl = jmxServer.getJmxUrl();
        state.jmxTunnelingUrl = jmxServer.getTunnelingJmxUrl();
    } else {
        jmxServer = null;
    }
    try {
        // initialize JobCoordinator state
        LOG.info("Starting cluster based job coordinator");
        // write the diagnostics metadata file
        String jobName = new JobConfig(config).getName().get();
        String jobId = new JobConfig(config).getJobId();
        Optional<String> execEnvContainerId = Optional.ofNullable(System.getenv("CONTAINER_ID"));
        DiagnosticsUtil.writeMetadataFile(jobName, jobId, METRICS_SOURCE_NAME, execEnvContainerId, config);
        // create necessary checkpoint and changelog streams, if not created
        JobModel jobModel = jobModelManager.jobModel();
        MetadataResourceUtil metadataResourceUtil = new MetadataResourceUtil(jobModel, this.metrics, config);
        metadataResourceUtil.createResources();
        // create all the resources required for state backend factories
        StorageConfig storageConfig = new StorageConfig(config);
        storageConfig.getBackupFactories().forEach(stateStorageBackendBackupFactory -> {
            StateBackendFactory stateBackendFactory = ReflectionUtil.getObj(stateStorageBackendBackupFactory, StateBackendFactory.class);
            StateBackendAdmin stateBackendAdmin = stateBackendFactory.getAdmin(jobModel, config);
            // Create resources required for state backend admin
            stateBackendAdmin.createResources();
            // Validate resources required for state backend admin
            stateBackendAdmin.validateResources();
        });
        /*
       * We fanout startpoint if and only if
       *  1. Startpoint is enabled in configuration
       *  2. If AM HA is enabled, fanout only if startpoint enabled and job coordinator metadata changed
       */
        if (shouldFanoutStartpoint()) {
            StartpointManager startpointManager = createStartpointManager();
            startpointManager.start();
            try {
                startpointManager.fanOut(JobModelUtil.getTaskToSystemStreamPartitions(jobModel));
            } finally {
                startpointManager.stop();
            }
        }
        // Remap changelog partitions to tasks
        Map<TaskName, Integer> prevPartitionMappings = changelogStreamManager.readPartitionMapping();
        Map<TaskName, Integer> taskPartitionMappings = new HashMap<>();
        Map<String, ContainerModel> containers = jobModel.getContainers();
        for (ContainerModel containerModel : containers.values()) {
            for (TaskModel taskModel : containerModel.getTasks().values()) {
                taskPartitionMappings.put(taskModel.getTaskName(), taskModel.getChangelogPartition().getPartitionId());
            }
        }
        changelogStreamManager.updatePartitionMapping(prevPartitionMappings, taskPartitionMappings);
        containerProcessManager.start();
        systemAdmins.start();
        partitionMonitor.start();
        inputStreamRegexMonitor.ifPresent(StreamRegexMonitor::start);
        // containerPlacementRequestAllocator thread has to start after the cpm is started
        LOG.info("Starting the container placement handler thread");
        containerPlacementMetadataStore.start();
        containerPlacementRequestAllocatorThread.start();
        boolean isInterrupted = false;
        while (!containerProcessManager.shouldShutdown() && !checkAndThrowException() && !isInterrupted && checkcontainerPlacementRequestAllocatorThreadIsAlive()) {
            try {
                Thread.sleep(jobCoordinatorSleepInterval);
            } catch (InterruptedException e) {
                isInterrupted = true;
                LOG.error("Interrupted in job coordinator loop", e);
                Thread.currentThread().interrupt();
            }
        }
    } catch (Throwable e) {
        LOG.error("Exception thrown in the JobCoordinator loop", e);
        throw new SamzaException(e);
    } finally {
        onShutDown();
    }
}

Example 8 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestStandbyAllocator method getJobModelWithStandby.

// Helper method to create a jobmodel with given number of containers, tasks and replication factor
public static JobModel getJobModelWithStandby(int nContainers, int nTasks, int replicationFactor) {
    Map<String, ContainerModel> containerModels = new HashMap<>();
    int taskID = 0;
    for (int j = 0; j < nContainers; j++) {
        Map<TaskName, TaskModel> tasks = new HashMap<>();
        for (int i = 0; i < nTasks; i++) {
            TaskModel taskModel = getTaskModel(taskID++);
            tasks.put(taskModel.getTaskName(), taskModel);
        }
        containerModels.put(String.valueOf(j), new ContainerModel(String.valueOf(j), tasks));
    }
    Map<String, ContainerModel> standbyContainerModels = new HashMap<>();
    for (int i = 0; i < replicationFactor - 1; i++) {
        for (String containerID : containerModels.keySet()) {
            String standbyContainerId = StandbyTaskUtil.getStandbyContainerId(containerID, i);
            Map<TaskName, TaskModel> standbyTasks = getStandbyTasks(containerModels.get(containerID).getTasks(), i);
            standbyContainerModels.put(standbyContainerId, new ContainerModel(standbyContainerId, standbyTasks));
        }
    }
    containerModels.putAll(standbyContainerModels);
    return new JobModel(new MapConfig(), containerModels);
}

Example 9 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerIds method testShouldGenerateCorrectContainerModelWhenTaskLocalityIsEmpty.

@Test
public void testShouldGenerateCorrectContainerModelWhenTaskLocalityIsEmpty() {
    TaskNameGrouper taskNameGrouper = buildSimpleGrouper(3);
    String testProcessorId1 = "testProcessorId1";
    String testProcessorId2 = "testProcessorId2";
    String testProcessorId3 = "testProcessorId3";
    LocationId testLocationId1 = new LocationId("testLocationId1");
    LocationId testLocationId2 = new LocationId("testLocationId2");
    LocationId testLocationId3 = new LocationId("testLocationId3");
    TaskName testTaskName1 = new TaskName("testTasKId1");
    TaskName testTaskName2 = new TaskName("testTaskId2");
    TaskName testTaskName3 = new TaskName("testTaskId3");
    TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0));
    TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1));
    TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2));
    Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1, testProcessorId2, testLocationId2, testProcessorId3, testLocationId3);
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>());
    Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3);
    Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1)), new ContainerModel(testProcessorId2, ImmutableMap.of(testTaskName2, testTaskModel2)), new ContainerModel(testProcessorId3, ImmutableMap.of(testTaskName3, testTaskModel3)));
    Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
}

Example 10 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerIds method testShouldGenerateIdenticalTaskDistributionWhenNoChangeInProcessorGroup.

@Test
public void testShouldGenerateIdenticalTaskDistributionWhenNoChangeInProcessorGroup() {
    TaskNameGrouper taskNameGrouper = buildSimpleGrouper(3);
    String testProcessorId1 = "testProcessorId1";
    String testProcessorId2 = "testProcessorId2";
    String testProcessorId3 = "testProcessorId3";
    LocationId testLocationId1 = new LocationId("testLocationId1");
    LocationId testLocationId2 = new LocationId("testLocationId2");
    LocationId testLocationId3 = new LocationId("testLocationId3");
    TaskName testTaskName1 = new TaskName("testTasKId1");
    TaskName testTaskName2 = new TaskName("testTaskId2");
    TaskName testTaskName3 = new TaskName("testTaskId3");
    TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0));
    TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1));
    TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2));
    Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1, testProcessorId2, testLocationId2, testProcessorId3, testLocationId3);
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1, testTaskName2, testLocationId2, testTaskName3, testLocationId3);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>());
    Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3);
    Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1)), new ContainerModel(testProcessorId2, ImmutableMap.of(testTaskName2, testTaskModel2)), new ContainerModel(testProcessorId3, ImmutableMap.of(testTaskName3, testTaskModel3)));
    Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
    actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
}