Examples with JobModel org.apache.samza.job.model.JobModel used on opensource projects

Example 1 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class ZkJobCoordinator method doOnProcessorChange.

void doOnProcessorChange(List<String> processors) {
    // if list of processors is empty - it means we are called from 'onBecomeLeader'
    // TODO: Handle empty currentProcessorIds or duplicate processorIds in the list
    List<String> currentProcessorIds = getActualProcessorIds(processors);
    // Generate the JobModel
    JobModel jobModel = generateNewJobModel(currentProcessorIds);
    // Assign the next version of JobModel
    String currentJMVersion = zkUtils.getJobModelVersion();
    String nextJMVersion;
    if (currentJMVersion == null) {
        nextJMVersion = "1";
    } else {
        nextJMVersion = Integer.toString(Integer.valueOf(currentJMVersion) + 1);
    }
    LOG.info("pid=" + processorId + "Generated new Job Model. Version = " + nextJMVersion);
    // Publish the new job model
    zkUtils.publishJobModel(nextJMVersion, jobModel);
    // Start the barrier for the job model update
    barrier.create(nextJMVersion, currentProcessorIds);
    // Notify all processors about the new JobModel by updating JobModel Version number
    zkUtils.publishJobModelVersion(currentJMVersion, nextJMVersion);
    LOG.info("pid=" + processorId + "Published new Job Model. Version = " + nextJMVersion);
}

Example 2 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class ZkUtils method getJobModel.

/**
   * get the job model from ZK by version
   * @param jobModelVersion jobModel version to get
   * @return job model for this version
   */
public JobModel getJobModel(String jobModelVersion) {
    LOG.info("read the model ver=" + jobModelVersion + " from " + keyBuilder.getJobModelPath(jobModelVersion));
    Object data = zkClient.readData(keyBuilder.getJobModelPath(jobModelVersion));
    ObjectMapper mmapper = SamzaObjectMapper.getObjectMapper();
    JobModel jm;
    try {
        jm = mmapper.readValue((String) data, JobModel.class);
    } catch (IOException e) {
        throw new SamzaException("failed to read JobModel from ZK", e);
    }
    return jm;
}

Example 3 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class TestSamzaObjectMapper method setup.

@Before
public void setup() throws IOException {
    Map<String, String> configMap = new HashMap<String, String>();
    Set<SystemStreamPartition> ssp = new HashSet<>();
    configMap.put("a", "b");
    Config config = new MapConfig(configMap);
    TaskName taskName = new TaskName("test");
    ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
    TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
    Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
    tasks.put(taskName, taskModel);
    ContainerModel containerModel = new ContainerModel("1", 1, tasks);
    Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
    containerMap.put("1", containerModel);
    jobModel = new JobModel(config, containerMap);
}

Example 4 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class TestHostAwareContainerAllocator method getJobModelManager.

private static JobModelManager getJobModelManager(int containerCount) {
    //Ideally, the JobModelReader should be constructed independent of HttpServer.
    //That way it becomes easier to mock objects. Save it for later.
    HttpServer server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
    Map<String, ContainerModel> containers = new java.util.HashMap<>();
    for (int i = 0; i < containerCount; i++) {
        ContainerModel container = new ContainerModel(String.valueOf(i), i, new HashMap<TaskName, TaskModel>());
        containers.put(String.valueOf(i), container);
    }
    JobModel jobModel = new JobModel(getConfig(), containers);
    return new JobModelManager(jobModel, server, null);
}

Example 5 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class TestZkUtils method testPublishNewJobModel.

@Test
public void testPublishNewJobModel() {
    ZkKeyBuilder keyBuilder = new ZkKeyBuilder("test");
    String root = keyBuilder.getRootPath();
    zkClient.deleteRecursive(root);
    String version = "1";
    String oldVersion = "0";
    zkUtils.makeSurePersistentPathsExists(new String[] { root, keyBuilder.getJobModelPathPrefix(), keyBuilder.getJobModelVersionPath() });
    zkUtils.publishJobModelVersion(oldVersion, version);
    Assert.assertEquals(version, zkUtils.getJobModelVersion());
    String newerVersion = Long.toString(Long.valueOf(version) + 1);
    zkUtils.publishJobModelVersion(version, newerVersion);
    Assert.assertEquals(newerVersion, zkUtils.getJobModelVersion());
    try {
        //invalid new version
        zkUtils.publishJobModelVersion(oldVersion, "10");
        Assert.fail("publish invalid version should've failed");
    } catch (SamzaException e) {
    // expected
    }
    // create job model
    Map<String, String> configMap = new HashMap<>();
    Map<String, ContainerModel> containers = new HashMap<>();
    MapConfig config = new MapConfig(configMap);
    JobModel jobModel = new JobModel(config, containers);
    zkUtils.publishJobModel(version, jobModel);
    Assert.assertEquals(jobModel, zkUtils.getJobModel(version));
}