use of org.apache.samza.system.SystemAdmin in project samza by apache.
the class PassthroughJobCoordinator method getJobModel.
@Override
public JobModel getJobModel() {
JavaSystemConfig systemConfig = new JavaSystemConfig(this.config);
Map<String, SystemAdmin> systemAdmins = new HashMap<>();
for (String systemName : systemConfig.getSystemNames()) {
String systemFactoryClassName = systemConfig.getSystemFactory(systemName);
if (systemFactoryClassName == null) {
LOGGER.error(String.format("A stream uses system %s, which is missing from the configuration.", systemName));
throw new SamzaException(String.format("A stream uses system %s, which is missing from the configuration.", systemName));
}
SystemFactory systemFactory = Util.<SystemFactory>getObj(systemFactoryClassName);
systemAdmins.put(systemName, systemFactory.getAdmin(systemName, this.config));
}
StreamMetadataCache streamMetadataCache = new StreamMetadataCache(Util.<String, SystemAdmin>javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
/** TODO:
Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also,
in SamzaContainer for writing locality info to the coordinator stream. This closely couples together
TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator
(job.coordinator.task.grouper, instead of task.systemstreampartition.grouper)
*/
return JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, null);
}
use of org.apache.samza.system.SystemAdmin in project samza by apache.
the class TestJobGraphJsonGenerator method test.
@Test
public void test() throws Exception {
/**
* the graph looks like the following. number of partitions in parentheses. quotes indicate expected value.
*
* input1 (64) -> map -> join -> output1 (8)
* |
* input2 (16) -> partitionBy ("64") -> filter -|
* |
* input3 (32) -> filter -> partitionBy ("64") -> map -> join -> output2 (16)
*
*/
Map<String, String> configMap = new HashMap<>();
configMap.put(JobConfig.JOB_NAME(), "test-app");
configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system");
Config config = new MapConfig(configMap);
StreamSpec input1 = new StreamSpec("input1", "input1", "system1");
StreamSpec input2 = new StreamSpec("input2", "input2", "system2");
StreamSpec input3 = new StreamSpec("input3", "input3", "system2");
StreamSpec output1 = new StreamSpec("output1", "output1", "system1");
StreamSpec output2 = new StreamSpec("output2", "output2", "system2");
ApplicationRunner runner = mock(ApplicationRunner.class);
when(runner.getStreamSpec("input1")).thenReturn(input1);
when(runner.getStreamSpec("input2")).thenReturn(input2);
when(runner.getStreamSpec("input3")).thenReturn(input3);
when(runner.getStreamSpec("output1")).thenReturn(output1);
when(runner.getStreamSpec("output2")).thenReturn(output2);
// intermediate streams used in tests
when(runner.getStreamSpec("test-app-1-partition_by-0")).thenReturn(new StreamSpec("test-app-1-partition_by-0", "test-app-1-partition_by-0", "default-system"));
when(runner.getStreamSpec("test-app-1-partition_by-1")).thenReturn(new StreamSpec("test-app-1-partition_by-1", "test-app-1-partition_by-1", "default-system"));
when(runner.getStreamSpec("test-app-1-partition_by-4")).thenReturn(new StreamSpec("test-app-1-partition_by-4", "test-app-1-partition_by-4", "default-system"));
// set up external partition count
Map<String, Integer> system1Map = new HashMap<>();
system1Map.put("input1", 64);
system1Map.put("output1", 8);
Map<String, Integer> system2Map = new HashMap<>();
system2Map.put("input2", 16);
system2Map.put("input3", 32);
system2Map.put("output2", 16);
Map<String, SystemAdmin> systemAdmins = new HashMap<>();
SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
systemAdmins.put("system1", systemAdmin1);
systemAdmins.put("system2", systemAdmin2);
StreamManager streamManager = new StreamManager(systemAdmins);
StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
BiFunction mockBuilder = mock(BiFunction.class);
MessageStream m1 = streamGraph.getInputStream("input1", mockBuilder).map(m -> m);
MessageStream m2 = streamGraph.getInputStream("input2", mockBuilder).partitionBy(m -> "haha").filter(m -> true);
MessageStream m3 = streamGraph.getInputStream("input3", mockBuilder).filter(m -> true).partitionBy(m -> "hehe").map(m -> m);
Function mockFn = mock(Function.class);
OutputStream<Object, Object, Object> outputStream1 = streamGraph.getOutputStream("output1", mockFn, mockFn);
OutputStream<Object, Object, Object> outputStream2 = streamGraph.getOutputStream("output2", mockFn, mockFn);
m1.join(m2, mock(JoinFunction.class), Duration.ofHours(2)).sendTo(outputStream1);
m2.sink((message, collector, coordinator) -> {
});
m3.join(m2, mock(JoinFunction.class), Duration.ofHours(1)).sendTo(outputStream2);
ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
ExecutionPlan plan = planner.plan(streamGraph);
String json = plan.getPlanAsJson();
System.out.println(json);
// deserialize
ObjectMapper mapper = new ObjectMapper();
JobGraphJsonGenerator.JobGraphJson nodes = mapper.readValue(json, JobGraphJsonGenerator.JobGraphJson.class);
assertTrue(nodes.jobs.get(0).operatorGraph.inputStreams.size() == 5);
assertTrue(nodes.jobs.get(0).operatorGraph.operators.size() == 13);
assertTrue(nodes.sourceStreams.size() == 3);
assertTrue(nodes.sinkStreams.size() == 2);
assertTrue(nodes.intermediateStreams.size() == 2);
}
use of org.apache.samza.system.SystemAdmin in project samza by apache.
the class TestExecutionPlanner method createSystemAdmin.
static SystemAdmin createSystemAdmin(Map<String, Integer> streamToPartitions) {
return new SystemAdmin() {
@Override
public Map<SystemStreamPartition, String> getOffsetsAfter(Map<SystemStreamPartition, String> offsets) {
return null;
}
@Override
public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
Map<String, SystemStreamMetadata> map = new HashMap<>();
for (String stream : streamNames) {
Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> m = new HashMap<>();
for (int i = 0; i < streamToPartitions.get(stream); i++) {
m.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", ""));
}
map.put(stream, new SystemStreamMetadata(stream, m));
}
return map;
}
@Override
public void createChangelogStream(String streamName, int numOfPartitions) {
}
@Override
public void validateChangelogStream(String streamName, int numOfPartitions) {
}
@Override
public void createCoordinatorStream(String streamName) {
}
@Override
public Integer offsetComparator(String offset1, String offset2) {
return null;
}
};
}
use of org.apache.samza.system.SystemAdmin in project samza by apache.
the class TestExecutionPlanner method setup.
@Before
public void setup() {
Map<String, String> configMap = new HashMap<>();
configMap.put(JobConfig.JOB_NAME(), "test-app");
configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), DEFAULT_SYSTEM);
config = new MapConfig(configMap);
input1 = new StreamSpec("input1", "input1", "system1");
input2 = new StreamSpec("input2", "input2", "system2");
input3 = new StreamSpec("input3", "input3", "system2");
output1 = new StreamSpec("output1", "output1", "system1");
output2 = new StreamSpec("output2", "output2", "system2");
// set up external partition count
Map<String, Integer> system1Map = new HashMap<>();
system1Map.put("input1", 64);
system1Map.put("output1", 8);
Map<String, Integer> system2Map = new HashMap<>();
system2Map.put("input2", 16);
system2Map.put("input3", 32);
system2Map.put("output2", 16);
SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
systemAdmins = new HashMap<>();
systemAdmins.put("system1", systemAdmin1);
systemAdmins.put("system2", systemAdmin2);
streamManager = new StreamManager(systemAdmins);
runner = mock(ApplicationRunner.class);
when(runner.getStreamSpec("input1")).thenReturn(input1);
when(runner.getStreamSpec("input2")).thenReturn(input2);
when(runner.getStreamSpec("input3")).thenReturn(input3);
when(runner.getStreamSpec("output1")).thenReturn(output1);
when(runner.getStreamSpec("output2")).thenReturn(output2);
// intermediate streams used in tests
when(runner.getStreamSpec("test-app-1-partition_by-0")).thenReturn(new StreamSpec("test-app-1-partition_by-0", "test-app-1-partition_by-0", "default-system"));
when(runner.getStreamSpec("test-app-1-partition_by-1")).thenReturn(new StreamSpec("test-app-1-partition_by-1", "test-app-1-partition_by-1", "default-system"));
when(runner.getStreamSpec("test-app-1-partition_by-4")).thenReturn(new StreamSpec("test-app-1-partition_by-4", "test-app-1-partition_by-4", "default-system"));
}
use of org.apache.samza.system.SystemAdmin in project samza by apache.
the class TestKafkaSystemAdminJava method testCreateStream.
@Test
public void testCreateStream() {
SystemAdmin admin = this.basicSystemAdmin;
StreamSpec spec = new StreamSpec("testId", "testStream", "testSystem", 8);
assertTrue("createStream should return true if the stream does not exist and then is created.", admin.createStream(spec));
admin.validateStream(spec);
assertFalse("createStream should return false if the stream already exists.", admin.createStream(spec));
}
Aggregations