use of org.apache.samza.config.StorageConfig in project samza by apache.
the class SamzaTaskProxy method getTasks.
/**
* Fetches the complete job model from the coordinator stream based upon the provided {@link JobInstance}
* param, transforms it to a list of {@link Task} and returns it.
* {@inheritDoc}
*/
@Override
public List<Task> getTasks(JobInstance jobInstance) throws IOException, InterruptedException {
Preconditions.checkArgument(installFinder.isInstalled(jobInstance), String.format("Invalid job instance : %s", jobInstance));
JobModel jobModel = getJobModel(jobInstance);
StorageConfig storageConfig = new StorageConfig(jobModel.getConfig());
List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava();
Map<String, String> containerLocality = jobModel.getAllContainerLocality();
List<Task> tasks = new ArrayList<>();
for (ContainerModel containerModel : jobModel.getContainers().values()) {
String containerId = containerModel.getProcessorId();
String host = containerLocality.get(containerId);
for (TaskModel taskModel : containerModel.getTasks().values()) {
String taskName = taskModel.getTaskName().getTaskName();
List<Partition> partitions = taskModel.getSystemStreamPartitions().stream().map(Partition::new).collect(Collectors.toList());
tasks.add(new Task(host, taskName, containerId, partitions, storeNames));
}
}
return tasks;
}
use of org.apache.samza.config.StorageConfig in project samza by apache.
the class StorageRecovery method getTaskStorageManagers.
/**
* create one TaskStorageManager for each task. Add all of them to the
* List<TaskStorageManager>
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
private void getTaskStorageManagers() {
StreamMetadataCache streamMetadataCache = new StreamMetadataCache(Util.javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
for (ContainerModel containerModel : containers.values()) {
HashMap<String, StorageEngine> taskStores = new HashMap<String, StorageEngine>();
SamzaContainerContext containerContext = new SamzaContainerContext(containerModel.getProcessorId(), jobConfig, containerModel.getTasks().keySet());
for (TaskModel taskModel : containerModel.getTasks().values()) {
HashMap<String, SystemConsumer> storeConsumers = getStoreConsumers();
for (Entry<String, StorageEngineFactory<?, ?>> entry : storageEngineFactories.entrySet()) {
String storeName = entry.getKey();
if (changeLogSystemStreams.containsKey(storeName)) {
SystemStreamPartition changeLogSystemStreamPartition = new SystemStreamPartition(changeLogSystemStreams.get(storeName), taskModel.getChangelogPartition());
File storePartitionDir = TaskStorageManager.getStorePartitionDir(storeBaseDir, storeName, taskModel.getTaskName());
log.info("Got storage engine directory: " + storePartitionDir);
StorageEngine storageEngine = (entry.getValue()).getStorageEngine(storeName, storePartitionDir, (Serde) new ByteSerde(), (Serde) new ByteSerde(), null, new MetricsRegistryMap(), changeLogSystemStreamPartition, containerContext);
taskStores.put(storeName, storageEngine);
}
}
TaskStorageManager taskStorageManager = new TaskStorageManager(taskModel.getTaskName(), Util.javaMapAsScalaMap(taskStores), Util.javaMapAsScalaMap(storeConsumers), Util.javaMapAsScalaMap(changeLogSystemStreams), maxPartitionNumber, streamMetadataCache, storeBaseDir, storeBaseDir, taskModel.getChangelogPartition(), Util.javaMapAsScalaMap(systemAdmins), new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
taskStorageManagers.add(taskStorageManager);
}
}
}
Aggregations