use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TaskConfigJava method getBroadcastSystemStreams.
/**
* Get the SystemStreams for the configured broadcast streams.
*
* @return the set of SystemStreams for which there are broadcast stream SSPs configured.
*/
public Set<SystemStream> getBroadcastSystemStreams() {
Set<SystemStream> broadcastSS = new HashSet<>();
Set<SystemStreamPartition> broadcastSSPs = getBroadcastSystemStreamPartitions();
for (SystemStreamPartition bssp : broadcastSSPs) {
broadcastSS.add(bssp.getSystemStream());
}
return Collections.unmodifiableSet(broadcastSS);
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TaskConfigJava method getBroadcastSystemStreamPartitions.
/**
* Get the systemStreamPartitions of the broadcast stream. Specifying
* one partition for one stream or a range of the partitions for one
* stream is allowed.
*
* @return a Set of SystemStreamPartitions
*/
public Set<SystemStreamPartition> getBroadcastSystemStreamPartitions() {
HashSet<SystemStreamPartition> systemStreamPartitionSet = new HashSet<SystemStreamPartition>();
List<String> systemStreamPartitions = getList(BROADCAST_INPUT_STREAMS, Collections.<String>emptyList());
for (String systemStreamPartition : systemStreamPartitions) {
int hashPosition = systemStreamPartition.indexOf("#");
if (hashPosition == -1) {
throw new IllegalArgumentException("incorrect format in " + systemStreamPartition + ". Broadcast stream names should be in the form 'system.stream#partitionId' or 'system.stream#[partitionN-partitionM]'");
} else {
String systemStreamName = systemStreamPartition.substring(0, hashPosition);
String partitionSegment = systemStreamPartition.substring(hashPosition + 1);
SystemStream systemStream = Util.getSystemStreamFromNames(systemStreamName);
if (Pattern.matches(BROADCAST_STREAM_PATTERN, partitionSegment)) {
systemStreamPartitionSet.add(new SystemStreamPartition(systemStream, new Partition(Integer.valueOf(partitionSegment))));
} else {
if (Pattern.matches(BROADCAST_STREAM_RANGE_PATTERN, partitionSegment)) {
int partitionStart = Integer.valueOf(partitionSegment.substring(1, partitionSegment.lastIndexOf("-")));
int partitionEnd = Integer.valueOf(partitionSegment.substring(partitionSegment.lastIndexOf("-") + 1, partitionSegment.indexOf("]")));
if (partitionStart > partitionEnd) {
LOGGER.warn("The starting partition in stream " + systemStream.toString() + " is bigger than the ending Partition. No partition is added");
}
for (int i = partitionStart; i <= partitionEnd; i++) {
systemStreamPartitionSet.add(new SystemStreamPartition(systemStream, new Partition(i)));
}
} else {
throw new IllegalArgumentException("incorrect format in " + systemStreamPartition + ". Broadcast stream names should be in the form 'system.stream#partitionId' or 'system.stream#[partitionN-partitionM]'");
}
}
}
}
return systemStreamPartitionSet;
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class StorageRecovery method getTaskStorageManagers.
/**
* create one TaskStorageManager for each task. Add all of them to the
* List<TaskStorageManager>
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
private void getTaskStorageManagers() {
StreamMetadataCache streamMetadataCache = new StreamMetadataCache(Util.javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
for (ContainerModel containerModel : containers.values()) {
HashMap<String, StorageEngine> taskStores = new HashMap<String, StorageEngine>();
SamzaContainerContext containerContext = new SamzaContainerContext(containerModel.getProcessorId(), jobConfig, containerModel.getTasks().keySet());
for (TaskModel taskModel : containerModel.getTasks().values()) {
HashMap<String, SystemConsumer> storeConsumers = getStoreConsumers();
for (Entry<String, StorageEngineFactory<?, ?>> entry : storageEngineFactories.entrySet()) {
String storeName = entry.getKey();
if (changeLogSystemStreams.containsKey(storeName)) {
SystemStreamPartition changeLogSystemStreamPartition = new SystemStreamPartition(changeLogSystemStreams.get(storeName), taskModel.getChangelogPartition());
File storePartitionDir = TaskStorageManager.getStorePartitionDir(storeBaseDir, storeName, taskModel.getTaskName());
log.info("Got storage engine directory: " + storePartitionDir);
StorageEngine storageEngine = (entry.getValue()).getStorageEngine(storeName, storePartitionDir, (Serde) new ByteSerde(), (Serde) new ByteSerde(), null, new MetricsRegistryMap(), changeLogSystemStreamPartition, containerContext);
taskStores.put(storeName, storageEngine);
}
}
TaskStorageManager taskStorageManager = new TaskStorageManager(taskModel.getTaskName(), Util.javaMapAsScalaMap(taskStores), Util.javaMapAsScalaMap(storeConsumers), Util.javaMapAsScalaMap(changeLogSystemStreams), maxPartitionNumber, streamMetadataCache, storeBaseDir, storeBaseDir, taskModel.getChangelogPartition(), Util.javaMapAsScalaMap(systemAdmins), new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
taskStorageManagers.add(taskStorageManager);
}
}
}
Aggregations