use of com.hazelcast.logging.Logger in project hazelcast by hazelcast.
the class ReadFilesP method init.
@Override
protected void init(@Nonnull Context context) {
ILogger logger = context.logger();
int processorIndex = sharedFileSystem ? context.globalProcessorIndex() : context.localProcessorIndex();
int parallelism = sharedFileSystem ? context.totalParallelism() : context.localParallelism();
traverser = new LocalFileTraverser<>(logger, directory, glob, ignoreFileNotFound, path -> shouldProcessEvent(path, parallelism, processorIndex), readFileFn);
}
use of com.hazelcast.logging.Logger in project hazelcast by hazelcast.
the class Planner method createDag.
@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
pipeline.makeNamesUnique();
Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
validateNoLeakage(adjacencyMap);
checkTopologicalSort(adjacencyMap.entrySet());
// Find the greatest common denominator of all frame lengths
// appearing in the pipeline
long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
if (frameSizeGcd == 0) {
// even if there are no window aggregations, we want the watermarks for latency debugging
frameSizeGcd = MAXIMUM_WATERMARK_GAP;
}
if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
}
LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
// Update watermark throttling frame length on all transforms with the determined length
for (Transform transform : adjacencyMap.keySet()) {
if (transform instanceof StreamSourceTransform) {
StreamSourceTransform t = (StreamSourceTransform) transform;
EventTimePolicy policy = t.getEventTimePolicy();
if (policy != null) {
t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
}
} else if (transform instanceof TimestampTransform) {
TimestampTransform t = (TimestampTransform) transform;
t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
}
}
// fuse subsequent map/filter/flatMap transforms into one
Map<Transform, List<Transform>> originalParents = new HashMap<>();
List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
for (int i = 0; i < transforms.size(); i++) {
Transform transform = transforms.get(i);
List<Transform> chain = findFusableChain(transform, adjacencyMap);
if (chain == null) {
continue;
}
// remove transforms in the chain and replace the parent with a fused transform
transforms.removeAll(chain.subList(1, chain.size()));
Transform fused = fuseFlatMapTransforms(chain);
transforms.set(i, fused);
Transform lastInChain = chain.get(chain.size() - 1);
for (Transform downstream : adjacencyMap.get(lastInChain)) {
originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
}
}
for (Transform transform : transforms) {
transform.addToDag(this, context);
}
// restore original parents
for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
List<Transform> upstream = en.getKey().upstream();
for (int i = 0; i < upstream.size(); i++) {
en.getKey().upstream().set(i, en.getValue().get(i));
}
}
return dag;
}
Aggregations