Examples with InputSplit com.baidu.hugegraph.computer.core.input.InputSplit used on opensource projects

Example 1 with InputSplit

use of com.baidu.hugegraph.computer.core.input.InputSplit in project hugegraph-computer by hugegraph.

the class LoaderFileInputSplitFetcher method fetchEdgeInputSplits.

@Override
public List<InputSplit> fetchEdgeInputSplits() {
    List<InputSplit> splits = new ArrayList<>();
    for (InputStruct edgeInputStruct : this.edgeInputStructs) {
        FileSource source = (FileSource) edgeInputStruct.input();
        List<String> paths = this.scanPaths(source);
        if (CollectionUtils.isNotEmpty(paths)) {
            for (String path : paths) {
                FileInputSplit split = new FileInputSplit(ElemType.EDGE, edgeInputStruct, path);
                splits.add(split);
            }
        }
    }
    return splits;
}

Example 2 with InputSplit

use of com.baidu.hugegraph.computer.core.input.InputSplit in project hugegraph-computer by hugegraph.

the class HugeInputSplitFetcher method fetchVertexInputSplits.

@Override
public List<InputSplit> fetchVertexInputSplits() {
    long splitSize = this.config.get(ComputerOptions.INPUT_SPLITS_SIZE);
    int maxSplits = this.config.get(ComputerOptions.INPUT_MAX_SPLITS);
    List<Shard> shards = this.client.traverser().vertexShards(splitSize);
    E.checkArgument(shards.size() <= maxSplits, "Too many shards due to too small splitSize");
    List<InputSplit> splits = new ArrayList<>();
    for (Shard shard : shards) {
        InputSplit split = new InputSplit(shard.start(), shard.end());
        splits.add(split);
    }
    return splits;
}

Example 3 with InputSplit

use of com.baidu.hugegraph.computer.core.input.InputSplit in project hugegraph-computer by hugegraph.

the class LoaderFileInputSplitFetcher method fetchVertexInputSplits.

@Override
public List<InputSplit> fetchVertexInputSplits() {
    List<InputSplit> splits = new ArrayList<>();
    for (InputStruct vertexInputStruct : this.vertexInputStructs) {
        FileSource source = (FileSource) vertexInputStruct.input();
        List<String> paths = this.scanPaths(source);
        if (CollectionUtils.isNotEmpty(paths)) {
            for (String path : paths) {
                FileInputSplit split = new FileInputSplit(ElemType.VERTEX, vertexInputStruct, path);
                splits.add(split);
            }
        }
    }
    return splits;
}

Example 4 with InputSplit

use of com.baidu.hugegraph.computer.core.input.InputSplit in project hugegraph-computer by hugegraph.

the class HugeInputSplitFetcher method fetchEdgeInputSplits.

@Override
public List<InputSplit> fetchEdgeInputSplits() {
    long splitSize = this.config.get(ComputerOptions.INPUT_SPLITS_SIZE);
    int maxSplits = this.config.get(ComputerOptions.INPUT_MAX_SPLITS);
    List<Shard> shards = this.client.traverser().edgeShards(splitSize);
    E.checkArgument(shards.size() <= maxSplits, "Too many shards due to too small splitSize");
    List<InputSplit> splits = new ArrayList<>();
    for (Shard shard : shards) {
        InputSplit split = new InputSplit(shard.start(), shard.end());
        splits.add(split);
    }
    return splits;
}