Examples with FileSource com.baidu.hugegraph.loader.source.file.FileSource used on opensource projects

Example 1 with FileSource

use of com.baidu.hugegraph.loader.source.file.FileSource in project incubator-hugegraph-toolchain by apache.

the class GraphService method fillProperties.

private void fillProperties(int connId, SchemaLabelEntity schema, GraphElement element, Map<String, Object> properties) {
    HugeClient client = this.client(connId);
    for (Map.Entry<String, Object> entry : properties.entrySet()) {
        String key = entry.getKey();
        Object rawValue = entry.getValue();
        // Skip nullable property
        if (schema.getNullableProps().contains(key)) {
            if (rawValue instanceof String && StringUtils.isEmpty((String) rawValue)) {
                continue;
            }
        }
        PropertyKeyEntity pkEntity = this.pkService.get(key, connId);
        PropertyKey propertyKey = PropertyKeyService.convert(pkEntity, client);
        assert propertyKey != null;
        Object value;
        try {
            // DataTypeUtil.convert in loader need param InputSource
            FileSource source = new FileSource();
            ListFormat listFormat = new ListFormat("", "", ",");
            source.listFormat(listFormat);
            value = DataTypeUtil.convert(rawValue, propertyKey, source);
        } catch (IllegalArgumentException e) {
            throw new ExternalException("graph.property.convert.failed", e, key, rawValue);
        }
        element.property(key, value);
    }
}

Example 2 with FileSource

use of com.baidu.hugegraph.loader.source.file.FileSource in project incubator-hugegraph-toolchain by apache.

the class LoadTaskService method buildFileSource.

private FileSource buildFileSource(FileMapping fileMapping) {
    // Set input source
    FileSource source = new FileSource();
    source.path(fileMapping.getPath());
    FileSetting setting = fileMapping.getFileSetting();
    Ex.check(setting.getColumnNames() != null, "Must do file setting firstly");
    source.header(setting.getColumnNames().toArray(new String[] {}));
    // NOTE: format and delimiter must be CSV and "," temporarily
    source.format(FileFormat.valueOf(setting.getFormat()));
    source.delimiter(setting.getDelimiter());
    source.charset(setting.getCharset());
    source.dateFormat(setting.getDateFormat());
    source.timeZone(setting.getTimeZone());
    source.skippedLine().regex(setting.getSkippedLine());
    // Set list format
    source.listFormat(new com.baidu.hugegraph.loader.source.file.ListFormat());
    ListFormat listFormat = setting.getListFormat();
    source.listFormat().startSymbol(listFormat.getStartSymbol());
    source.listFormat().endSymbol(listFormat.getEndSymbol());
    source.listFormat().elemDelimiter(listFormat.getElemDelimiter());
    return source;
}

Example 3 with FileSource

use of com.baidu.hugegraph.loader.source.file.FileSource in project incubator-hugegraph-toolchain by apache.

the class LoadMapping method structsForFailure.

public List<InputStruct> structsForFailure(LoadOptions options) {
    List<InputStruct> targetStructs = new ArrayList<>();
    String dir = LoadUtil.getStructDirPrefix(options);
    String path = Paths.get(dir, Constants.FAILURE_DATA).toString();
    File pathDir = FileUtils.getFile(path);
    // It means no failure data if the path directory does not exist
    if (!pathDir.exists()) {
        return targetStructs;
    }
    Map<String, FailureFile> failureFiles = this.groupFailureFiles(pathDir);
    for (String inputId : failureFiles.keySet()) {
        InputStruct struct = this.struct(inputId);
        String charset = struct.input().charset();
        FailureFile failureFile = failureFiles.get(inputId);
        FileSource source = struct.input().asFileSource();
        if (failureFile.headerFile != null) {
            // It means that header file existed
            String json;
            try {
                json = FileUtils.readFileToString(failureFile.headerFile, charset);
            } catch (IOException e) {
                throw new LoadException("Failed to read header file %s", failureFile.headerFile);
            }
            List<String> header = JsonUtil.convertList(json, String.class);
            source.header(header.toArray(new String[] {}));
        }
        // Set failure data path
        source.path(failureFile.dataFile.getAbsolutePath());
        source.skippedLine().regex(Constants.SKIPPED_LINE_REGEX);
        struct.input(source);
        // Add to target structs
        targetStructs.add(struct);
    }
    return targetStructs;
}

Example 4 with FileSource

use of com.baidu.hugegraph.loader.source.file.FileSource in project incubator-hugegraph-toolchain by apache.

the class MappingUtil method parseV1.

private static LoadMapping parseV1(String json) {
    GraphStructV1 graphStruct = JsonUtil.fromJson(json, GraphStructV1.class);
    Map<FileSourceKey, InputStruct> fileSourceInputStructs = InsertionOrderUtil.newMap();
    List<InputStruct> jdbcSourceInputStructs = new ArrayList<>();
    for (ElementStructV1 originStruct : graphStruct.structs()) {
        InputSource inputSource = originStruct.input();
        ElementMapping targetStruct = convertV1ToV2(originStruct);
        SourceType type = inputSource.type();
        if (type == SourceType.FILE || type == SourceType.HDFS) {
            FileSource source = (FileSource) inputSource;
            FileSourceKey key = new FileSourceKey(type, source.path());
            fileSourceInputStructs.compute(key, (k, inputStruct) -> {
                if (inputStruct == null) {
                    inputStruct = new InputStruct(null, null);
                    inputStruct.input(source);
                }
                inputStruct.add(targetStruct);
                return inputStruct;
            });
        } else {
            assert type == SourceType.JDBC;
            InputStruct inputStruct = new InputStruct(null, null);
            inputStruct.input(inputSource);
            inputStruct.add(targetStruct);
            jdbcSourceInputStructs.add(inputStruct);
        }
    }
    // Generate id for every input mapping
    List<InputStruct> inputStructs = new ArrayList<>();
    int id = 0;
    for (InputStruct inputStruct : fileSourceInputStructs.values()) {
        inputStruct.id(String.valueOf(++id));
        inputStructs.add(inputStruct);
    }
    for (InputStruct inputStruct : jdbcSourceInputStructs) {
        inputStruct.id(String.valueOf(++id));
        inputStructs.add(inputStruct);
    }
    return new LoadMapping(inputStructs);
}

Example 5 with FileSource

use of com.baidu.hugegraph.loader.source.file.FileSource in project incubator-hugegraph-toolchain by apache.

the class JDBCSource method asFileSource.

@Override
public FileSource asFileSource() {
    FileSource source = new FileSource();
    source.header(this.header());
    source.charset(this.charset());
    source.listFormat(this.listFormat());
    return source;
}