Examples with Line com.baidu.hugegraph.loader.reader.line.Line used on opensource projects

Example 6 with Line

use of com.baidu.hugegraph.loader.reader.line.Line in project incubator-hugegraph-toolchain by apache.

the class FileReader method next.

@Override
public Line next() {
    if (!this.hasNext()) {
        throw new NoSuchElementException("Reached the end of file");
    }
    Line line = this.nextLine;
    this.nextLine = null;
    return line;
}

Example 7 with Line

use of com.baidu.hugegraph.loader.reader.line.Line in project incubator-hugegraph-toolchain by apache.

the class JsonLineParser method parse.

@Override
public Line parse(String[] header, String rawLine) {
    Map<String, Object> keyValues;
    try {
        keyValues = JsonUtil.convertMap(rawLine, String.class, Object.class);
        String[] names = names(keyValues);
        Object[] values = values(keyValues, names);
        return new Line(rawLine, names, values);
    } catch (SerializeException e) {
        throw new ReadException(rawLine, "Deserialize line '%s' error", e, rawLine);
    }
}

Example 8 with Line

use of com.baidu.hugegraph.loader.reader.line.Line in project incubator-hugegraph-toolchain by apache.

the class TextLineParser method parse.

@Override
public Line parse(String[] header, String rawLine) throws ReadException {
    String[] columns = this.split(rawLine);
    if (columns.length > header.length) {
        // Ignore extra empty string at the tail of line
        int extra = columns.length - header.length;
        if (!this.tailColumnEmpty(columns, extra)) {
            throw new ReadException(rawLine, "The column length '%s' doesn't match with " + "header length '%s' on: %s", columns.length, header.length, rawLine);
        }
        String[] subColumns = new String[header.length];
        System.arraycopy(columns, 0, subColumns, 0, header.length);
        return new Line(rawLine, header, subColumns);
    } else if (columns.length < header.length) {
        // Fill with an empty string
        String[] supColumns = new String[header.length];
        System.arraycopy(columns, 0, supColumns, 0, columns.length);
        Arrays.fill(supColumns, columns.length, supColumns.length, Constants.EMPTY_STR);
        return new Line(rawLine, header, supColumns);
    }
    return new Line(rawLine, header, columns);
}

Example 9 with Line

use of com.baidu.hugegraph.loader.reader.line.Line in project incubator-hugegraph-toolchain by apache.

the class ParquetFileLineFetcher method fetch.

@Override
public Line fetch() {
    boolean needFetchNext = this.pages == null || this.currRowOffset >= this.pagesRowCount;
    // Read next row group
    if (needFetchNext && !this.fetchNextPage()) {
        return null;
    }
    int fieldSize = this.schema.getFields().size();
    Object[] values = new Object[fieldSize];
    SimpleGroup group = (SimpleGroup) this.recordReader.read();
    for (int fieldIndex = 0; fieldIndex < fieldSize; fieldIndex++) {
        values[fieldIndex] = ParquetUtil.convertObject(group, fieldIndex);
    }
    String rawLine = StringUtils.join(values, Constants.COMMA_STR);
    this.currRowOffset++;
    this.increaseOffset();
    /*
         * NOTE: parquet file actually corresponds to a table structure,
         * doesn't need to skip line or match header
         */
    return new Line(rawLine, this.source().header(), values);
}