Examples with ByteVec water.fvec.ByteVec used on opensource projects

Example 1 with ByteVec

use of water.fvec.ByteVec in project h2o-3 by h2oai.

the class FrameV3 method fillFromImpl.

public FrameV3 fillFromImpl(Frame f, long row_offset, int row_count, int column_offset, int column_count) {
    // 100 rows by default
    if (row_count == 0)
        row_count = 100;
    // full width by default
    if (column_count == 0)
        column_count = f.numCols() - column_offset;
    row_count = (int) Math.min(row_count, row_offset + f.numRows());
    column_count = Math.min(column_count, column_offset + f.numCols());
    this.frame_id = new FrameKeyV3(f._key);
    this.checksum = f.checksum();
    this.byte_size = f.byteSize();
    this.row_offset = row_offset;
    this.rows = f.numRows();
    this.num_columns = f.numCols();
    this.row_count = row_count;
    this.total_column_count = f.numCols();
    this.column_offset = column_offset;
    this.column_count = column_count;
    this.columns = new ColV3[column_count];
    Vec[] vecs = f.vecs();
    Futures fs = new Futures();
    // NOTE: SKIP deleted Vecs!  The columns entry will be null for deleted Vecs.
    for (int i = 0; i < column_count; i++) if (null == DKV.get(vecs[column_offset + i]._key))
        Log.warn("For Frame: " + f._key + ", Vec number: " + (column_offset + i) + " (" + f.name((column_offset + i)) + ") is missing; not returning it.");
    else
        vecs[column_offset + i].startRollupStats(fs);
    for (int i = 0; i < column_count; i++) if (null == DKV.get(vecs[column_offset + i]._key))
        Log.warn("For Frame: " + f._key + ", Vec number: " + (column_offset + i) + " (" + f.name((column_offset + i)) + ") is missing; not returning it.");
    else
        columns[i] = new ColV3(f._names[column_offset + i], vecs[column_offset + i], this.row_offset, this.row_count);
    fs.blockForPending();
    this.is_text = f.numCols() == 1 && vecs[0] instanceof ByteVec;
    this.default_percentiles = Vec.PERCENTILES;
    ChunkSummary cs = FrameUtils.chunkSummary(f);
    this.chunk_summary = new TwoDimTableV3(cs.toTwoDimTableChunkTypes());
    this.distribution_summary = new TwoDimTableV3(cs.toTwoDimTableDistribution());
    this._fr = f;
    return this;
}

Example 2 with ByteVec

use of water.fvec.ByteVec in project h2o-3 by h2oai.

the class AvroParserProvider method createParserSetup.

@Override
public ParseSetup createParserSetup(Key[] inputs, ParseSetup requiredSetup) {
    // Also expect that files are not compressed
    assert inputs != null && inputs.length > 0 : "Inputs cannot be empty!";
    Key firstInput = inputs[0];
    Iced ice = DKV.getGet(firstInput);
    if (ice == null)
        throw new H2OIllegalArgumentException("Missing data", "Did not find any data under key " + firstInput);
    ByteVec bv = (ByteVec) (ice instanceof ByteVec ? ice : ((Frame) ice).vecs()[0]);
    byte[] bits = bv.getFirstBytes();
    try {
        AvroParser.AvroInfo avroInfo = AvroParser.extractAvroInfo(bits, requiredSetup);
        return new AvroParser.AvroParseSetup(requiredSetup, avroInfo.header, avroInfo.firstBlockSize, avroInfo.domains);
    } catch (Throwable e) {
        throw new H2OIllegalArgumentException("Wrong data", "Cannot find Avro header in input file: " + firstInput, e);
    }
}

Example 3 with ByteVec

use of water.fvec.ByteVec in project h2o-3 by h2oai.

the class ZipUtil method isZipDirectory.

/**
   * This method check if the input argument is a zip directory containing files.
   *
   * @param key
   * @return true if bv is a zip directory containing files, false otherwise.
   */
static boolean isZipDirectory(Key key) {
    Iced ice = DKV.getGet(key);
    if (ice == null)
        throw new H2OIllegalArgumentException("Missing data", "Did not find any data under " + "key " + key);
    ByteVec bv = (ByteVec) (ice instanceof ByteVec ? ice : ((Frame) ice).vecs()[0]);
    return isZipDirectory(bv);
}