Examples with BufferedString water.parser.BufferedString used on opensource projects

Example 16 with BufferedString

use of water.parser.BufferedString in project h2o-3 by h2oai.

the class CStrChunkTest method test_writer.

@Test
public void test_writer() {
    Frame frame = null;
    try {
        frame = parse_test_file("smalldata/junit/iris.csv");
        //Create a label vector
        byte[] typeArr = { Vec.T_STR };
        Vec labels = frame.lastVec().makeCons(1, 0, null, typeArr)[0];
        Vec.Writer writer = labels.open();
        int rowCnt = (int) frame.lastVec().length();
        for (// adding labels in reverse order
        int r = 0; // adding labels in reverse order
        r < rowCnt; // adding labels in reverse order
        r++) writer.set(rowCnt - r - 1, "Foo" + (r + 1));
        writer.close();
        //Append label vector and spot check
        frame.add("Labels", labels);
        Assert.assertTrue("Failed to create a new String based label column", frame.lastVec().atStr(new BufferedString(), 42).compareTo(new BufferedString("Foo108")) == 0);
    } finally {
        if (frame != null)
            frame.delete();
    }
}

Example 17 with BufferedString

use of water.parser.BufferedString in project h2o-3 by h2oai.

the class FrameTestUtil method assertValues.

public static void assertValues(Vec v, String[] expValues) {
    Assert.assertEquals("Number of rows", expValues.length, v.length());
    BufferedString tmpStr = new BufferedString();
    for (int i = 0; i < v.length(); i++) {
        if (v.isNA(i))
            Assert.assertEquals("NAs should match", null, expValues[i]);
        else
            Assert.assertEquals("Values should match", expValues[i], v.atStr(tmpStr, i).toString());
    }
}

Example 18 with BufferedString

use of water.parser.BufferedString in project h2o-3 by h2oai.

the class TestFrameBuilderTest method testSetChunks.

@Test
public void testSetChunks() {
    final Frame fr = new TestFrameBuilder().withName("frameName").withColNames("ColA", "ColB").withVecTypes(Vec.T_NUM, Vec.T_STR).withDataForCol(0, ard(Double.NaN, 1, 2, 3, 4, 5.6, 7)).withDataForCol(1, ar("A", "B", "C", null, "F", "I", "J")).withChunkLayout(2, 2, 2, 1).build();
    assertEquals(fr.anyVec().nChunks(), 4);
    assertArrayEquals(fr.anyVec().espc(), new long[] { 0, 2, 4, 6, 7 });
    // check data in the frame
    assertEquals(fr.vec(0).at(0), Double.NaN, DELTA);
    assertEquals(fr.vec(0).at(5), 5.6, DELTA);
    assertEquals(fr.vec(0).at(6), 7, DELTA);
    BufferedString strBuf = new BufferedString();
    assertEquals(fr.vec(1).atStr(strBuf, 0).toString(), "A");
    assertNull(fr.vec(1).atStr(strBuf, 3));
    assertEquals(fr.vec(1).atStr(strBuf, 6).toString(), "J");
    fr.remove();
}

Example 19 with BufferedString

use of water.parser.BufferedString in project h2o-3 by h2oai.

the class AvroParser method write2frame.

/**
   * The main method transforming Avro record into a row in H2O frame.
   *
   * @param gr  Avro generic record
   * @param columnNames Column names prepared by parser setup
   * @param inSchema  Flattenized Avro schema which corresponds to passed column names
   * @param columnTypes  Target H2O types
   * @param dout  Parser writer
   */
private static void write2frame(GenericRecord gr, String[] columnNames, Schema.Field[] inSchema, byte[] columnTypes, ParseWriter dout) {
    assert inSchema.length == columnTypes.length : "AVRO field flatenized schema has to match to parser setup";
    BufferedString bs = new BufferedString();
    for (int cIdx = 0; cIdx < columnNames.length; cIdx++) {
        int inputFieldIdx = inSchema[cIdx].pos();
        Schema.Type inputType = toPrimitiveType(inSchema[cIdx].schema());
        // FIXME: support target conversions
        byte targetType = columnTypes[cIdx];
        Object value = gr.get(inputFieldIdx);
        if (value == null) {
            dout.addInvalidCol(cIdx);
        } else {
            switch(inputType) {
                case BOOLEAN:
                    dout.addNumCol(cIdx, ((Boolean) value) ? 1 : 0);
                    break;
                case INT:
                    dout.addNumCol(cIdx, ((Integer) value), 0);
                    break;
                case LONG:
                    dout.addNumCol(cIdx, ((Long) value), 0);
                    break;
                case FLOAT:
                    dout.addNumCol(cIdx, (Float) value);
                    break;
                case DOUBLE:
                    dout.addNumCol(cIdx, (Double) value);
                    break;
                case ENUM:
                    // Note: this code expects ordering of categoricals provided by Avro remain same
                    // as in H2O!!!
                    GenericData.EnumSymbol es = (GenericData.EnumSymbol) value;
                    dout.addNumCol(cIdx, es.getSchema().getEnumOrdinal(es.toString()));
                    break;
                case BYTES:
                    dout.addStrCol(cIdx, bs.set(((ByteBuffer) value).array()));
                    break;
                case STRING:
                    dout.addStrCol(cIdx, bs.set(((Utf8) value).getBytes()));
                    break;
                case NULL:
                    dout.addInvalidCol(cIdx);
                    break;
            }
        }
    }
}

Example 20 with BufferedString

use of water.parser.BufferedString in project h2o-3 by h2oai.

the class DeepWaterMXNetIntegrationTest method inceptionPredictionMX.

// This test has nothing to do with H2O - Pure integration test of deepwater/backends/mxnet
@Test
public void inceptionPredictionMX() throws IOException {
    for (boolean gpu : new boolean[] { true, false }) {
        // Set model parameters
        int w = 224, h = 224, channels = 3, nclasses = 1000;
        ImageDataSet id = new ImageDataSet(w, h, channels, nclasses);
        RuntimeOptions opts = new RuntimeOptions();
        opts.setSeed(1234);
        opts.setUseGPU(gpu);
        BackendParams bparm = new BackendParams();
        bparm.set("mini_batch_size", 1);
        // Load the model
        String path = "deepwater/backends/mxnet/models/Inception/";
        BackendModel _model = backend.buildNet(id, opts, bparm, nclasses, StringUtils.expandPath(extractFile(path, "Inception_BN-symbol.json")));
        backend.loadParam(_model, StringUtils.expandPath(extractFile(path, "Inception_BN-0039.params")));
        water.fvec.Frame labels = parse_test_file(extractFile(path, "synset.txt"));
        float[] mean = backend.loadMeanImage(_model, extractFile(path, "mean_224.nd"));
        // Turn the image into a vector of the correct size
        File imgFile = FileUtils.getFile("smalldata/deepwater/imagenet/test2.jpg");
        BufferedImage img = ImageIO.read(imgFile);
        BufferedImage scaledImg = new BufferedImage(w, h, img.getType());
        Graphics2D g2d = scaledImg.createGraphics();
        g2d.drawImage(img, 0, 0, w, h, null);
        g2d.dispose();
        float[] pixels = new float[w * h * channels];
        int r_idx = 0;
        int g_idx = r_idx + w * h;
        int b_idx = g_idx + w * h;
        for (int i = 0; i < h; i++) {
            for (int j = 0; j < w; j++) {
                Color mycolor = new Color(scaledImg.getRGB(j, i));
                int red = mycolor.getRed();
                int green = mycolor.getGreen();
                int blue = mycolor.getBlue();
                pixels[r_idx] = red - mean[r_idx];
                r_idx++;
                pixels[g_idx] = green - mean[g_idx];
                g_idx++;
                pixels[b_idx] = blue - mean[b_idx];
                b_idx++;
            }
        }
        float[] preds = backend.predict(_model, pixels);
        int K = 5;
        int[] topK = new int[K];
        for (int i = 0; i < preds.length; i++) {
            for (int j = 0; j < K; j++) {
                if (preds[i] > preds[topK[j]]) {
                    topK[j] = i;
                    break;
                }
            }
        }
        // Display the top 5 predictions
        StringBuilder sb = new StringBuilder();
        sb.append("\nTop " + K + " predictions:\n");
        BufferedString str = new BufferedString();
        for (int j = 0; j < K; j++) {
            String label = labels.anyVec().atStr(str, topK[j]).toString();
            sb.append(" Score: " + String.format("%.4f", preds[topK[j]]) + "\t" + label + "\n");
        }
        System.out.println("\n\n" + sb.toString() + "\n\n");
        Assert.assertTrue("Illegal predictions!", sb.toString().substring(40, 60).contains("Pembroke"));
        labels.remove();
    }
}