Examples with DelimitedTextParser smile.data.parser.DelimitedTextParser used on opensource projects

Example 31 with DelimitedTextParser

use of smile.data.parser.DelimitedTextParser in project smile by haifengl.

the class NumericAttributeFeatureTest method testSTANDARDIZATION.

/**
     * Test of f method, of class NumericAttributeFeature.
     */
@Test
public void testSTANDARDIZATION() {
    System.out.println("STANDARDIZATION");
    DelimitedTextParser parser = new DelimitedTextParser();
    parser.setResponseIndex(new NominalAttribute("class"), 0);
    try {
        AttributeDataset data = parser.parse("USPS Train", smile.data.parser.IOUtils.getTestDataFile("usps/zip.train"));
        double[][] x = data.toArray(new double[data.size()][]);
        double[] mean = Math.colMean(x);
        double[] sd = Math.colSd(x);
        NumericAttributeFeature naf = new NumericAttributeFeature(data.attributes(), NumericAttributeFeature.Scaling.STANDARDIZATION, x);
        Attribute[] attributes = naf.attributes();
        assertEquals(256, attributes.length);
        for (int i = 0; i < x.length; i++) {
            double[] y = new double[attributes.length];
            for (int j = 0; j < y.length; j++) {
                y[j] = naf.f(x[i], j);
                assertEquals((x[i][j] - mean[j]) / sd[j], y[j], 1E-7);
            }
        }
    } catch (Exception ex) {
        System.err.println(ex);
    }
}

Example 32 with DelimitedTextParser

use of smile.data.parser.DelimitedTextParser in project smile by haifengl.

the class NumericAttributeFeatureTest method testNONE.

/**
     * Test of f method, of class NumericAttributeFeature.
     */
@Test
public void testNONE() {
    System.out.println("NONE");
    DelimitedTextParser parser = new DelimitedTextParser();
    parser.setResponseIndex(new NominalAttribute("class"), 0);
    try {
        AttributeDataset data = parser.parse("USPS Train", smile.data.parser.IOUtils.getTestDataFile("usps/zip.train"));
        double[][] x = data.toArray(new double[data.size()][]);
        NumericAttributeFeature naf = new NumericAttributeFeature(data.attributes(), NumericAttributeFeature.Scaling.NONE);
        Attribute[] attributes = naf.attributes();
        assertEquals(256, attributes.length);
        for (int i = 0; i < x.length; i++) {
            double[] y = new double[attributes.length];
            for (int j = 0; j < y.length; j++) {
                y[j] = naf.f(x[i], j);
                assertEquals(x[i][j], y[j], 1E-7);
            }
        }
    } catch (Exception ex) {
        System.err.println(ex);
    }
}

Example 33 with DelimitedTextParser

use of smile.data.parser.DelimitedTextParser in project smile by haifengl.

the class NumericAttributeFeatureTest method testAttributes.

/**
     * Test of attributes method, of class NumericAttributeFeature.
     */
@SuppressWarnings("unused")
@Test
public void testAttributes() {
    System.out.println("attributes");
    DelimitedTextParser parser = new DelimitedTextParser();
    parser.setResponseIndex(new NominalAttribute("class"), 0);
    try {
        AttributeDataset data = parser.parse("USPS Train", smile.data.parser.IOUtils.getTestDataFile("usps/zip.train"));
        double[][] x = data.toArray(new double[data.size()][]);
        NumericAttributeFeature naf = new NumericAttributeFeature(data.attributes(), NumericAttributeFeature.Scaling.LOGARITHM);
        Attribute[] attributes = naf.attributes();
        assertEquals(256, attributes.length);
        for (int i = 0; i < attributes.length; i++) {
            System.out.println(attributes[i]);
            assertEquals(Attribute.Type.NUMERIC, attributes[i].getType());
        }
    } catch (Exception ex) {
        System.err.println(ex);
    }
}

Example 34 with DelimitedTextParser

use of smile.data.parser.DelimitedTextParser in project smile by haifengl.

the class KDTreeSpeedTest method testUSPS.

/**
     * Test of nearest method, of class KDTree.
     */
@Test
public void testUSPS() {
    System.out.println("USPS");
    double[][] x = null;
    double[][] testx = null;
    long start = System.currentTimeMillis();
    DelimitedTextParser parser = new DelimitedTextParser();
    parser.setResponseIndex(new NominalAttribute("class"), 0);
    try {
        AttributeDataset train = parser.parse("USPS Train", smile.data.parser.IOUtils.getTestDataFile("usps/zip.train"));
        AttributeDataset test = parser.parse("USPS Test", smile.data.parser.IOUtils.getTestDataFile("usps/zip.test"));
        x = train.toArray(new double[train.size()][]);
        testx = test.toArray(new double[test.size()][]);
    } catch (Exception ex) {
        System.err.println(ex);
    }
    double time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Loading USPS: %.2fs%n", time);
    start = System.currentTimeMillis();
    KDTree<double[]> kdtree = new KDTree<>(x, x);
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Building KD-tree: %.2fs%n", time);
    start = System.currentTimeMillis();
    for (int i = 0; i < testx.length; i++) {
        kdtree.nearest(testx[i]);
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("NN: %.2fs%n", time);
    start = System.currentTimeMillis();
    for (int i = 0; i < testx.length; i++) {
        kdtree.knn(testx[i], 10);
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("10-NN: %.2fs%n", time);
    start = System.currentTimeMillis();
    List<Neighbor<double[], double[]>> n = new ArrayList<>();
    for (int i = 0; i < testx.length; i++) {
        kdtree.range(testx[i], 8.0, n);
        n.clear();
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Range: %.2fs%n", time);
}

Example 35 with DelimitedTextParser

use of smile.data.parser.DelimitedTextParser in project smile by haifengl.

the class LSHSpeedTest method testUSPS.

/**
     * Test of nearest method, of class KDTree.
     */
@Test
public void testUSPS() {
    System.out.println("USPS");
    double[][] x = null;
    double[][] testx = null;
    long start = System.currentTimeMillis();
    DelimitedTextParser parser = new DelimitedTextParser();
    parser.setResponseIndex(new NominalAttribute("class"), 0);
    try {
        AttributeDataset train = parser.parse("USPS Train", smile.data.parser.IOUtils.getTestDataFile("usps/zip.train"));
        AttributeDataset test = parser.parse("USPS Test", smile.data.parser.IOUtils.getTestDataFile("usps/zip.test"));
        x = train.toArray(new double[train.size()][]);
        testx = test.toArray(new double[test.size()][]);
    } catch (Exception ex) {
        System.err.println(ex);
    }
    double time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Loading USPS: %.2fs%n", time);
    start = System.currentTimeMillis();
    LSH<double[]> lsh = new LSH<>(x, x);
    /**
        LSH<double[]> lsh = new LSH<double[]>(256, 100, 3, 4.0);
        for (double[] xi : x) {
            lsh.put(xi, xi);
        }
         */
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Building LSH: %.2fs%n", time);
    start = System.currentTimeMillis();
    for (int i = 0; i < testx.length; i++) {
        lsh.nearest(testx[i]);
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("NN: %.2fs%n", time);
    start = System.currentTimeMillis();
    for (int i = 0; i < testx.length; i++) {
        lsh.knn(testx[i], 10);
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("10-NN: %.2fs%n", time);
    start = System.currentTimeMillis();
    List<Neighbor<double[], double[]>> n = new ArrayList<>();
    for (int i = 0; i < testx.length; i++) {
        lsh.range(testx[i], 8.0, n);
        n.clear();
    }
    time = (System.currentTimeMillis() - start) / 1000.0;
    System.out.format("Range: %.2fs%n", time);
}