Examples with Row org.apache.spark.sql.Row used on opensource projects

Example 71 with Row

use of org.apache.spark.sql.Row in project incubator-systemml by apache.

the class DataFrameMatrixConversionTest method testDataFrameConversionUltraSparse.

private void testDataFrameConversionUltraSparse(boolean vector, boolean unknownDims) {
    boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform;
    try {
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
        // generate input data and setup metadata
        double[][] A = getRandomMatrix(rows1, 1, -10, 10, 0.7, 2373);
        MatrixBlock mbA0 = DataConverter.convertToMatrixBlock(A);
        MatrixBlock mbA = LibMatrixReorg.diag(mbA0, new MatrixBlock(rows1, rows1, true));
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, rows1, blksz, blksz, mbA.getNonZeros());
        MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
        // get binary block input rdd
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = SparkExecutionContext.toMatrixJavaPairRDD(sc, mbA, blksz, blksz);
        // matrix - dataframe - matrix conversion
        Dataset<Row> df = RDDConverterUtils.binaryBlockToDataFrame(spark, in, mc1, vector);
        JavaPairRDD<MatrixIndexes, MatrixBlock> out = RDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, true, vector);
        // get output matrix block
        MatrixBlock mbB0 = SparkExecutionContext.toMatrixBlock(out, rows1, rows1, blksz, blksz, -1);
        MatrixBlock mbB = LibMatrixReorg.diag(mbB0, new MatrixBlock(rows1, 1, false));
        // compare matrix blocks
        double[][] B = DataConverter.convertToDoubleMatrix(mbB);
        TestUtils.compareMatrices(A, B, rows1, 1, eps);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
        DMLScript.rtplatform = oldPlatform;
    }
}

Example 72 with Row

use of org.apache.spark.sql.Row in project incubator-systemml by apache.

the class DataFrameRowFrameConversionTest method testDataFrameConversion.

private void testDataFrameConversion(ValueType vt, boolean singleColBlock, boolean dense, boolean unknownDims) {
    boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform;
    try {
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
        // generate input data and setup metadata
        int cols = singleColBlock ? cols1 : cols2;
        double sparsity = dense ? sparsity1 : sparsity2;
        double[][] A = getRandomMatrix(rows1, cols, -10, 10, sparsity, 2373);
        A = (vt == ValueType.INT) ? TestUtils.round(A) : A;
        MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
        FrameBlock fbA = DataConverter.convertToFrameBlock(mbA, vt);
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
        MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
        ValueType[] schema = UtilFunctions.nCopies(cols, vt);
        // get binary block input rdd
        JavaPairRDD<Long, FrameBlock> in = SparkExecutionContext.toFrameJavaPairRDD(sc, fbA);
        // frame - dataframe - frame conversion
        Dataset<Row> df = FrameRDDConverterUtils.binaryBlockToDataFrame(spark, in, mc1, schema);
        JavaPairRDD<Long, FrameBlock> out = FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, true);
        // get output frame block
        FrameBlock fbB = SparkExecutionContext.toFrameBlock(out, schema, rows1, cols);
        // compare frame blocks
        MatrixBlock mbB = DataConverter.convertToMatrixBlock(fbB);
        double[][] B = DataConverter.convertToDoubleMatrix(mbB);
        TestUtils.compareMatrices(A, B, rows1, cols, eps);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
        DMLScript.rtplatform = oldPlatform;
    }
}

Example 73 with Row

use of org.apache.spark.sql.Row in project incubator-systemml by apache.

the class DataFrameVectorFrameConversionTest method testDataFrameConversion.

private void testDataFrameConversion(ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) {
    boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform;
    try {
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
        // generate input data and setup metadata
        int cols = schema.length + colsVector - 1;
        double sparsity = dense ? sparsity1 : sparsity2;
        double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373));
        MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
        MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
        // create input data frame
        Dataset<Row> df = createDataFrame(spark, mbA, containsID, schema);
        // dataframe - frame conversion
        JavaPairRDD<Long, FrameBlock> out = FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, containsID);
        // get output frame block
        FrameBlock fbB = SparkExecutionContext.toFrameBlock(out, UtilFunctions.nCopies(cols, ValueType.DOUBLE), rows1, cols);
        // compare frame blocks
        MatrixBlock mbB = DataConverter.convertToMatrixBlock(fbB);
        double[][] B = DataConverter.convertToDoubleMatrix(mbB);
        TestUtils.compareMatrices(A, B, rows1, cols, eps);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
        DMLScript.rtplatform = oldPlatform;
    }
}

Example 74 with Row

use of org.apache.spark.sql.Row in project incubator-systemml by apache.

the class DataFrameVectorScriptTest method testDataFrameScriptInput.

private void testDataFrameScriptInput(ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) {
    // TODO fix inconsistency ml context vs jmlc register Xf
    try {
        // generate input data and setup metadata
        int cols = schema.length + colsVector - 1;
        double sparsity = dense ? sparsity1 : sparsity2;
        double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373));
        MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
        MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
        // create input data frame
        Dataset<Row> df = createDataFrame(spark, mbA, containsID, schema);
        // Create full frame metadata, and empty frame metadata
        FrameMetadata meta = new FrameMetadata(containsID ? FrameFormat.DF_WITH_INDEX : FrameFormat.DF, mc2.getRows(), mc2.getCols());
        FrameMetadata metaEmpty = new FrameMetadata();
        // run scripts and obtain result
        Script script1 = dml("Xm = as.matrix(Xf);").in("Xf", df, meta).out("Xm");
        Script script2 = dml("Xm = as.matrix(Xf);").in("Xf", df, metaEmpty).out(// empty metadata
        "Xm");
        Matrix Xm1 = ml.execute(script1).getMatrix("Xm");
        Matrix Xm2 = ml.execute(script2).getMatrix("Xm");
        double[][] B1 = Xm1.to2DDoubleArray();
        double[][] B2 = Xm2.to2DDoubleArray();
        TestUtils.compareMatrices(A, B1, rows1, cols, eps);
        TestUtils.compareMatrices(A, B2, rows1, cols, eps);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 75 with Row

use of org.apache.spark.sql.Row in project incubator-systemml by apache.

the class FrameTest method testFrameGeneral.

private void testFrameGeneral(InputInfo iinfo, OutputInfo oinfo, boolean bFromDataFrame, boolean bToDataFrame) throws IOException, DMLException, ParseException {
    boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    RUNTIME_PLATFORM oldRT = DMLScript.rtplatform;
    DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
    int rowstart = 234, rowend = 1478, colstart = 125, colend = 568;
    int bRows = rowend - rowstart + 1, bCols = colend - colstart + 1;
    int rowstartC = 124, rowendC = 1178, colstartC = 143, colendC = 368;
    int cRows = rowendC - rowstartC + 1, cCols = colendC - colstartC + 1;
    HashMap<String, ValueType[]> outputSchema = new HashMap<String, ValueType[]>();
    HashMap<String, MatrixCharacteristics> outputMC = new HashMap<String, MatrixCharacteristics>();
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    List<String> proArgs = new ArrayList<String>();
    proArgs.add(input("A"));
    proArgs.add(Integer.toString(rows));
    proArgs.add(Integer.toString(cols));
    proArgs.add(input("B"));
    proArgs.add(Integer.toString(bRows));
    proArgs.add(Integer.toString(bCols));
    proArgs.add(Integer.toString(rowstart));
    proArgs.add(Integer.toString(rowend));
    proArgs.add(Integer.toString(colstart));
    proArgs.add(Integer.toString(colend));
    proArgs.add(output("A"));
    proArgs.add(Integer.toString(rowstartC));
    proArgs.add(Integer.toString(rowendC));
    proArgs.add(Integer.toString(colstartC));
    proArgs.add(Integer.toString(colendC));
    proArgs.add(output("C"));
    fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml";
    ValueType[] schema = schemaMixedLarge;
    // initialize the frame data.
    List<ValueType> lschema = Arrays.asList(schema);
    fullRScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir() + " " + rowstartC + " " + rowendC + " " + colstartC + " " + colendC;
    double sparsity = sparsity1;
    double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111);
    writeInputFrameWithMTD("A", A, true, schema, oinfo);
    sparsity = sparsity2;
    double[][] B = getRandomMatrix((int) (bRows), (int) (bCols), min, max, sparsity, 2345);
    ValueType[] schemaB = new ValueType[bCols];
    for (int i = 0; i < bCols; ++i) schemaB[i] = schema[colstart - 1 + i];
    List<ValueType> lschemaB = Arrays.asList(schemaB);
    writeInputFrameWithMTD("B", B, true, schemaB, oinfo);
    ValueType[] schemaC = new ValueType[colendC - colstartC + 1];
    for (int i = 0; i < cCols; ++i) schemaC[i] = schema[colstartC - 1 + i];
    Dataset<Row> dfA = null, dfB = null;
    if (bFromDataFrame) {
        // Create DataFrame for input A
        StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schema, false);
        JavaRDD<Row> rowRDDA = FrameRDDConverterUtils.csvToRowRDD(sc, input("A"), DataExpression.DEFAULT_DELIM_DELIMITER, schema);
        dfA = spark.createDataFrame(rowRDDA, dfSchemaA);
        // Create DataFrame for input B
        StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaB, false);
        JavaRDD<Row> rowRDDB = FrameRDDConverterUtils.csvToRowRDD(sc, input("B"), DataExpression.DEFAULT_DELIM_DELIMITER, schemaB);
        dfB = spark.createDataFrame(rowRDDB, dfSchemaB);
    }
    try {
        Script script = ScriptFactory.dmlFromFile(fullDMLScriptName);
        String format = "csv";
        if (oinfo == OutputInfo.TextCellOutputInfo)
            format = "text";
        if (bFromDataFrame) {
            script.in("A", dfA);
        } else {
            JavaRDD<String> aIn = sc.textFile(input("A"));
            FrameSchema fs = new FrameSchema(lschema);
            FrameFormat ff = (format.equals("text")) ? FrameFormat.IJV : FrameFormat.CSV;
            FrameMetadata fm = new FrameMetadata(ff, fs, rows, cols);
            script.in("A", aIn, fm);
        }
        if (bFromDataFrame) {
            script.in("B", dfB);
        } else {
            JavaRDD<String> bIn = sc.textFile(input("B"));
            FrameSchema fs = new FrameSchema(lschemaB);
            FrameFormat ff = (format.equals("text")) ? FrameFormat.IJV : FrameFormat.CSV;
            FrameMetadata fm = new FrameMetadata(ff, fs, bRows, bCols);
            script.in("B", bIn, fm);
        }
        // Output one frame to HDFS and get one as RDD //TODO HDFS input/output to do
        script.out("A", "C");
        // set positional argument values
        for (int argNum = 1; argNum <= proArgs.size(); argNum++) {
            script.in("$" + argNum, proArgs.get(argNum - 1));
        }
        MLResults results = ml.execute(script);
        format = "csv";
        if (iinfo == InputInfo.TextCellInputInfo)
            format = "text";
        String fName = output("AB");
        try {
            MapReduceTool.deleteFileIfExistOnHDFS(fName);
        } catch (IOException e) {
            throw new DMLRuntimeException("Error: While deleting file on HDFS");
        }
        if (!bToDataFrame) {
            if (format.equals("text")) {
                JavaRDD<String> javaRDDStringIJV = results.getJavaRDDStringIJV("A");
                javaRDDStringIJV.saveAsTextFile(fName);
            } else {
                JavaRDD<String> javaRDDStringCSV = results.getJavaRDDStringCSV("A");
                javaRDDStringCSV.saveAsTextFile(fName);
            }
        } else {
            Dataset<Row> df = results.getDataFrame("A");
            // Convert back DataFrame to binary block for comparison using original binary to converted DF and back to binary
            MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, -1, -1, -1);
            JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc, bFromDataFrame).mapToPair(new LongFrameToLongWritableFrameFunction());
            rddOut.saveAsHadoopFile(output("AB"), LongWritable.class, FrameBlock.class, OutputInfo.BinaryBlockOutputInfo.outputFormatClass);
        }
        fName = output("C");
        try {
            MapReduceTool.deleteFileIfExistOnHDFS(fName);
        } catch (IOException e) {
            throw new DMLRuntimeException("Error: While deleting file on HDFS");
        }
        if (!bToDataFrame) {
            if (format.equals("text")) {
                JavaRDD<String> javaRDDStringIJV = results.getJavaRDDStringIJV("C");
                javaRDDStringIJV.saveAsTextFile(fName);
            } else {
                JavaRDD<String> javaRDDStringCSV = results.getJavaRDDStringCSV("C");
                javaRDDStringCSV.saveAsTextFile(fName);
            }
        } else {
            Dataset<Row> df = results.getDataFrame("C");
            // Convert back DataFrame to binary block for comparison using original binary to converted DF and back to binary
            MatrixCharacteristics mc = new MatrixCharacteristics(cRows, cCols, -1, -1, -1);
            JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc, bFromDataFrame).mapToPair(new LongFrameToLongWritableFrameFunction());
            rddOut.saveAsHadoopFile(fName, LongWritable.class, FrameBlock.class, OutputInfo.BinaryBlockOutputInfo.outputFormatClass);
        }
        runRScript(true);
        outputSchema.put("AB", schema);
        outputMC.put("AB", new MatrixCharacteristics(rows, cols, -1, -1));
        outputSchema.put("C", schemaC);
        outputMC.put("C", new MatrixCharacteristics(cRows, cCols, -1, -1));
        for (String file : config.getOutputFiles()) {
            MatrixCharacteristics md = outputMC.get(file);
            FrameBlock frameBlock = readDMLFrameFromHDFS(file, iinfo, md);
            FrameBlock frameRBlock = readRFrameFromHDFS(file + ".csv", InputInfo.CSVInputInfo, md);
            ValueType[] schemaOut = outputSchema.get(file);
            verifyFrameData(frameBlock, frameRBlock, schemaOut);
            System.out.println("File " + file + " processed successfully.");
        }
        System.out.println("Frame MLContext test completed successfully.");
    } finally {
        DMLScript.rtplatform = oldRT;
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
    }
}