Examples with Attribute org.apache.spark.sql.catalyst.expressions.Attribute used on opensource projects

Example 1 with Attribute

use of org.apache.spark.sql.catalyst.expressions.Attribute in project zeppelin by apache.

the class ZeppelinContext method showDF.

public static String showDF(SparkContext sc, InterpreterContext interpreterContext, Object df, int maxResult) {
    Object[] rows = null;
    Method take;
    String jobGroup = Utils.buildJobGroupId(interpreterContext);
    sc.setJobGroup(jobGroup, "Zeppelin", false);
    try {
        // and assume it is type Row.
        if (df.getClass().getCanonicalName().equals("org.apache.spark.sql.Dataset")) {
            Method convertToDFMethod = df.getClass().getMethod("toDF");
            df = convertToDFMethod.invoke(df);
        }
        take = df.getClass().getMethod("take", int.class);
        rows = (Object[]) take.invoke(df, maxResult + 1);
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | ClassCastException e) {
        sc.clearJobGroup();
        throw new InterpreterException(e);
    }
    List<Attribute> columns = null;
    // get field names
    try {
        // Use reflection because of classname returned by queryExecution changes from
        // Spark <1.5.2 org.apache.spark.sql.SQLContext$QueryExecution
        // Spark 1.6.0> org.apache.spark.sql.hive.HiveContext$QueryExecution
        Object qe = df.getClass().getMethod("queryExecution").invoke(df);
        Object a = qe.getClass().getMethod("analyzed").invoke(qe);
        scala.collection.Seq seq = (scala.collection.Seq) a.getClass().getMethod("output").invoke(a);
        columns = (List<Attribute>) scala.collection.JavaConverters.seqAsJavaListConverter(seq).asJava();
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
        throw new InterpreterException(e);
    }
    StringBuilder msg = new StringBuilder();
    msg.append("%table ");
    for (Attribute col : columns) {
        msg.append(col.name() + "\t");
    }
    String trim = msg.toString().trim();
    msg = new StringBuilder(trim);
    msg.append("\n");
    try {
        for (int r = 0; r < maxResult && r < rows.length; r++) {
            Object row = rows[r];
            Method isNullAt = row.getClass().getMethod("isNullAt", int.class);
            Method apply = row.getClass().getMethod("apply", int.class);
            for (int i = 0; i < columns.size(); i++) {
                if (!(Boolean) isNullAt.invoke(row, i)) {
                    msg.append(apply.invoke(row, i).toString());
                } else {
                    msg.append("null");
                }
                if (i != columns.size() - 1) {
                    msg.append("\t");
                }
            }
            msg.append("\n");
        }
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
        throw new InterpreterException(e);
    }
    if (rows.length > maxResult) {
        msg.append("<!--TABLE_COMMENT-->");
        msg.append("\n");
        msg.append("<font color=red>Results are limited by " + maxResult + ".</font>");
    }
    sc.clearJobGroup();
    return msg.toString();
}