Examples with SparkSession$ org.apache.spark.sql.SparkSession$ used on opensource projects

Example 1 with SparkSession$

use of org.apache.spark.sql.SparkSession$ in project Taier by DTStack.

the class SqlProxy method runJob.

public void runJob(String submitSql, String appName, String logLevel, SparkConf conf) {
    if (appName == null) {
        appName = DEFAULT_APP_NAME;
    }
    SparkSession spark = SparkSession.builder().config(conf).appName(appName).enableHiveSupport().getOrCreate();
    setLogLevel(spark, logLevel);
    // 解压sql
    String unzipSql = ZipUtil.unzip(submitSql);
    // 屏蔽引号内的 分号
    Splitter splitter = new Splitter(';');
    List<String> sqlArray = splitter.splitEscaped(unzipSql);
    for (String sql : sqlArray) {
        if (sql == null || sql.trim().length() == 0) {
            continue;
        }
        logger.info("processed sql statement {}", sql);
        spark.sql(sql);
    }
    spark.close();
}

Example 2 with SparkSession$

use of org.apache.spark.sql.SparkSession$ in project coral by linkedin.

the class TransportableUDFMapTest method testScalaVersionWithSparkSession.

@Test
public void testScalaVersionWithSparkSession() {
    SparkSession ss = SparkSession.builder().appName(TransportableUDFMapTest.class.getSimpleName()).master("local[1]").enableHiveSupport().getOrCreate();
    Assert.assertEquals(TransportableUDFMap.getScalaVersion(), TransportableUDFMap.ScalaVersion.SCALA_2_11);
    ss.close();
}

Example 3 with SparkSession$

use of org.apache.spark.sql.SparkSession$ in project iceberg by apache.

the class TestSparkSchema method stopSpark.

@AfterClass
public static void stopSpark() {
    SparkSession currentSpark = TestSparkSchema.spark;
    TestSparkSchema.spark = null;
    currentSpark.stop();
}

Example 4 with SparkSession$

use of org.apache.spark.sql.SparkSession$ in project iceberg by apache.

the class TestNameMappingProjection method stopSpark.

@AfterClass
public static void stopSpark() {
    SparkSession currentSpark = TestNameMappingProjection.spark;
    // Stop the spark session.
    TestNameMappingProjection.spark = null;
    currentSpark.stop();
}

Example 5 with SparkSession$

use of org.apache.spark.sql.SparkSession$ in project iceberg by apache.

the class Spark3Util method getPartitions.

/**
 * Use Spark to list all partitions in the table.
 *
 * @param spark a Spark session
 * @param rootPath a table identifier
 * @param format format of the file
 * @return all table's partitions
 */
public static List<SparkPartition> getPartitions(SparkSession spark, Path rootPath, String format) {
    FileStatusCache fileStatusCache = FileStatusCache.getOrCreate(spark);
    Map<String, String> emptyMap = Collections.emptyMap();
    InMemoryFileIndex fileIndex = new InMemoryFileIndex(spark, JavaConverters.collectionAsScalaIterableConverter(ImmutableList.of(rootPath)).asScala().toSeq(), JavaConverters.mapAsScalaMapConverter(emptyMap).asScala().toMap(Predef.conforms()), Option.empty(), fileStatusCache, Option.empty(), Option.empty());
    org.apache.spark.sql.execution.datasources.PartitionSpec spec = fileIndex.partitionSpec();
    StructType schema = spec.partitionColumns();
    return JavaConverters.seqAsJavaListConverter(spec.partitions()).asJava().stream().map(partition -> {
        Map<String, String> values = Maps.newHashMap();
        JavaConverters.asJavaIterableConverter(schema).asJava().forEach(field -> {
            int fieldIndex = schema.fieldIndex(field.name());
            Object catalystValue = partition.values().get(fieldIndex, field.dataType());
            Object value = CatalystTypeConverters.convertToScala(catalystValue, field.dataType());
            values.put(field.name(), String.valueOf(value));
        });
        return new SparkPartition(values, partition.path().toString(), format);
    }).collect(Collectors.toList());
}