Examples with LocalizeResource co.cask.cdap.internal.app.runtime.distributed.LocalizeResource used on opensource projects

Example 16 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class SparkPackageUtils method prepareSpark2Framework.

/**
   * Prepares the Spark 2 framework on the location.
   *
   * @param sparkConf the spark configuration
   * @param locationFactory the {@link LocationFactory} for saving the spark framework jar
   * @param tempDir directory for temporary file creation
   * @return A {@link SparkFramework} containing information about the spark framework in localization context.
   * @throws IOException If failed to prepare the framework.
   */
private static SparkFramework prepareSpark2Framework(Properties sparkConf, LocationFactory locationFactory, File tempDir) throws IOException {
    String sparkYarnArchive = sparkConf.getProperty(SPARK_YARN_ARCHIVE);
    if (sparkYarnArchive != null) {
        URI sparkYarnArchiveURI = URI.create(sparkYarnArchive);
        if (locationFactory.getHomeLocation().toURI().getScheme().equals(sparkYarnArchiveURI.getScheme())) {
            Location frameworkLocation = locationFactory.create(URI.create(sparkYarnArchive));
            if (frameworkLocation.exists()) {
                return new SparkFramework(new LocalizeResource(resolveURI(frameworkLocation), true), SPARK_YARN_ARCHIVE);
            }
            LOG.warn("The location {} set by '{}' does not exist.", frameworkLocation, SPARK_YARN_ARCHIVE);
        }
    }
    // If spark.yarn.archive is not defined or doesn't exists, build a archive zip from local FS and upload it
    String sparkVersion = System.getenv(SPARK_VERSION);
    sparkVersion = sparkVersion == null ? SparkCompat.SPARK2_2_11.getCompat() : sparkVersion;
    String archiveName = "spark.archive-" + sparkVersion + ".zip";
    Location frameworkDir = locationFactory.create("/framework/spark");
    Location frameworkLocation = frameworkDir.append(archiveName);
    if (!frameworkLocation.exists()) {
        File archive = new File(tempDir, archiveName);
        try {
            try (ZipOutputStream zipOutput = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(archive)))) {
                zipOutput.setLevel(Deflater.NO_COMPRESSION);
                for (File file : getLocalSparkLibrary(SparkCompat.SPARK2_2_11)) {
                    zipOutput.putNextEntry(new ZipEntry(file.getName()));
                    Files.copy(file.toPath(), zipOutput);
                    zipOutput.closeEntry();
                }
            }
            // Upload spark archive to the framework location
            frameworkDir.mkdirs("755");
            try (OutputStream os = frameworkLocation.getOutputStream("644")) {
                Files.copy(archive.toPath(), os);
            }
        } finally {
            archive.delete();
        }
    }
    return new SparkFramework(new LocalizeResource(resolveURI(frameworkLocation), true), SPARK_YARN_ARCHIVE);
}

Example 17 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class BasicSparkClientContext method localize.

@Override
public void localize(String name, URI uri, boolean archive) {
    try {
        URI actualURI = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), name);
        localizeResources.put(name, new LocalizeResource(actualURI, archive));
    } catch (URISyntaxException e) {
        // If it does though, there is nothing that clients can do to recover, so not propagating a checked exception.
        throw Throwables.propagate(e);
    }
}