Examples with LocalFile com.continuuity.weave.api.LocalFile used on opensource projects

Example 1 with LocalFile

use of com.continuuity.weave.api.LocalFile in project weave by continuuity.

the class WeaveContainerMain method renameLocalFiles.

private static void renameLocalFiles(RuntimeSpecification runtimeSpec) {
    for (LocalFile file : runtimeSpec.getLocalFiles()) {
        if (file.isArchive()) {
            String path = file.getURI().toString();
            String name = file.getName() + (path.endsWith(".tar.gz") ? ".tar.gz" : path.substring(path.lastIndexOf('.')));
            Preconditions.checkState(new File(name).renameTo(new File(file.getName())), "Fail to rename file from %s to %s.", name, file.getName());
        }
    }
}

Example 2 with LocalFile

use of com.continuuity.weave.api.LocalFile in project weave by continuuity.

the class YarnWeavePreparer method saveLocalFiles.

/**
   * Serializes the list of files that needs to localize from AM to Container.
   */
private void saveLocalFiles(Map<String, LocalFile> localFiles, Set<String> includes) throws IOException {
    Map<String, LocalFile> localize = ImmutableMap.copyOf(Maps.filterKeys(localFiles, Predicates.in(includes)));
    LOG.debug("Create and copy {}", Constants.Files.LOCALIZE_FILES);
    Location location = createTempLocation(Constants.Files.LOCALIZE_FILES);
    Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
    try {
        new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec()).create().toJson(localize.values(), new TypeToken<List<LocalFile>>() {
        }.getType(), writer);
    } finally {
        writer.close();
    }
    LOG.debug("Done {}", Constants.Files.LOCALIZE_FILES);
    localFiles.put(Constants.Files.LOCALIZE_FILES, createLocalFile(Constants.Files.LOCALIZE_FILES, location));
}

Example 3 with LocalFile

use of com.continuuity.weave.api.LocalFile in project weave by continuuity.

the class WeaveContainerLauncher method start.

public WeaveContainerController start(RunId runId, int instanceId, Class<?> mainClass, String classPath) {
    ProcessLauncher.PrepareLaunchContext.AfterResources afterResources = null;
    ProcessLauncher.PrepareLaunchContext.ResourcesAdder resourcesAdder = null;
    // Clean up zookeeper path in case this is a retry and there are old messages and state there.
    Futures.getUnchecked(ZKOperations.ignoreError(ZKOperations.recursiveDelete(zkClient, "/" + runId), KeeperException.NoNodeException.class, null));
    // Adds all file to be localized to container
    if (!runtimeSpec.getLocalFiles().isEmpty()) {
        resourcesAdder = launchContext.withResources();
        for (LocalFile localFile : runtimeSpec.getLocalFiles()) {
            afterResources = resourcesAdder.add(localFile);
        }
    }
    // Optionally localize secure store.
    try {
        if (secureStoreLocation != null && secureStoreLocation.exists()) {
            if (resourcesAdder == null) {
                resourcesAdder = launchContext.withResources();
            }
            afterResources = resourcesAdder.add(new DefaultLocalFile(Constants.Files.CREDENTIALS, secureStoreLocation.toURI(), secureStoreLocation.lastModified(), secureStoreLocation.length(), false, null));
        }
    } catch (IOException e) {
        LOG.warn("Failed to launch container with secure store {}.", secureStoreLocation.toURI());
    }
    if (afterResources == null) {
        afterResources = launchContext.noResources();
    }
    int memory = runtimeSpec.getResourceSpecification().getMemorySize();
    if (((double) (memory - reservedMemory) / memory) >= HEAP_MIN_RATIO) {
        // Reduce -Xmx by the reserved memory size.
        memory = runtimeSpec.getResourceSpecification().getMemorySize() - reservedMemory;
    } else {
        // If it is a small VM, just discount it by the min ratio.
        memory = (int) Math.ceil(memory * HEAP_MIN_RATIO);
    }
    // Currently no reporting is supported for runnable containers
    ProcessController<Void> processController = afterResources.withEnvironment().add(EnvKeys.WEAVE_RUN_ID, runId.getId()).add(EnvKeys.WEAVE_RUNNABLE_NAME, runtimeSpec.getName()).add(EnvKeys.WEAVE_INSTANCE_ID, Integer.toString(instanceId)).add(EnvKeys.WEAVE_INSTANCE_COUNT, Integer.toString(instanceCount)).withCommands().add("java", "-Djava.io.tmpdir=tmp", "-Dyarn.container=$" + EnvKeys.YARN_CONTAINER_ID, "-Dweave.runnable=$" + EnvKeys.WEAVE_APP_NAME + ".$" + EnvKeys.WEAVE_RUNNABLE_NAME, "-cp", Constants.Files.LAUNCHER_JAR + ":" + classPath, "-Xmx" + memory + "m", jvmOpts, WeaveLauncher.class.getName(), Constants.Files.CONTAINER_JAR, mainClass.getName(), Boolean.TRUE.toString()).redirectOutput(Constants.STDOUT).redirectError(Constants.STDERR).launch();
    WeaveContainerControllerImpl controller = new WeaveContainerControllerImpl(zkClient, runId, processController);
    controller.start();
    return controller;
}

Example 4 with LocalFile

use of com.continuuity.weave.api.LocalFile in project weave by continuuity.

the class RuntimeSpecificationCodec method deserialize.

@Override
public RuntimeSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String name = jsonObj.get("name").getAsString();
    WeaveRunnableSpecification runnable = context.deserialize(jsonObj.get("runnable"), WeaveRunnableSpecification.class);
    ResourceSpecification resources = context.deserialize(jsonObj.get("resources"), ResourceSpecification.class);
    Collection<LocalFile> files = context.deserialize(jsonObj.get("files"), new TypeToken<Collection<LocalFile>>() {
    }.getType());
    return new DefaultRuntimeSpecification(name, runnable, resources, files);
}

Example 5 with LocalFile

use of com.continuuity.weave.api.LocalFile in project weave by continuuity.

the class YarnWeavePreparer method populateRunnableLocalFiles.

/**
   * Based on the given {@link WeaveSpecification}, upload LocalFiles to Yarn Cluster.
   * @param weaveSpec The {@link WeaveSpecification} for populating resource.
   * @param localFiles A Multimap to store runnable name to transformed LocalFiles.
   * @throws IOException
   */
private void populateRunnableLocalFiles(WeaveSpecification weaveSpec, Multimap<String, LocalFile> localFiles) throws IOException {
    LOG.debug("Populating Runnable LocalFiles");
    for (Map.Entry<String, RuntimeSpecification> entry : weaveSpec.getRunnables().entrySet()) {
        String runnableName = entry.getKey();
        for (LocalFile localFile : entry.getValue().getLocalFiles()) {
            Location location;
            URI uri = localFile.getURI();
            if ("hdfs".equals(uri.getScheme())) {
                // Assuming the location factory is HDFS one. If it is not, it will failed, which is the correct behavior.
                location = locationFactory.create(uri);
            } else {
                URL url = uri.toURL();
                LOG.debug("Create and copy {} : {}", runnableName, url);
                // Preserves original suffix for expansion.
                location = copyFromURL(url, createTempLocation(Paths.appendSuffix(url.getFile(), localFile.getName())));
                LOG.debug("Done {} : {}", runnableName, url);
            }
            localFiles.put(runnableName, new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern()));
        }
    }
    LOG.debug("Done Runnable LocalFiles");
}