Examples with FileSystem org.flyte.jflyte.api.FileSystem used on opensource projects

Example 1 with FileSystem

use of org.flyte.jflyte.api.FileSystem in project flytekit-java by flyteorg.

the class PackageLoader method loadPackage.

private static ClassLoader loadPackage(Map<String, FileSystem> fileSystems, List<Artifact> artifacts) {
    try {
        Path tmp = Files.createTempDirectory("tasks");
        for (Artifact artifact : artifacts) {
            FileSystem fileSystem = FileSystemLoader.getFileSystem(fileSystems, artifact.location());
            try (ReadableByteChannel reader = fileSystem.reader(artifact.location())) {
                Path path = tmp.resolve(artifact.name());
                if (path.toFile().exists()) {
                    // file already exists, but we have checksums, so we should be ok
                    LOG.warn("Duplicate entry in artifacts: [{}]", artifact);
                    continue;
                }
                LOG.info("Copied {} to {}", artifact.location(), path);
                Files.copy(Channels.newInputStream(reader), path);
            }
        }
        return ClassLoaders.forDirectory(tmp.toFile());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 2 with FileSystem

use of org.flyte.jflyte.api.FileSystem in project flytekit-java by flyteorg.

the class FileSystemLoader method getFileSystem.

static FileSystem getFileSystem(Map<String, FileSystem> fileSystems, URI uri) {
    String scheme = uri.getScheme();
    FileSystem fileSystem = fileSystems.get(scheme);
    return Verify.verifyNotNull(fileSystem, "Can't find FileSystem for [%s]", scheme);
}

Example 3 with FileSystem

use of org.flyte.jflyte.api.FileSystem in project flytekit-java by flyteorg.

the class FileSystemLoader method loadFileSystems.

static List<FileSystem> loadFileSystems() {
    ServiceLoader<FileSystemRegistrar> loader = ServiceLoader.load(FileSystemRegistrar.class);
    LOG.debug("Discovering FileSystemRegistrar");
    List<FileSystem> fileSystems = new ArrayList<>();
    Map<String, String> env = ImmutableMap.copyOf(System.getenv());
    for (FileSystemRegistrar registrar : loader) {
        for (FileSystem fileSystem : registrar.load(env)) {
            LOG.debug(String.format("Discovered FileSystem [%s]", fileSystem.getClass().getName()));
            fileSystems.add(fileSystem);
        }
    }
    return fileSystems;
}

Example 4 with FileSystem

use of org.flyte.jflyte.api.FileSystem in project flytekit-java by flyteorg.

the class ArtifactStager method create.

static ArtifactStager create(Config config, Collection<ClassLoader> modules) {
    try {
        String stagingLocation = config.stagingLocation();
        if (stagingLocation == null) {
            throw new IllegalArgumentException("Environment variable 'FLYTE_STAGING_LOCATION' isn't set");
        }
        URI stagingUri = new URI(stagingLocation);
        Map<String, FileSystem> fileSystems = FileSystemLoader.loadFileSystems(modules);
        FileSystem stagingFileSystem = FileSystemLoader.getFileSystem(fileSystems, stagingUri);
        return new ArtifactStager(stagingLocation, stagingFileSystem);
    } catch (URISyntaxException e) {
        throw new IllegalArgumentException("Failed to parse stagingLocation", e);
    }
}

Example 5 with FileSystem

use of org.flyte.jflyte.api.FileSystem in project flytekit-java by flyteorg.

the class Execute method execute.

private void execute() {
    Config config = Config.load();
    Collection<ClassLoader> modules = ClassLoaders.forModuleDir(config.moduleDir()).values();
    Map<String, FileSystem> fileSystems = FileSystemLoader.loadFileSystems(modules);
    FileSystem outputFs = FileSystemLoader.getFileSystem(fileSystems, outputPrefix);
    ProtoWriter protoWriter = new ProtoWriter(outputPrefix, outputFs);
    try {
        FileSystem inputFs = FileSystemLoader.getFileSystem(fileSystems, inputs);
        ProtoReader protoReader = new ProtoReader(inputFs);
        TaskTemplate taskTemplate = protoReader.getTaskTemplate(taskTemplatePath);
        ClassLoader packageClassLoader = PackageLoader.load(fileSystems, taskTemplate);
        // before we run anything, switch class loader, otherwise,
        // ServiceLoaders and other things wouldn't work, for instance,
        // FileSystemRegister in Apache Beam
        Map<String, Literal> outputs = withClassLoader(packageClassLoader, () -> {
            Map<String, Literal> input = protoReader.getInput(inputs);
            RunnableTask runnableTask = getTask(task);
            return runnableTask.run(input);
        });
        protoWriter.writeOutputs(outputs);
    } catch (ContainerError e) {
        LOG.error("failed to run task", e);
        protoWriter.writeError(ProtoUtil.serializeContainerError(e));
    } catch (Throwable e) {
        LOG.error("failed to run task", e);
        protoWriter.writeError(ProtoUtil.serializeThrowable(e));
    }
}