Search in sources :

Example 1 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class ClassLoaderPureGraphCache method buildRepoAndSources.

@Override
public boolean buildRepoAndSources(ModelRepository modelRepository, SourceRegistry sourceRegistry, ParserLibrary parserLibrary, Context context, ProcessorSupport processorSupport, Message message) {
    if (this.runtime == null) {
        return false;
    }
    try {
        CodeStorage codeStorage = this.runtime.getCodeStorage();
        MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
        PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(GraphLoader.findJars(repoNames, this.classLoader, message));
        GraphLoader loader = new GraphLoader(modelRepository, context, parserLibrary, this.runtime.getIncrementalCompiler().getDslLibrary(), sourceRegistry, null, jarLibrary, this.forkJoinPool);
        for (String repoName : repoNames) {
            loader.loadRepository(repoName, message);
        }
        this.state.update(true, -1L, true, null);
        return true;
    } catch (Exception e) {
        modelRepository.clear();
        this.state.update(false, -1L, false, e);
        return false;
    } catch (Error e) {
        this.state.update(false, -1L, false, e);
        throw e;
    }
}
Also used : CodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.CodeStorage) GraphLoader(org.finos.legend.pure.m3.serialization.runtime.GraphLoader) RepositoryComparator(org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator) SimplePureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary) PureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary)

Example 2 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class FSGraphLoaderPureGraphCache method buildFromCaches.

@Override
public boolean buildFromCaches(ModelRepository modelRepository, SourceRegistry sources, ParserLibrary library, Context context, ProcessorSupport processorSupport, Message message) {
    PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibraryFromDirectory(getCacheLocation());
    final GraphLoader loader = new GraphLoader(modelRepository, context, library, this.pureRuntime.getIncrementalCompiler().getDslLibrary(), sources, null, jarLibrary, this.forkJoinPool);
    CodeStorage codeStorage = this.pureRuntime.getCodeStorage();
    MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
    if (shouldAddRootRepo()) {
        repoNames.add(ROOT_REPOSITORY_NAME);
    }
    if (this.allowBuildingFromRepoSubset) {
        repoNames.removeIf(new Predicate<String>() {

            @Override
            public boolean accept(String repoName) {
                return !loader.isKnownRepository(repoName);
            }
        });
    }
    for (String repoName : repoNames) {
        loader.loadRepository(repoName, message);
    }
    updateCacheState();
    return true;
}
Also used : PureCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.PureCodeStorage) CodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.CodeStorage) GraphLoader(org.finos.legend.pure.m3.serialization.runtime.GraphLoader) RepositoryComparator(org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator) SimplePureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary) PureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary)

Example 3 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class MemoryGraphLoaderPureGraphCache method buildFromCaches.

@Override
protected boolean buildFromCaches(ModelRepository modelRepository, SourceRegistry sources, ParserLibrary library, Context context, ProcessorSupport processorSupport, Message message) {
    PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(this.jars);
    GraphLoader loader = new GraphLoader(modelRepository, context, library, this.pureRuntime.getIncrementalCompiler().getDslLibrary(), sources, null, jarLibrary, this.forkJoinPool);
    loader.loadAll(message);
    updateCacheState();
    return true;
}
Also used : GraphLoader(org.finos.legend.pure.m3.serialization.runtime.GraphLoader) SimplePureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary) PureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary)

Example 4 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class AbstractPureRepositoryJarLibraryTest method testReadRepositories.

protected static void testReadRepositories(PureRuntime runtime, PureRepositoryJarLibrary library, String... repositories) {
    SetIterable<String> repositorySet = Sets.immutable.with(repositories);
    MapIterable<String, byte[]> bytesByFile = library.readRepositoryFiles(repositorySet);
    MutableMap<String, byte[]> expected = Maps.mutable.empty();
    ByteArrayOutputStream expectedStream = new ByteArrayOutputStream();
    Writer writer = BinaryWriters.newBinaryWriter(expectedStream);
    for (Source source : runtime.getSourceRegistry().getSources()) {
        String purePath = source.getId();
        String repo = getRepo(purePath);
        if ((repo != null) && repositorySet.contains(repo)) {
            String binPath = PureRepositoryJarTools.purePathToBinaryPath(purePath);
            expectedStream.reset();
            BinaryModelSourceSerializer.serialize(writer, source, runtime);
            expected.put(binPath, expectedStream.toByteArray());
        }
    }
    assertFileByteMapsEqual(expected, bytesByFile);
}
Also used : ByteArrayOutputStream(java.io.ByteArrayOutputStream) Writer(org.finos.legend.pure.m4.serialization.Writer) Source(org.finos.legend.pure.m3.serialization.runtime.Source)

Example 5 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class PureJarSerializer method writePureRepositoryJars.

private static void writePureRepositoryJars(Path outputDirectory, Path sourceDirectory, String platformVersion, RichIterable<CodeRepository> repositories, Log log) throws IOException {
    SetIterable<CodeRepository> repositoriesForCompilation = PureCodeStorage.getRepositoryDependencies(PureRepositoriesExternal.repositories(), repositories);
    PureRuntime runtime;
    RichIterable<CodeRepository> repositoriesToSerialize;
    if (null == sourceDirectory) {
        log.info("    *Building code storage leveraging the class loader (as sourceDirectory is not specified)");
        ClassLoaderCodeStorage cs = new ClassLoaderCodeStorage(repositoriesForCompilation);
        log.info("      " + cs.getRepositories().collect(CodeRepository::getName) + " - " + cs.getUserFiles().size() + " files");
        MutableCodeStorage codeStorage = new PureCodeStorage(null, cs);
        Message message = getMessage(log, "      ");
        log.info("    *Starting file compilation");
        runtime = new PureRuntimeBuilder(codeStorage).setTransactionalByDefault(false).buildAndInitialize(message);
        repositoriesToSerialize = repositoriesForCompilation;
        log.info("      -> Finished compilation");
    } else {
        log.info("    *Building code storage leveraging the sourceDirectory " + sourceDirectory);
        PartitionIterable<CodeRepository> partition = repositories.partition(r -> Files.exists(sourceDirectory.resolve(r.getName())));
        RichIterable<CodeRepository> repositoriesWithSource = partition.getSelected();
        RichIterable<CodeRepository> repositoriesWithoutSource = partition.getRejected();
        // Code Storages
        MutableList<AbstractRepositoryCodeStorage> codeStoragesFromSource = Lists.mutable.withAll(repositoriesWithSource.collect(r -> new MutableFSCodeStorage(r, sourceDirectory.resolve(r.getName()))));
        MutableList<AbstractRepositoryCodeStorage> allCodeStorage = Lists.mutable.withAll(codeStoragesFromSource).withAll(repositoriesWithoutSource.collect(EmptyCodeStorage::new));
        log.info("    *Loading the following repo from PARs: " + repositoriesWithoutSource.collect(CodeRepository::getName));
        // Build the runtime
        MutableCodeStorage codeStorage = new PureCodeStorage(sourceDirectory, allCodeStorage.toArray(new RepositoryCodeStorage[0]));
        runtime = new PureRuntimeBuilder(codeStorage).setTransactionalByDefault(false).build();
        // Load the PARS
        MutableList<String> namesOfRepoWithoutAvailableSource = repositoriesWithoutSource.collect(CodeRepository::getName).toList();
        Message message = getMessage(log, "    ");
        PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(GraphLoader.findJars(namesOfRepoWithoutAvailableSource, PureJarSerializer.class.getClassLoader(), message));
        final GraphLoader loader = new GraphLoader(runtime.getModelRepository(), runtime.getContext(), runtime.getIncrementalCompiler().getParserLibrary(), runtime.getIncrementalCompiler().getDslLibrary(), runtime.getSourceRegistry(), runtime.getURLPatternLibrary(), jarLibrary);
        loader.loadAll(message);
        // Compile Sources
        log.info("    *Starting file compilation");
        codeStoragesFromSource.forEach(r -> log.info("      " + r.getRepositories().collect(CodeRepository::getName) + " - " + r.getUserFiles().size() + " files (from: " + ((FSCodeStorage) r).getRoot() + ")"));
        for (CodeRepository repository : repositoriesWithSource) {
            Path path = sourceDirectory.resolve(repository.getName());
            log.info("      Compiling repository '" + repository.getName() + "' in " + path);
            MutableFSCodeStorage fs = new MutableFSCodeStorage(repository, path);
            log.info("        Found " + fs.getUserFiles().size() + " files");
            runtime.loadAndCompile(new PureCodeStorage(sourceDirectory, fs).getUserFiles());
            log.info("        -> Finished compiling repository '" + repository.getName() + "'");
        }
        repositoriesToSerialize = repositoriesWithSource;
        log.info("      -> Finished compilation");
    }
    Files.createDirectories(outputDirectory);
    log.info("    *Starting serialization");
    for (String repositoryName : repositoriesToSerialize.collect(CodeRepository::getName)) {
        Path outputFile = outputDirectory.resolve("pure-" + repositoryName + "." + ARCHIVE_FILE_EXTENSION);
        log.info("      Writing " + outputFile);
        try (OutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(outputFile))) {
            BinaryModelRepositorySerializer.serialize(outputStream, platformVersion, null, repositoryName, runtime);
        }
    }
}
Also used : Message(org.finos.legend.pure.m3.serialization.runtime.Message) CodeRepository(org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository) Lists(org.eclipse.collections.api.factory.Lists) PureRepositoriesExternal(org.finos.legend.pure.configuration.PureRepositoriesExternal) SetIterable(org.eclipse.collections.api.set.SetIterable) SimplePureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary) PureCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.PureCodeStorage) MutableList(org.eclipse.collections.api.list.MutableList) AbstractRepositoryCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.AbstractRepositoryCodeStorage) FSCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.fs.FSCodeStorage) BufferedOutputStream(java.io.BufferedOutputStream) PureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary) RichIterable(org.eclipse.collections.api.RichIterable) RepositoryCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.RepositoryCodeStorage) PureRuntimeBuilder(org.finos.legend.pure.m3.serialization.runtime.PureRuntimeBuilder) Path(java.nio.file.Path) OutputStream(java.io.OutputStream) Files(java.nio.file.Files) PureRuntime(org.finos.legend.pure.m3.serialization.runtime.PureRuntime) EmptyCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.EmptyCodeStorage) IOException(java.io.IOException) Log(org.apache.maven.plugin.logging.Log) ClassLoaderCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.classpath.ClassLoaderCodeStorage) MutableCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.MutableCodeStorage) GraphLoader(org.finos.legend.pure.m3.serialization.runtime.GraphLoader) Sets(org.eclipse.collections.impl.factory.Sets) PartitionIterable(org.eclipse.collections.api.partition.PartitionIterable) BinaryModelRepositorySerializer(org.finos.legend.pure.m3.serialization.runtime.binary.BinaryModelRepositorySerializer) MutableFSCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.fs.MutableFSCodeStorage) Path(java.nio.file.Path) PureRuntime(org.finos.legend.pure.m3.serialization.runtime.PureRuntime) CodeRepository(org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository) Message(org.finos.legend.pure.m3.serialization.runtime.Message) BufferedOutputStream(java.io.BufferedOutputStream) OutputStream(java.io.OutputStream) PureRuntimeBuilder(org.finos.legend.pure.m3.serialization.runtime.PureRuntimeBuilder) SimplePureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary) PureRepositoryJarLibrary(org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary) AbstractRepositoryCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.AbstractRepositoryCodeStorage) GraphLoader(org.finos.legend.pure.m3.serialization.runtime.GraphLoader) ClassLoaderCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.classpath.ClassLoaderCodeStorage) MutableCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.MutableCodeStorage) FSCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.fs.FSCodeStorage) MutableFSCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.fs.MutableFSCodeStorage) PureCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.PureCodeStorage) MutableFSCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.fs.MutableFSCodeStorage) BufferedOutputStream(java.io.BufferedOutputStream) AbstractRepositoryCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.AbstractRepositoryCodeStorage) RepositoryCodeStorage(org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.RepositoryCodeStorage)

Aggregations

GraphLoader (org.finos.legend.pure.m3.serialization.runtime.GraphLoader)4 PureRepositoryJarLibrary (org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary)4 SimplePureRepositoryJarLibrary (org.finos.legend.pure.m3.serialization.runtime.binary.SimplePureRepositoryJarLibrary)4 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3 Source (org.finos.legend.pure.m3.serialization.runtime.Source)3 Writer (org.finos.legend.pure.m4.serialization.Writer)3 PureCodeStorage (org.finos.legend.pure.m3.serialization.filesystem.PureCodeStorage)2 CodeStorage (org.finos.legend.pure.m3.serialization.filesystem.usercodestorage.CodeStorage)2 RepositoryComparator (org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator)2 BufferedOutputStream (java.io.BufferedOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 Log (org.apache.maven.plugin.logging.Log)1 RichIterable (org.eclipse.collections.api.RichIterable)1 Lists (org.eclipse.collections.api.factory.Lists)1 MutableList (org.eclipse.collections.api.list.MutableList)1 PartitionIterable (org.eclipse.collections.api.partition.PartitionIterable)1 SetIterable (org.eclipse.collections.api.set.SetIterable)1