Examples with PureRepositoryJarLibrary org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary used on opensource projects

Example 1 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class ClassLoaderPureGraphCache method buildRepoAndSources.

@Override
public boolean buildRepoAndSources(ModelRepository modelRepository, SourceRegistry sourceRegistry, ParserLibrary parserLibrary, Context context, ProcessorSupport processorSupport, Message message) {
    if (this.runtime == null) {
        return false;
    }
    try {
        CodeStorage codeStorage = this.runtime.getCodeStorage();
        MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
        PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(GraphLoader.findJars(repoNames, this.classLoader, message));
        GraphLoader loader = new GraphLoader(modelRepository, context, parserLibrary, this.runtime.getIncrementalCompiler().getDslLibrary(), sourceRegistry, null, jarLibrary, this.forkJoinPool);
        for (String repoName : repoNames) {
            loader.loadRepository(repoName, message);
        }
        this.state.update(true, -1L, true, null);
        return true;
    } catch (Exception e) {
        modelRepository.clear();
        this.state.update(false, -1L, false, e);
        return false;
    } catch (Error e) {
        this.state.update(false, -1L, false, e);
        throw e;
    }
}

Example 2 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class FSGraphLoaderPureGraphCache method buildFromCaches.

@Override
public boolean buildFromCaches(ModelRepository modelRepository, SourceRegistry sources, ParserLibrary library, Context context, ProcessorSupport processorSupport, Message message) {
    PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibraryFromDirectory(getCacheLocation());
    final GraphLoader loader = new GraphLoader(modelRepository, context, library, this.pureRuntime.getIncrementalCompiler().getDslLibrary(), sources, null, jarLibrary, this.forkJoinPool);
    CodeStorage codeStorage = this.pureRuntime.getCodeStorage();
    MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
    if (shouldAddRootRepo()) {
        repoNames.add(ROOT_REPOSITORY_NAME);
    }
    if (this.allowBuildingFromRepoSubset) {
        repoNames.removeIf(new Predicate<String>() {

            @Override
            public boolean accept(String repoName) {
                return !loader.isKnownRepository(repoName);
            }
        });
    }
    for (String repoName : repoNames) {
        loader.loadRepository(repoName, message);
    }
    updateCacheState();
    return true;
}

Example 3 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class MemoryGraphLoaderPureGraphCache method buildFromCaches.

@Override
protected boolean buildFromCaches(ModelRepository modelRepository, SourceRegistry sources, ParserLibrary library, Context context, ProcessorSupport processorSupport, Message message) {
    PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(this.jars);
    GraphLoader loader = new GraphLoader(modelRepository, context, library, this.pureRuntime.getIncrementalCompiler().getDslLibrary(), sources, null, jarLibrary, this.forkJoinPool);
    loader.loadAll(message);
    updateCacheState();
    return true;
}

Example 4 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class AbstractPureRepositoryJarLibraryTest method testReadRepositories.

protected static void testReadRepositories(PureRuntime runtime, PureRepositoryJarLibrary library, String... repositories) {
    SetIterable<String> repositorySet = Sets.immutable.with(repositories);
    MapIterable<String, byte[]> bytesByFile = library.readRepositoryFiles(repositorySet);
    MutableMap<String, byte[]> expected = Maps.mutable.empty();
    ByteArrayOutputStream expectedStream = new ByteArrayOutputStream();
    Writer writer = BinaryWriters.newBinaryWriter(expectedStream);
    for (Source source : runtime.getSourceRegistry().getSources()) {
        String purePath = source.getId();
        String repo = getRepo(purePath);
        if ((repo != null) && repositorySet.contains(repo)) {
            String binPath = PureRepositoryJarTools.purePathToBinaryPath(purePath);
            expectedStream.reset();
            BinaryModelSourceSerializer.serialize(writer, source, runtime);
            expected.put(binPath, expectedStream.toByteArray());
        }
    }
    assertFileByteMapsEqual(expected, bytesByFile);
}

Example 5 with PureRepositoryJarLibrary

use of org.finos.legend.pure.m3.serialization.runtime.binary.PureRepositoryJarLibrary in project legend-pure by finos.

the class PureJarSerializer method writePureRepositoryJars.

private static void writePureRepositoryJars(Path outputDirectory, Path sourceDirectory, String platformVersion, RichIterable<CodeRepository> repositories, Log log) throws IOException {
    SetIterable<CodeRepository> repositoriesForCompilation = PureCodeStorage.getRepositoryDependencies(PureRepositoriesExternal.repositories(), repositories);
    PureRuntime runtime;
    RichIterable<CodeRepository> repositoriesToSerialize;
    if (null == sourceDirectory) {
        log.info("    *Building code storage leveraging the class loader (as sourceDirectory is not specified)");
        ClassLoaderCodeStorage cs = new ClassLoaderCodeStorage(repositoriesForCompilation);
        log.info("      " + cs.getRepositories().collect(CodeRepository::getName) + " - " + cs.getUserFiles().size() + " files");
        MutableCodeStorage codeStorage = new PureCodeStorage(null, cs);
        Message message = getMessage(log, "      ");
        log.info("    *Starting file compilation");
        runtime = new PureRuntimeBuilder(codeStorage).setTransactionalByDefault(false).buildAndInitialize(message);
        repositoriesToSerialize = repositoriesForCompilation;
        log.info("      -> Finished compilation");
    } else {
        log.info("    *Building code storage leveraging the sourceDirectory " + sourceDirectory);
        PartitionIterable<CodeRepository> partition = repositories.partition(r -> Files.exists(sourceDirectory.resolve(r.getName())));
        RichIterable<CodeRepository> repositoriesWithSource = partition.getSelected();
        RichIterable<CodeRepository> repositoriesWithoutSource = partition.getRejected();
        // Code Storages
        MutableList<AbstractRepositoryCodeStorage> codeStoragesFromSource = Lists.mutable.withAll(repositoriesWithSource.collect(r -> new MutableFSCodeStorage(r, sourceDirectory.resolve(r.getName()))));
        MutableList<AbstractRepositoryCodeStorage> allCodeStorage = Lists.mutable.withAll(codeStoragesFromSource).withAll(repositoriesWithoutSource.collect(EmptyCodeStorage::new));
        log.info("    *Loading the following repo from PARs: " + repositoriesWithoutSource.collect(CodeRepository::getName));
        // Build the runtime
        MutableCodeStorage codeStorage = new PureCodeStorage(sourceDirectory, allCodeStorage.toArray(new RepositoryCodeStorage[0]));
        runtime = new PureRuntimeBuilder(codeStorage).setTransactionalByDefault(false).build();
        // Load the PARS
        MutableList<String> namesOfRepoWithoutAvailableSource = repositoriesWithoutSource.collect(CodeRepository::getName).toList();
        Message message = getMessage(log, "    ");
        PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(GraphLoader.findJars(namesOfRepoWithoutAvailableSource, PureJarSerializer.class.getClassLoader(), message));
        final GraphLoader loader = new GraphLoader(runtime.getModelRepository(), runtime.getContext(), runtime.getIncrementalCompiler().getParserLibrary(), runtime.getIncrementalCompiler().getDslLibrary(), runtime.getSourceRegistry(), runtime.getURLPatternLibrary(), jarLibrary);
        loader.loadAll(message);
        // Compile Sources
        log.info("    *Starting file compilation");
        codeStoragesFromSource.forEach(r -> log.info("      " + r.getRepositories().collect(CodeRepository::getName) + " - " + r.getUserFiles().size() + " files (from: " + ((FSCodeStorage) r).getRoot() + ")"));
        for (CodeRepository repository : repositoriesWithSource) {
            Path path = sourceDirectory.resolve(repository.getName());
            log.info("      Compiling repository '" + repository.getName() + "' in " + path);
            MutableFSCodeStorage fs = new MutableFSCodeStorage(repository, path);
            log.info("        Found " + fs.getUserFiles().size() + " files");
            runtime.loadAndCompile(new PureCodeStorage(sourceDirectory, fs).getUserFiles());
            log.info("        -> Finished compiling repository '" + repository.getName() + "'");
        }
        repositoriesToSerialize = repositoriesWithSource;
        log.info("      -> Finished compilation");
    }
    Files.createDirectories(outputDirectory);
    log.info("    *Starting serialization");
    for (String repositoryName : repositoriesToSerialize.collect(CodeRepository::getName)) {
        Path outputFile = outputDirectory.resolve("pure-" + repositoryName + "." + ARCHIVE_FILE_EXTENSION);
        log.info("      Writing " + outputFile);
        try (OutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(outputFile))) {
            BinaryModelRepositorySerializer.serialize(outputStream, platformVersion, null, repositoryName, runtime);
        }
    }
}