Examples with Obj org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj used on opensource projects

Example 1 with Obj

use of org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj in project legend-pure by finos.

the class TestDistributedBinaryGraphSerialization method getExpectedObjsFromRuntime.

private ListIterable<Obj> getExpectedObjsFromRuntime(String metadataName) {
    MutableSet<CoreInstance> ignoredClassifiers = PrimitiveUtilities.getPrimitiveTypes(repository).toSet();
    ArrayAdapter.adapt(M3Paths.EnumStub, M3Paths.ImportStub, M3Paths.PropertyStub, M3Paths.RouteNodePropertyStub).collect(processorSupport::package_getByUserPath, ignoredClassifiers);
    IdBuilder idBuilder = IdBuilder.newIdBuilder(DistributedMetadataHelper.getMetadataIdPrefix(metadataName), processorSupport);
    GraphSerializer.ClassifierCaches classifierCaches = new GraphSerializer.ClassifierCaches(processorSupport);
    return GraphNodeIterable.fromModelRepository(repository).reject(i -> ignoredClassifiers.contains(i.getClassifier())).collect(i -> GraphSerializer.buildObj(i, idBuilder, classifierCaches, processorSupport), Lists.mutable.empty());
}

Example 2 with Obj

use of org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj in project legend-pure by finos.

the class GraphSerializer method buildObj.

public static Obj buildObj(CoreInstance instance, IdBuilder idBuilder, ClassifierCaches classifierCaches, ProcessorSupport processorSupport) {
    SourceInformation sourceInformation = instance.getSourceInformation();
    String identifier = idBuilder.buildId(instance);
    String classifierString = classifierCaches.getClassifierId(instance.getClassifier());
    ListIterable<PropertyValue> propertyValues = collectProperties(instance, idBuilder, classifierCaches, processorSupport);
    return classifierCaches.isEnum(instance) ? new Enum(sourceInformation, identifier, classifierString, instance.getName(), propertyValues) : new Obj(sourceInformation, identifier, classifierString, instance.getName(), propertyValues);
}

Example 3 with Obj

use of org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj in project legend-pure by finos.

the class DistributedBinaryGraphDeserializer method getInstances.

public ListIterable<Obj> getInstances(String classifierId, Iterable<String> instanceIds) {
    if (!hasClassifier(classifierId)) {
        throw new RuntimeException("Unknown classifier id: '" + classifierId + "'");
    }
    ClassifierIndex classifierIndex = getClassifierIndex(classifierId);
    MutableMap<String, MutableList<SourceCoordinates>> sourceCoordinatesByFile = Maps.mutable.empty();
    int size = 0;
    for (String instanceId : instanceIds) {
        SourceCoordinates sourceCoordinates = classifierIndex.getSourceCoordinates(instanceId);
        if (sourceCoordinates == null) {
            throw new RuntimeException("Unknown instance: classifier='" + classifierId + "', id='" + instanceId + "'");
        }
        sourceCoordinatesByFile.getIfAbsentPut(sourceCoordinates.getFilePath(), Lists.mutable::empty).add(sourceCoordinates);
        size++;
    }
    if (size == 0) {
        return Lists.immutable.empty();
    }
    MutableList<Obj> objs = Lists.mutable.withInitialCapacity(size);
    sourceCoordinatesByFile.forEachKeyValue((filePath, fileSourceCoordinates) -> {
        fileSourceCoordinates.sortThis(SourceCoordinates::compareByOffset);
        try (Reader reader = this.fileReader.getReader(filePath)) {
            int offset = 0;
            for (SourceCoordinates sourceCoordinates : fileSourceCoordinates) {
                objs.add(sourceCoordinates.getObj(reader, offset, this.stringIndex, classifierIndex));
                offset = sourceCoordinates.getOffsetAfterReading();
            }
        }
    });
    return objs;
}

Example 4 with Obj

use of org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj in project legend-pure by finos.

the class DistributedMetadataTool method instance.

private void instance(String classifier, String id) {
    if (this.deserializer.hasInstance(classifier, id)) {
        Obj instance = this.deserializer.getInstance(classifier, id);
        new ObjPrinter(instance).printObj();
    } else {
        this.out.println("Instance not found");
    }
}

Example 5 with Obj

use of org.finos.legend.pure.runtime.java.compiled.serialization.model.Obj in project legend-pure by finos.

the class DistributedBinaryGraphSerializer method serialize.

public void serialize(FileWriter fileWriter) {
    MutableMap<String, MutableList<CoreInstance>> nodesByClassifierId = getNodesByClassifierId();
    // Build string cache
    DistributedStringCache stringCache = DistributedStringCache.fromNodes(nodesByClassifierId.valuesView().flatCollect(l -> l), this.idBuilder, this.processorSupport);
    BinaryObjSerializer serializer = new BinaryObjSerializerWithStringCacheAndImplicitIdentifiers(stringCache);
    // Write string cache
    stringCache.write(this.metadataName, fileWriter);
    // Write instances
    int partition = 0;
    int partitionTotalBytes = 0;
    ByteArrayOutputStream binByteStream = new ByteArrayOutputStream(MAX_BIN_FILE_BYTES);
    try (Writer binFileWriter = BinaryWriters.newBinaryWriter(binByteStream)) {
        ByteArrayOutputStream indexByteStream = new ByteArrayOutputStream();
        try (Writer indexWriter = BinaryWriters.newBinaryWriter(indexByteStream)) {
            for (String classifierId : nodesByClassifierId.keysView().toSortedList()) {
                ListIterable<CoreInstance> classifierObjs = nodesByClassifierId.get(classifierId).sortThisBy(this.idBuilder::buildId);
                // Initial index information
                // total obj count
                indexWriter.writeInt(classifierObjs.size());
                // initial partition
                indexWriter.writeInt(partition);
                // initial byte offset in partition
                indexWriter.writeInt(partitionTotalBytes);
                MutableList<ObjIndexInfo> partitionObjIndexInfos = Lists.mutable.empty();
                ByteArrayOutputStream objByteStream = new ByteArrayOutputStream();
                try (Writer objWriter = BinaryWriters.newBinaryWriter(objByteStream)) {
                    GraphSerializer.ClassifierCaches classifierCaches = new GraphSerializer.ClassifierCaches(this.processorSupport);
                    for (CoreInstance coreInstance : classifierObjs) {
                        // Obj serialization
                        Obj obj = GraphSerializer.buildObj(coreInstance, this.idBuilder, classifierCaches, this.processorSupport);
                        objByteStream.reset();
                        serializer.serializeObj(objWriter, obj);
                        int objByteCount = objByteStream.size();
                        if (partitionTotalBytes + objByteCount > MAX_BIN_FILE_BYTES) {
                            // Write current partition
                            try (Writer writer1 = fileWriter.getWriter(DistributedMetadataHelper.getMetadataPartitionBinFilePath(this.metadataName, partition))) {
                                writer1.writeBytes(binByteStream.toByteArray());
                                binByteStream.reset();
                            }
                            // Write partition portion of classifier index
                            indexWriter.writeInt(partitionObjIndexInfos.size());
                            partitionObjIndexInfos.forEach(info -> info.write(indexWriter, stringCache));
                            // New partition
                            partition++;
                            if (partition < 0) {
                                throw new RuntimeException("Too many partitions");
                            }
                            partitionTotalBytes = 0;
                            partitionObjIndexInfos.clear();
                        }
                        binFileWriter.writeBytes(objByteStream.toByteArray());
                        partitionTotalBytes += objByteCount;
                        partitionObjIndexInfos.add(new ObjIndexInfo(obj.getIdentifier(), objByteCount));
                    }
                }
                // Write final partition portion of classifier index
                if (partitionObjIndexInfos.notEmpty()) {
                    indexWriter.writeInt(partitionObjIndexInfos.size());
                    partitionObjIndexInfos.forEach(info -> info.write(indexWriter, stringCache));
                }
                // Write classifier index
                try (Writer writer1 = fileWriter.getWriter(DistributedMetadataHelper.getMetadataClassifierIndexFilePath(this.metadataName, classifierId))) {
                    writer1.writeBytes(indexByteStream.toByteArray());
                    indexByteStream.reset();
                }
            }
        }
    }
    // Write final partition
    if (binByteStream.size() > 0) {
        try (Writer writer = fileWriter.getWriter(DistributedMetadataHelper.getMetadataPartitionBinFilePath(this.metadataName, partition))) {
            writer.writeBytes(binByteStream.toByteArray());
        }
    }
}