Search in sources :

Example 1 with EngineJavaCompiler

use of org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler in project legend-engine by finos.

the class TestPlanExecutionWithGraphFetchCrossKeyCache method testCrossPropertyCachingWithSerializableCache.

@Test
public void testCrossPropertyCachingWithSerializableCache() throws JavaCompileException, IOException, ClassNotFoundException {
    String fetchFunction = "###Pure\n" + "function test::fetch(): String[1]\n" + "{\n" + "  test::Person.all()\n" + "    ->graphFetch(#{\n" + "      test::Person {\n" + "        fullName,\n" + "        firm {\n" + "          name,\n" + "          address {\n" + "            name\n" + "          }\n" + "        }\n" + "      }\n" + "    }#, 1)\n" + "    ->serialize(#{\n" + "      test::Person {\n" + "        fullName,\n" + "        firm {\n" + "          name,\n" + "          address {\n" + "            name\n" + "          }\n" + "        }\n" + "      }\n" + "    }#)\n" + "}";
    String expectedRes = "[" + "{\"fullName\":\"P1\",\"firm\":{\"name\":\"F1\",\"address\":{\"name\":\"A4\"}}}," + "{\"fullName\":\"P2\",\"firm\":{\"name\":\"F2\",\"address\":{\"name\":\"A3\"}}}," + "{\"fullName\":\"P3\",\"firm\":null}," + "{\"fullName\":\"P4\",\"firm\":null}," + "{\"fullName\":\"P5\",\"firm\":{\"name\":\"F1\",\"address\":{\"name\":\"A4\"}}}" + "]";
    SingleExecutionPlan plan = buildPlanForFetchFunction(fetchFunction);
    EngineJavaCompiler compiler = JavaHelper.compilePlan(plan, null);
    GraphFetchCrossAssociationKeys graphFetchCrossAssociationKeys = GraphFetchCrossAssociationKeys.graphFetchCrossAssociationKeysForPlan(plan).stream().filter(x -> x.getName().equals("<default, root.firm>")).findFirst().orElse(null);
    try (JavaHelper.ThreadContextClassLoaderScope ignored = JavaHelper.withCurrentThreadContextClassLoader(compiler.getClassLoader())) {
        Map<GraphFetchCacheKey, List<Object>> firmMap1 = Maps.mutable.empty();
        GraphFetchCacheByTargetCrossKeys firmCache1 = ExecutionCacheBuilder.buildGraphFetchCacheByTargetCrossKeysFromExecutionCache(buildExecutionCacheFromMap(firmMap1), graphFetchCrossAssociationKeys);
        Assert.assertEquals(expectedRes, executePlan(plan, new PlanExecutionContext(firmCache1)));
        assertCacheStats(firmCache1.getExecutionCache(), 3, 5, 2, 3);
        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
        ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
        objectOutputStream.writeObject(firmMap1);
        objectOutputStream.flush();
        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
        ClassLoaderObjectInputStream objectInputStream = new ClassLoaderObjectInputStream(compiler.getClassLoader(), byteArrayInputStream);
        Object o = objectInputStream.readObject();
        @SuppressWarnings("unchecked") Map<GraphFetchCacheKey, List<Object>> firmMap2 = (Map<GraphFetchCacheKey, List<Object>>) o;
        GraphFetchCacheByTargetCrossKeys firmCache2 = ExecutionCacheBuilder.buildGraphFetchCacheByTargetCrossKeysFromExecutionCache(buildExecutionCacheFromMap(firmMap2), graphFetchCrossAssociationKeys);
        Assert.assertEquals(expectedRes, executePlan(plan, new PlanExecutionContext(firmCache2)));
        assertCacheStats(firmCache2.getExecutionCache(), 3, 5, 5, 0);
    }
}
Also used : EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ClassLoaderObjectInputStream(org.apache.commons.io.input.ClassLoaderObjectInputStream) ObjectOutputStream(java.io.ObjectOutputStream) GraphFetchCrossAssociationKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) GraphFetchCacheByTargetCrossKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByTargetCrossKeys) ByteArrayInputStream(java.io.ByteArrayInputStream) List(java.util.List) SingleExecutionPlan(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan) PlanExecutionContext(org.finos.legend.engine.plan.execution.PlanExecutionContext) Map(java.util.Map) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) Test(org.junit.Test)

Example 2 with EngineJavaCompiler

use of org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler in project legend-engine by finos.

the class ExecutionNodeJavaPlatformHelper method getClassToExecute.

public static Class<?> getClassToExecute(ExecutionNode node, String _class, ExecutionState executionState, MutableList<CommonProfile> pm) {
    if (executionState.isJavaCompilationForbidden()) {
        try {
            return Thread.currentThread().getContextClassLoader().loadClass(_class);
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
    }
    JavaPlatformImplementation j = (JavaPlatformImplementation) node.implementation;
    List<JavaClass> javaClasses = FastList.newList();
    if (j.code != null) {
        JavaClass executeClass = JavaHelper.newJavaClass(JavaHelper.getExecutionClassFullName(j));
        executeClass.source = j.code;
        javaClasses.add(executeClass);
    }
    javaClasses.addAll(getLocalImplementationSupportClasses(node));
    if (javaClasses.isEmpty()) {
        ClassLoader classLoader = executionState.hasJavaCompiler() ? executionState.getJavaCompiler().getClassLoader() : Thread.currentThread().getContextClassLoader();
        try {
            return classLoader.loadClass(_class);
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
    }
    try {
        // TODO Confirm we can delete this
        MutableMap<String, String> compiledClassesWithByteCode = getCompiledClasses(node);
        List<StringJavaSource> classesToCompile = javaClasses.stream().filter(c -> !compiledClassesWithByteCode.containsKey(JavaHelper.getJavaClassFullName(c))).map(JavaHelper::buildStringJavaSource).collect(Collectors.toList());
        EngineJavaCompiler compiler = new EngineJavaCompiler(executionState.getJavaCompiler());
        long start = System.currentTimeMillis();
        try {
            if (!compiledClassesWithByteCode.isEmpty()) {
                compiler.load(compiledClassesWithByteCode);
            }
            if (!classesToCompile.isEmpty()) {
                LOGGER.info(new LogInfo(pm, LoggingEventType.JAVA_COMPILATION_START, "Node: " + node.getClass().getName()).toString());
                compiler.compile(classesToCompile);
                LOGGER.info(new LogInfo(pm, LoggingEventType.JAVA_COMPILATION_STOP, (double) System.currentTimeMillis() - start).toString());
            }
        } catch (Exception jce) {
            LOGGER.info(new LogInfo(pm, LoggingEventType.JAVA_COMPILATION_ERROR, new ErrorResult(1, jce).getMessage()).toString());
            throw jce;
        }
        return compiler.getClassLoader().loadClass(_class);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Also used : LogInfo(org.finos.legend.engine.shared.core.operational.logs.LogInfo) EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) InvocationTargetException(java.lang.reflect.InvocationTargetException) JavaClass(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaClass) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) StringJavaSource(org.finos.legend.engine.shared.javaCompiler.StringJavaSource)

Example 3 with EngineJavaCompiler

use of org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler in project legend-engine by finos.

the class JavaHelper method compilePlanFast.

private static EngineJavaCompiler compilePlanFast(SingleExecutionPlan singleExecutionPlan) throws JavaCompileException, IOException, CompileException {
    Map<JavaPlatformImplementation, List<JavaClass>> javaClassesMap = Maps.mutable.empty();
    if (singleExecutionPlan.globalImplementationSupport != null) {
        JavaPlatformImplementation platformImplementation = (JavaPlatformImplementation) singleExecutionPlan.globalImplementationSupport;
        List<JavaClass> globalImplementationSupportClasses = collectJavaClasses(platformImplementation);
        if (!globalImplementationSupportClasses.isEmpty()) {
            javaClassesMap.put(platformImplementation, globalImplementationSupportClasses);
        }
    }
    collectJavaClasses(singleExecutionPlan.rootExecutionNode, javaClassesMap);
    if (javaClassesMap.isEmpty()) {
        return null;
    }
    EngineJavaCompiler javaCompiler = createNewJavaCompiler();
    Map<JavaClass, JavaPlatformImplementation> reverseClassMap = Maps.mutable.empty();
    List<JavaClass> executeClasses = new ArrayList<>();
    List<JavaClass> nonExecuteClasses = new ArrayList<>();
    javaClassesMap.forEach((jimpl, jclasses) -> jclasses.forEach(jclass -> {
        reverseClassMap.put(jclass, jimpl);
        (javaClassHasFullName(jclass, jimpl.executionClassFullName) ? executeClasses : nonExecuteClasses).add(jclass);
    }));
    Map<String, JavaClass> classMap = nonExecuteClasses.stream().collect(Collectors.toMap(JavaHelper::getJavaClassFullName, x -> x, (x, y) -> {
        throw new RuntimeException("Conflicting classes compilation in fast mode not supported!");
    }));
    Map<String, String> classToBytecodeMap = compileJavaClasses(nonExecuteClasses, javaCompiler);
    classToBytecodeMap.forEach((name, bytecode) -> {
        JavaClass _class = classMap.get(name);
        if (_class == null) {
            // Handling Inner Classes only
            int firstDollar = name.indexOf('$');
            if (firstDollar == -1) {
                throw new RuntimeException("Non Inner classes are not supported yet!");
            }
            JavaClass topClass = classMap.get(name.substring(0, firstDollar));
            JavaPlatformImplementation impl = reverseClassMap.get(topClass);
            _class = createGeneratedJavaClass(name);
            if (impl.classes == null) {
                impl.classes = FastList.newListWith(_class);
            } else {
                impl.classes.add(_class);
            }
        }
        _class.byteCode = bytecode;
    });
    ClassLoader globalClassLoader = javaCompiler.getClassLoader();
    for (JavaClass executeClass : executeClasses) {
        if (executeClass.byteCode == null) {
            Map<String, byte[]> classes = SingleFileCompiler.compileFile(buildStringJavaSource(executeClass), globalClassLoader);
            executeClass.byteCode = Base64.getEncoder().encodeToString(classes.get(getJavaClassFullName(executeClass)));
        }
    }
    return javaCompiler;
}
Also used : JavaClass(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaClass) LoggingEventType(org.finos.legend.engine.shared.core.operational.logs.LoggingEventType) SingleFileCompiler(org.finos.legend.engine.shared.javaCompiler.SingleFileCompiler) CommonProfile(org.pac4j.core.profile.CommonProfile) ExecutionPlanDependenciesFilter(org.finos.legend.engine.plan.compilation.ExecutionPlanDependenciesFilter) ClassPathFilter(org.finos.legend.engine.shared.javaCompiler.ClassPathFilter) CompositeClassPathFilter(org.finos.legend.engine.shared.javaCompiler.CompositeClassPathFilter) MutableList(org.eclipse.collections.api.list.MutableList) FastList(org.eclipse.collections.impl.list.mutable.FastList) ArrayList(java.util.ArrayList) Maps(org.eclipse.collections.api.factory.Maps) CompileException(org.codehaus.commons.compiler.CompileException) Map(java.util.Map) ExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) UnifiedMap(org.eclipse.collections.impl.map.mutable.UnifiedMap) Logger(org.slf4j.Logger) EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) SingleExecutionPlan(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) LogInfo(org.finos.legend.engine.shared.core.operational.logs.LogInfo) JavaVersion(org.finos.legend.engine.shared.javaCompiler.JavaVersion) Base64(java.util.Base64) List(java.util.List) JavaCompileException(org.finos.legend.engine.shared.javaCompiler.JavaCompileException) Lists(org.eclipse.collections.impl.factory.Lists) StringJavaSource(org.finos.legend.engine.shared.javaCompiler.StringJavaSource) CompiledClass(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.CompiledClass) Collections(java.util.Collections) EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) ArrayList(java.util.ArrayList) JavaClass(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaClass) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) MutableList(org.eclipse.collections.api.list.MutableList) FastList(org.eclipse.collections.impl.list.mutable.FastList) ArrayList(java.util.ArrayList) List(java.util.List)

Example 4 with EngineJavaCompiler

use of org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler in project legend-engine by finos.

the class JavaHelper method compilePlanSlow.

private static EngineJavaCompiler compilePlanSlow(SingleExecutionPlan singleExecutionPlan) throws JavaCompileException {
    EngineJavaCompiler javaCompiler = createNewJavaCompiler();
    if (singleExecutionPlan.globalImplementationSupport != null) {
        JavaPlatformImplementation platformImplementation = (JavaPlatformImplementation) singleExecutionPlan.globalImplementationSupport;
        compileImplementation(platformImplementation, javaCompiler);
    }
    compileNode(singleExecutionPlan.rootExecutionNode, javaCompiler);
    return javaCompiler;
}
Also used : EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation)

Example 5 with EngineJavaCompiler

use of org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler in project legend-engine by finos.

the class JavaHelper method compileNode.

private static void compileNode(ExecutionNode executionNode, EngineJavaCompiler globalJavaCompiler) throws JavaCompileException {
    if (executionNode.implementation != null) {
        JavaPlatformImplementation platformImplementation = (JavaPlatformImplementation) executionNode.implementation;
        if (platformImplementation.classes != null && !platformImplementation.classes.isEmpty()) {
            EngineJavaCompiler javaCompiler = new EngineJavaCompiler(globalJavaCompiler);
            JavaHelper.compileImplementation(platformImplementation, javaCompiler);
        }
    }
    for (ExecutionNode childNode : executionNode.executionNodes()) {
        compileNode(childNode, globalJavaCompiler);
    }
}
Also used : EngineJavaCompiler(org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) ExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode)

Aggregations

EngineJavaCompiler (org.finos.legend.engine.shared.javaCompiler.EngineJavaCompiler)20 Test (org.junit.Test)11 ClassPathFilter (org.finos.legend.engine.shared.javaCompiler.ClassPathFilter)8 JavaPlatformImplementation (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation)5 JavaVersion (org.finos.legend.engine.shared.javaCompiler.JavaVersion)5 ErrorResult (org.finos.legend.engine.plan.execution.result.ErrorResult)4 LogInfo (org.finos.legend.engine.shared.core.operational.logs.LogInfo)4 IOException (java.io.IOException)3 List (java.util.List)3 Map (java.util.Map)3 CompileException (org.codehaus.commons.compiler.CompileException)3 ExecutionPlanDependenciesFilter (org.finos.legend.engine.plan.compilation.ExecutionPlanDependenciesFilter)3 SingleExecutionPlan (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan)3 ExecutionNode (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode)3 JavaClass (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaClass)3 CompositeClassPathFilter (org.finos.legend.engine.shared.javaCompiler.CompositeClassPathFilter)3 JavaCompileException (org.finos.legend.engine.shared.javaCompiler.JavaCompileException)3 StringJavaSource (org.finos.legend.engine.shared.javaCompiler.StringJavaSource)3 ArrayList (java.util.ArrayList)2 Base64 (java.util.Base64)2