Examples with CodeMap org.lflang.generator.CodeMap used on opensource projects

Example 1 with CodeMap

use of org.lflang.generator.CodeMap in project lingua-franca by lf-lang.

the class PythonValidator method getPossibleStrategies.

@Override
protected Collection<ValidationStrategy> getPossibleStrategies() {
    return List.of(new ValidationStrategy() {

        @Override
        public LFCommand getCommand(Path generatedFile) {
            return LFCommand.get("python3", List.of("-c", "import compileall; compileall.compile_dir('.', quiet=1)"), true, fileConfig.getSrcGenPkgPath());
        }

        @Override
        public Strategy getErrorReportingStrategy() {
            return (a, b, c) -> {
            };
        }

        @Override
        public Strategy getOutputReportingStrategy() {
            return (String validationOutput, ErrorReporter errorReporter, Map<Path, CodeMap> map) -> {
                String[] lines = (validationOutput + "\n\n\n").lines().toArray(String[]::new);
                for (int i = 0; i < lines.length - 3; i++) {
                    if (!tryReportTypical(lines, i)) {
                        tryReportAlternative(lines, i);
                    }
                }
            };
        }

        /**
         * Try to report a typical error message from the Python compiler.
         *
         * @param lines The lines of output from the compiler.
         * @param i     The current index at which a message may start. Guaranteed to be less
         *              than
         *              {@code lines.length - 3}.
         * @return Whether an error message was reported.
         */
        private boolean tryReportTypical(String[] lines, int i) {
            Matcher main = DIAGNOSTIC_MESSAGE_PATTERN.matcher(lines[i]);
            Matcher messageMatcher = MESSAGE.matcher(lines[i + 3]);
            String message = messageMatcher.matches() ? messageMatcher.group() : "Syntax Error";
            if (main.matches()) {
                int line = Integer.parseInt(main.group("line"));
                CodeMap map = codeMaps.get(fileConfig.getSrcGenPkgPath().resolve(Path.of(main.group("path"))).normalize());
                // Column is just a placeholder.
                Position genPosition = Position.fromOneBased(line, Integer.MAX_VALUE);
                if (map == null) {
                    // Undesirable fallback
                    errorReporter.report(null, DiagnosticSeverity.Error, message, 1);
                } else {
                    for (Path lfFile : map.lfSourcePaths()) {
                        Position lfPosition = map.adjusted(lfFile, genPosition);
                        // TODO: We could be more precise than just getting the right line, but the way the output
                        // is formatted (with leading whitespace possibly trimmed) does not make it easy.
                        errorReporter.report(lfFile, DiagnosticSeverity.Error, message, lfPosition.getOneBasedLine());
                    }
                }
                return true;
            }
            return false;
        }

        /**
         * Try to report an alternative error message from the Python compiler.
         *
         * @param lines The lines of output from the compiler.
         * @param i     The current index at which a message may start.
         */
        private void tryReportAlternative(String[] lines, int i) {
            Matcher main = ALT_DIAGNOSTIC_MESSAGE_PATTERN.matcher(lines[i]);
            if (main.matches()) {
                int line = Integer.parseInt(main.group("line"));
                Iterable<CodeMap> relevantMaps = codeMaps.keySet().stream().filter(p -> main.group().contains(p.getFileName().toString())).map(codeMaps::get)::iterator;
                for (CodeMap map : relevantMaps) {
                    // There should almost always be exactly one of these
                    for (Path lfFile : map.lfSourcePaths()) {
                        errorReporter.report(lfFile, DiagnosticSeverity.Error, main.group().replace("*** ", "").replace("Sorry: ", ""), map.adjusted(lfFile, Position.fromOneBased(line, 1)).getOneBasedLine());
                    }
                }
            }
        }

        @Override
        public boolean isFullBatch() {
            return true;
        }

        @Override
        public int getPriority() {
            return 0;
        }
    }, new ValidationStrategy() {

        @Override
        public LFCommand getCommand(Path generatedFile) {
            return LFCommand.get("pylint", List.of("--output-format=json", generatedFile.getFileName().toString()), true, fileConfig.getSrcGenPath());
        }

        @Override
        public Strategy getErrorReportingStrategy() {
            return (a, b, c) -> {
            };
        }

        @Override
        public Strategy getOutputReportingStrategy() {
            return (validationOutput, errorReporter, codeMaps) -> {
                if (validationOutput.isBlank())
                    return;
                try {
                    for (PylintMessage message : mapper.readValue(validationOutput, PylintMessage[].class)) {
                        if (shouldIgnore(message))
                            continue;
                        CodeMap map = codeMaps.get(message.getPath(fileConfig.getSrcGenPath()));
                        if (map != null) {
                            for (Path lfFile : map.lfSourcePaths()) {
                                Function<Position, Position> adjust = p -> map.adjusted(lfFile, p);
                                String humanMessage = DiagnosticReporting.messageOf(message.message, message.getPath(fileConfig.getSrcGenPath()), message.getStart());
                                Position lfStart = adjust.apply(message.getStart());
                                Position lfEnd = adjust.apply(message.getEnd());
                                bestEffortReport(errorReporter, adjust, lfStart, lfEnd, lfFile, message.getSeverity(), humanMessage);
                            }
                        }
                    }
                } catch (JsonProcessingException e) {
                    System.err.printf("Failed to parse \"%s\":%n", validationOutput);
                    e.printStackTrace();
                    errorReporter.reportWarning("Failed to parse linter output. The Lingua Franca code generator is tested with Pylint " + "version 2.12.2. Consider updating PyLint if you have an older version.");
                }
            };
        }

        /**
         * Return whether the given message should be ignored.
         * @param message A Pylint message that is a candidate to be reported.
         * @return whether {@code message} should be reported.
         */
        private boolean shouldIgnore(PylintMessage message) {
            // Code generation does not preserve whitespace, so this check is unreliable.
            if (message.symbol.equals("trailing-whitespace") || message.symbol.equals("line-too-long"))
                return true;
            // This filters out Pylint messages concerning missing members in types defined by protocol buffers.
            // FIXME: Make this unnecessary, perhaps using https://github.com/nelfin/pylint-protobuf.
            Matcher matcher = PylintNoNamePattern.matcher(message.message);
            return message.symbol.equals("no-member") && matcher.matches() && protoNames.contains(matcher.group("name"));
        }

        /**
         * Make a best-effort attempt to place the diagnostic on the correct line.
         */
        private void bestEffortReport(ErrorReporter errorReporter, Function<Position, Position> adjust, Position lfStart, Position lfEnd, Path file, DiagnosticSeverity severity, String humanMessage) {
            if (!lfEnd.equals(Position.ORIGIN) && !lfStart.equals(Position.ORIGIN)) {
                // Ideal case
                errorReporter.report(file, severity, humanMessage, lfStart, lfEnd);
            } else {
                // Fallback: Try to report on the correct line, or failing that, just line 1.
                if (lfStart.equals(Position.ORIGIN))
                    lfStart = adjust.apply(Position.fromZeroBased(lfStart.getZeroBasedLine(), Integer.MAX_VALUE));
                // FIXME: It might be better to improve style of generated code instead of quietly returning here.
                if (lfStart.equals(Position.ORIGIN) && severity != DiagnosticSeverity.Error)
                    return;
                errorReporter.report(file, severity, humanMessage, lfStart.getOneBasedLine());
            }
        }

        @Override
        public boolean isFullBatch() {
            return false;
        }

        @Override
        public int getPriority() {
            return 1;
        }
    });
}

Example 2 with CodeMap

use of org.lflang.generator.CodeMap in project lingua-franca by lf-lang.

the class PythonGenerator method doGenerate.

/**
 * Generate C code from the Lingua Franca model contained by the
 *  specified resource. This is the main entry point for code
 *  generation.
 *  @param resource The resource containing the source code.
 *  @param context Context relating to invocation of the code generator.
 */
@Override
public void doGenerate(Resource resource, LFGeneratorContext context) {
    // If there are federates, assign the number of threads in the CGenerator to 1
    if (isFederated) {
        targetConfig.threads = 1;
    }
    // Prevent the CGenerator from compiling the C code.
    // The PythonGenerator will compiler it.
    boolean compileStatus = targetConfig.noCompile;
    targetConfig.noCompile = true;
    // Force disable the CMake because
    targetConfig.useCmake = false;
    // it interferes with the Python target functionality
    int cGeneratedPercentProgress = (IntegratedBuilder.VALIDATED_PERCENT_PROGRESS + 100) / 2;
    super.doGenerate(resource, new SubContext(context, IntegratedBuilder.VALIDATED_PERCENT_PROGRESS, cGeneratedPercentProgress));
    SubContext compilingFederatesContext = new SubContext(context, cGeneratedPercentProgress, 100);
    targetConfig.noCompile = compileStatus;
    if (errorsOccurred()) {
        context.unsuccessfulFinish();
        return;
    }
    String baseFileName = topLevelName;
    // Keep a separate file config for each federate
    FileConfig oldFileConfig = fileConfig;
    var federateCount = 0;
    Map<Path, CodeMap> codeMaps = new HashMap<>();
    for (FederateInstance federate : federates) {
        federateCount++;
        if (isFederated) {
            topLevelName = baseFileName + '_' + federate.name;
            try {
                fileConfig = new FedFileConfig(fileConfig, federate.name);
            } catch (IOException e) {
                throw Exceptions.sneakyThrow(e);
            }
        }
        // Don't generate code if there is no main reactor
        if (this.main != null) {
            try {
                Map<Path, CodeMap> codeMapsForFederate = generatePythonFiles(federate);
                codeMaps.putAll(codeMapsForFederate);
                PyUtil.copyTargetFiles(fileConfig);
                if (!targetConfig.noCompile) {
                    compilingFederatesContext.reportProgress(String.format("Validating %d/%d sets of generated files...", federateCount, federates.size()), 100 * federateCount / federates.size());
                    // If there are no federates, compile and install the generated code
                    new PythonValidator(fileConfig, errorReporter, codeMaps, protoNames).doValidate(context);
                    if (!errorsOccurred() && !Objects.equal(context.getMode(), LFGeneratorContext.Mode.LSP_MEDIUM)) {
                        compilingFederatesContext.reportProgress(String.format("Validation complete. Compiling and installing %d/%d Python modules...", federateCount, federates.size()), 100 * federateCount / federates.size());
                        // Why is this invoked here if the current federate is not a parameter?
                        pythonCompileCode(context);
                    }
                } else {
                    System.out.println(PythonInfoGenerator.generateSetupInfo(fileConfig));
                }
            } catch (Exception e) {
                throw Exceptions.sneakyThrow(e);
            }
            if (!isFederated) {
                System.out.println(PythonInfoGenerator.generateRunInfo(fileConfig, topLevelName));
            }
        }
        fileConfig = oldFileConfig;
    }
    if (isFederated) {
        System.out.println(PythonInfoGenerator.generateFedRunInfo(fileConfig));
    }
    // Restore filename
    topLevelName = baseFileName;
    if (errorReporter.getErrorsOccurred()) {
        context.unsuccessfulFinish();
    } else if (!isFederated) {
        context.finish(GeneratorResult.Status.COMPILED, topLevelName + ".py", fileConfig.getSrcGenPath(), fileConfig, codeMaps, "python3");
    } else {
        context.finish(GeneratorResult.Status.COMPILED, fileConfig.name, fileConfig.binPath, fileConfig, codeMaps, "bash");
    }
}

Example 3 with CodeMap

use of org.lflang.generator.CodeMap in project lingua-franca by lf-lang.

the class PythonGenerator method generatePythonFiles.

/**
 * Generate the necessary Python files.
 * @param federate The federate instance
 */
public Map<Path, CodeMap> generatePythonFiles(FederateInstance federate) throws IOException {
    Path filePath = fileConfig.getSrcGenPath().resolve(topLevelName + ".py");
    File file = filePath.toFile();
    Files.deleteIfExists(filePath);
    // Create the necessary directories
    if (!file.getParentFile().exists()) {
        file.getParentFile().mkdirs();
    }
    Map<Path, CodeMap> codeMaps = new HashMap<>();
    codeMaps.put(filePath, CodeMap.fromGeneratedCode(generatePythonCode(federate).toString()));
    FileUtil.writeToFile(codeMaps.get(filePath).getGeneratedCode(), filePath);
    Path setupPath = fileConfig.getSrcGenPath().resolve("setup.py");
    // Handle Python setup
    System.out.println("Generating setup file to " + setupPath);
    Files.deleteIfExists(setupPath);
    // Create the setup file
    FileUtil.writeToFile(generatePythonSetupFile(), setupPath);
    return codeMaps;
}