Examples with ChangedFileReportTask com.linkedin.pegasus.gradle.tasks.ChangedFileReportTask used on opensource projects

Example 1 with ChangedFileReportTask

use of com.linkedin.pegasus.gradle.tasks.ChangedFileReportTask in project rest.li by linkedin.

the class PegasusPlugin method configureRestModelGeneration.

protected void configureRestModelGeneration(Project project, SourceSet sourceSet) {
    if (sourceSet.getAllSource().isEmpty()) {
        project.getLogger().info("No source files found for sourceSet {}.  Skipping idl generation.", sourceSet.getName());
        return;
    }
    // afterEvaluate needed so that api project can be overridden via ext.apiProject
    project.afterEvaluate(p -> {
        // find api project here instead of in each project's plugin configuration
        // this allows api project relation options (ext.api*) to be specified anywhere in the build.gradle file
        // alternatively, pass closures to task configuration, and evaluate the closures when task is executed
        Project apiProject = getCheckedApiProject(project);
        // make sure the api project is evaluated. Important for configure-on-demand mode.
        if (apiProject != null) {
            project.evaluationDependsOn(apiProject.getPath());
            if (!apiProject.getPlugins().hasPlugin(_thisPluginType)) {
                apiProject = null;
            }
        }
        if (apiProject == null) {
            return;
        }
        Task untypedJarTask = project.getTasks().findByName(sourceSet.getJarTaskName());
        if (!(untypedJarTask instanceof Jar)) {
            return;
        }
        Jar jarTask = (Jar) untypedJarTask;
        String snapshotCompatPropertyName = findProperty(FileCompatibilityType.SNAPSHOT);
        if (project.hasProperty(snapshotCompatPropertyName) && "off".equalsIgnoreCase((String) project.property(snapshotCompatPropertyName))) {
            project.getLogger().lifecycle("Project {} snapshot compatibility level \"OFF\" is deprecated. Default to \"IGNORE\".", project.getPath());
        }
        // generate the rest model
        FileCollection restModelCodegenClasspath = project.getConfigurations().getByName(PEGASUS_PLUGIN_CONFIGURATION).plus(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)).plus(sourceSet.getRuntimeClasspath());
        String destinationDirPrefix = getGeneratedDirPath(project, sourceSet, REST_GEN_TYPE) + File.separatorChar;
        FileCollection restModelResolverPath = apiProject.files(getDataSchemaPath(project, sourceSet)).plus(getDataModelConfig(apiProject, sourceSet));
        Set<File> watchedRestModelInputDirs = buildWatchedRestModelInputDirs(project, sourceSet);
        Set<File> restModelInputDirs = difference(sourceSet.getAllSource().getSrcDirs(), sourceSet.getResources().getSrcDirs());
        Task generateRestModelTask = project.getTasks().create(sourceSet.getTaskName("generate", "restModel"), GenerateRestModelTask.class, task -> {
            task.dependsOn(project.getTasks().getByName(sourceSet.getClassesTaskName()));
            task.setCodegenClasspath(restModelCodegenClasspath);
            task.setWatchedCodegenClasspath(restModelCodegenClasspath.filter(file -> !"main".equals(file.getName()) && !"classes".equals(file.getName())));
            task.setInputDirs(restModelInputDirs);
            task.setWatchedInputDirs(watchedRestModelInputDirs.isEmpty() ? restModelInputDirs : watchedRestModelInputDirs);
            // we need all the artifacts from runtime for any private implementation classes the server code might need.
            task.setSnapshotDestinationDir(project.file(destinationDirPrefix + "snapshot"));
            task.setIdlDestinationDir(project.file(destinationDirPrefix + "idl"));
            @SuppressWarnings("unchecked") Map<String, PegasusOptions> pegasusOptions = (Map<String, PegasusOptions>) project.getExtensions().getExtraProperties().get("pegasus");
            task.setIdlOptions(pegasusOptions.get(sourceSet.getName()).idlOptions);
            task.setResolverPath(restModelResolverPath);
            if (isPropertyTrue(project, ENABLE_ARG_FILE)) {
                task.setEnableArgFile(true);
            }
            task.onlyIf(t -> !isPropertyTrue(project, SKIP_GENERATE_REST_MODEL));
            task.doFirst(new CacheableAction<>(t -> deleteGeneratedDir(project, sourceSet, REST_GEN_TYPE)));
        });
        File apiSnapshotDir = apiProject.file(getSnapshotPath(apiProject, sourceSet));
        File apiIdlDir = apiProject.file(getIdlPath(apiProject, sourceSet));
        apiSnapshotDir.mkdirs();
        if (!isPropertyTrue(project, SKIP_IDL_CHECK)) {
            apiIdlDir.mkdirs();
        }
        CheckRestModelTask checkRestModelTask = project.getTasks().create(sourceSet.getTaskName("check", "RestModel"), CheckRestModelTask.class, task -> {
            task.dependsOn(generateRestModelTask);
            task.setCurrentSnapshotFiles(SharedFileUtils.getSnapshotFiles(project, destinationDirPrefix));
            task.setPreviousSnapshotDirectory(apiSnapshotDir);
            task.setCurrentIdlFiles(SharedFileUtils.getIdlFiles(project, destinationDirPrefix));
            task.setPreviousIdlDirectory(apiIdlDir);
            task.setCodegenClasspath(project.getConfigurations().getByName(PEGASUS_PLUGIN_CONFIGURATION));
            task.setModelCompatLevel(PropertyUtil.findCompatLevel(project, FileCompatibilityType.SNAPSHOT));
            task.onlyIf(t -> !isPropertyTrue(project, SKIP_IDL_CHECK));
            task.doLast(new CacheableAction<>(t -> {
                if (!task.isEquivalent()) {
                    _restModelCompatMessage.append(task.getWholeMessage());
                }
            }));
        });
        CheckSnapshotTask checkSnapshotTask = project.getTasks().create(sourceSet.getTaskName("check", "Snapshot"), CheckSnapshotTask.class, task -> {
            task.dependsOn(generateRestModelTask);
            task.setCurrentSnapshotFiles(SharedFileUtils.getSnapshotFiles(project, destinationDirPrefix));
            task.setPreviousSnapshotDirectory(apiSnapshotDir);
            task.setCodegenClasspath(project.getConfigurations().getByName(PEGASUS_PLUGIN_CONFIGURATION));
            task.setSnapshotCompatLevel(PropertyUtil.findCompatLevel(project, FileCompatibilityType.SNAPSHOT));
            task.onlyIf(t -> isPropertyTrue(project, SKIP_IDL_CHECK));
        });
        CheckIdlTask checkIdlTask = project.getTasks().create(sourceSet.getTaskName("check", "Idl"), CheckIdlTask.class, task -> {
            task.dependsOn(generateRestModelTask);
            task.setCurrentIdlFiles(SharedFileUtils.getIdlFiles(project, destinationDirPrefix));
            task.setPreviousIdlDirectory(apiIdlDir);
            task.setResolverPath(restModelResolverPath);
            task.setCodegenClasspath(project.getConfigurations().getByName(PEGASUS_PLUGIN_CONFIGURATION));
            task.setIdlCompatLevel(PropertyUtil.findCompatLevel(project, FileCompatibilityType.IDL));
            if (isPropertyTrue(project, ENABLE_ARG_FILE)) {
                task.setEnableArgFile(true);
            }
            task.onlyIf(t -> !isPropertyTrue(project, SKIP_IDL_CHECK) && !"OFF".equals(PropertyUtil.findCompatLevel(project, FileCompatibilityType.IDL)));
        });
        // rest model publishing involves cross-project reference
        // configure after all projects have been evaluated
        // the file copy can be turned off by "rest.model.noPublish" flag
        Task publishRestliSnapshotTask = project.getTasks().create(sourceSet.getTaskName("publish", "RestliSnapshot"), PublishRestModelTask.class, task -> {
            task.dependsOn(checkRestModelTask, checkSnapshotTask, checkIdlTask);
            task.from(SharedFileUtils.getSnapshotFiles(project, destinationDirPrefix));
            task.into(apiSnapshotDir);
            task.setSuffix(SNAPSHOT_FILE_SUFFIX);
            task.onlyIf(t -> !isPropertyTrue(project, SNAPSHOT_NO_PUBLISH) && ((isPropertyTrue(project, SKIP_IDL_CHECK) && isTaskSuccessful(checkSnapshotTask) && checkSnapshotTask.getSummaryTarget().exists() && !isResultEquivalent(checkSnapshotTask.getSummaryTarget())) || (!isPropertyTrue(project, SKIP_IDL_CHECK) && isTaskSuccessful(checkRestModelTask) && checkRestModelTask.getSummaryTarget().exists() && !isResultEquivalent(checkRestModelTask.getSummaryTarget()))));
        });
        Task publishRestliIdlTask = project.getTasks().create(sourceSet.getTaskName("publish", "RestliIdl"), PublishRestModelTask.class, task -> {
            task.dependsOn(checkRestModelTask, checkIdlTask, checkSnapshotTask);
            task.from(SharedFileUtils.getIdlFiles(project, destinationDirPrefix));
            task.into(apiIdlDir);
            task.setSuffix(IDL_FILE_SUFFIX);
            task.onlyIf(t -> !isPropertyTrue(project, IDL_NO_PUBLISH) && ((isPropertyTrue(project, SKIP_IDL_CHECK) && isTaskSuccessful(checkSnapshotTask) && checkSnapshotTask.getSummaryTarget().exists() && !isResultEquivalent(checkSnapshotTask.getSummaryTarget(), true)) || (!isPropertyTrue(project, SKIP_IDL_CHECK) && ((isTaskSuccessful(checkRestModelTask) && checkRestModelTask.getSummaryTarget().exists() && !isResultEquivalent(checkRestModelTask.getSummaryTarget(), true)) || (isTaskSuccessful(checkIdlTask) && checkIdlTask.getSummaryTarget().exists() && !isResultEquivalent(checkIdlTask.getSummaryTarget()))))));
        });
        project.getLogger().info("API project selected for {} is {}", publishRestliIdlTask.getPath(), apiProject.getPath());
        jarTask.from(SharedFileUtils.getIdlFiles(project, destinationDirPrefix));
        // add generated .restspec.json files as resources to the jar
        jarTask.dependsOn(publishRestliSnapshotTask, publishRestliIdlTask);
        ChangedFileReportTask changedFileReportTask = (ChangedFileReportTask) project.getTasks().getByName("changedFilesReport");
        // Use the files from apiDir for generating the changed files report as we need to notify user only when
        // source system files are modified.
        changedFileReportTask.setIdlFiles(SharedFileUtils.getSuffixedFiles(project, apiIdlDir, IDL_FILE_SUFFIX));
        changedFileReportTask.setSnapshotFiles(SharedFileUtils.getSuffixedFiles(project, apiSnapshotDir, SNAPSHOT_FILE_SUFFIX));
        changedFileReportTask.mustRunAfter(publishRestliSnapshotTask, publishRestliIdlTask);
        changedFileReportTask.doLast(new CacheableAction<>(t -> {
            if (!changedFileReportTask.getNeedCheckinFiles().isEmpty()) {
                project.getLogger().info("Adding modified files to need checkin list...");
                _needCheckinFiles.addAll(changedFileReportTask.getNeedCheckinFiles());
                _needBuildFolders.add(getCheckedApiProject(project).getPath());
            }
        }));
    });
}

Example 2 with ChangedFileReportTask

use of com.linkedin.pegasus.gradle.tasks.ChangedFileReportTask in project rest.li by linkedin.

the class PegasusPlugin method apply.

@Override
public void apply(Project project) {
    checkGradleVersion(project);
    project.getPlugins().apply(JavaPlugin.class);
    // this HashMap will have a PegasusOptions per sourceSet
    project.getExtensions().getExtraProperties().set("pegasus", new HashMap<>());
    // this map will extract PegasusOptions.GenerationMode to project property
    project.getExtensions().getExtraProperties().set("PegasusGenerationMode", Arrays.stream(PegasusOptions.GenerationMode.values()).collect(Collectors.toMap(PegasusOptions.GenerationMode::name, Function.identity())));
    synchronized (STATIC_PROJECT_EVALUATED_LOCK) {
        // multiple sub-projects applied the plugin.
        if (!project.getRootProject().hasProperty(RUN_ONCE) || !Boolean.parseBoolean(String.valueOf(project.getRootProject().property(RUN_ONCE)))) {
            project.getGradle().projectsEvaluated(gradle -> gradle.getRootProject().subprojects(subproject -> UNUSED_CONFIGURATIONS.forEach(configurationName -> {
                Configuration conf = subproject.getConfigurations().findByName(configurationName);
                if (conf != null && !conf.getDependencies().isEmpty()) {
                    subproject.getLogger().warn("*** Project {} declares dependency to unused configuration \"{}\". " + "This configuration is deprecated and you can safely remove the dependency. ***", subproject.getPath(), configurationName);
                }
            })));
            // Re-initialize the static variables as they might have stale values from previous run. With Gradle 3.0 and
            // gradle daemon enabled, the plugin class might not be loaded for every run.
            DATA_TEMPLATE_FILE_SUFFIXES.clear();
            DATA_TEMPLATE_FILE_SUFFIXES.add(DATA_TEMPLATE_FILE_SUFFIX);
            DATA_TEMPLATE_FILE_SUFFIXES.add(PDL_FILE_SUFFIX);
            _restModelCompatMessage = new StringBuffer();
            _needCheckinFiles.clear();
            _needBuildFolders.clear();
            _possibleMissingFilesInEarlierCommit.clear();
            project.getGradle().buildFinished(result -> {
                StringBuilder endOfBuildMessage = new StringBuilder();
                if (_restModelCompatMessage.length() > 0) {
                    endOfBuildMessage.append(_restModelCompatMessage);
                }
                if (!_needCheckinFiles.isEmpty()) {
                    endOfBuildMessage.append(createModifiedFilesMessage(_needCheckinFiles, _needBuildFolders));
                }
                if (!_possibleMissingFilesInEarlierCommit.isEmpty()) {
                    endOfBuildMessage.append(createPossibleMissingFilesMessage(_possibleMissingFilesInEarlierCommit));
                }
                if (endOfBuildMessage.length() > 0) {
                    result.getGradle().getRootProject().getLogger().quiet(endOfBuildMessage.toString());
                }
            });
            // Set an extra property on the root project to indicate the initialization is complete for the current build.
            project.getRootProject().getExtensions().getExtraProperties().set(RUN_ONCE, true);
        }
    }
    ConfigurationContainer configurations = project.getConfigurations();
    // configuration for getting the required classes to make pegasus call main methods
    configurations.maybeCreate(PEGASUS_PLUGIN_CONFIGURATION);
    // configuration for compiling generated data templates
    Configuration dataTemplateCompile = configurations.maybeCreate("dataTemplateCompile");
    dataTemplateCompile.setVisible(false);
    // configuration for running rest client generator
    Configuration restClientCompile = configurations.maybeCreate("restClientCompile");
    restClientCompile.setVisible(false);
    // configuration for running data template generator
    // DEPRECATED! This configuration is no longer used. Please stop using it.
    Configuration dataTemplateGenerator = configurations.maybeCreate("dataTemplateGenerator");
    dataTemplateGenerator.setVisible(false);
    // configuration for running rest client generator
    // DEPRECATED! This configuration is no longer used. Please stop using it.
    Configuration restTools = configurations.maybeCreate("restTools");
    restTools.setVisible(false);
    // configuration for running Avro schema generator
    // DEPRECATED! To skip avro schema generation, use PegasusOptions.generationModes
    Configuration avroSchemaGenerator = configurations.maybeCreate("avroSchemaGenerator");
    avroSchemaGenerator.setVisible(false);
    // configuration for depending on data schemas and potentially generated data templates
    // and for publishing jars containing data schemas to the project artifacts for including in the ivy.xml
    Configuration dataModel = configurations.maybeCreate("dataModel");
    Configuration testDataModel = configurations.maybeCreate("testDataModel");
    testDataModel.extendsFrom(dataModel);
    // configuration for depending on data schemas and potentially generated data templates
    // and for publishing jars containing data schemas to the project artifacts for including in the ivy.xml
    Configuration avroSchema = configurations.maybeCreate("avroSchema");
    Configuration testAvroSchema = configurations.maybeCreate("testAvroSchema");
    testAvroSchema.extendsFrom(avroSchema);
    // configuration for depending on rest idl and potentially generated client builders
    // and for publishing jars containing rest idl to the project artifacts for including in the ivy.xml
    Configuration restModel = configurations.maybeCreate("restModel");
    Configuration testRestModel = configurations.maybeCreate("testRestModel");
    testRestModel.extendsFrom(restModel);
    // configuration for publishing jars containing data schemas and generated data templates
    // to the project artifacts for including in the ivy.xml
    // 
    // published data template jars depends on the configurations used to compile the classes
    // in the jar, this includes the data models/templates used by the data template generator
    // and the classes used to compile the generated classes.
    Configuration dataTemplate = configurations.maybeCreate("dataTemplate");
    dataTemplate.extendsFrom(dataTemplateCompile, dataModel);
    Configuration testDataTemplate = configurations.maybeCreate("testDataTemplate");
    testDataTemplate.extendsFrom(dataTemplate, testDataModel);
    // configuration for processing and validating schema annotation during build time.
    // 
    // The configuration contains dependencies to schema annotation handlers which would process schema annotations
    // and validate.
    Configuration schemaAnnotationHandler = configurations.maybeCreate(SCHEMA_ANNOTATION_HANDLER_CONFIGURATION);
    // configuration for publishing jars containing rest idl and generated client builders
    // to the project artifacts for including in the ivy.xml
    // 
    // published client builder jars depends on the configurations used to compile the classes
    // in the jar, this includes the data models/templates (potentially generated by this
    // project and) used by the data template generator and the classes used to compile
    // the generated classes.
    Configuration restClient = configurations.maybeCreate("restClient");
    restClient.extendsFrom(restClientCompile, dataTemplate);
    Configuration testRestClient = configurations.maybeCreate("testRestClient");
    testRestClient.extendsFrom(restClient, testDataTemplate);
    Properties properties = new Properties();
    InputStream inputStream = getClass().getResourceAsStream("/pegasus-version.properties");
    if (inputStream != null) {
        try {
            properties.load(inputStream);
        } catch (IOException e) {
            throw new GradleException("Unable to read pegasus-version.properties file.", e);
        }
        String version = properties.getProperty("pegasus.version");
        project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, "com.linkedin.pegasus:data:" + version);
        project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, "com.linkedin.pegasus:data-avro-generator:" + version);
        project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, "com.linkedin.pegasus:generator:" + version);
        project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, "com.linkedin.pegasus:restli-tools:" + version);
    } else {
        project.getLogger().lifecycle("Unable to add pegasus dependencies to {}. Please be sure that " + "'com.linkedin.pegasus:data', 'com.linkedin.pegasus:data-avro-generator', 'com.linkedin.pegasus:generator', 'com.linkedin.pegasus:restli-tools'" + " are available on the configuration pegasusPlugin", project.getPath());
    }
    project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, "org.slf4j:slf4j-simple:1.7.2");
    project.getDependencies().add(PEGASUS_PLUGIN_CONFIGURATION, project.files(System.getProperty("java.home") + "/../lib/tools.jar"));
    // this call has to be here because:
    // 1) artifact cannot be published once projects has been evaluated, so we need to first
    // create the tasks and artifact handler, then progressively append sources
    // 2) in order to append sources progressively, the source and documentation tasks and artifacts must be
    // configured/created before configuring and creating the code generation tasks.
    configureGeneratedSourcesAndJavadoc(project);
    ChangedFileReportTask changedFileReportTask = project.getTasks().create("changedFilesReport", ChangedFileReportTask.class);
    project.getTasks().getByName("check").dependsOn(changedFileReportTask);
    SourceSetContainer sourceSets = project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
    sourceSets.all(sourceSet -> {
        if (sourceSet.getName().toLowerCase(Locale.US).contains("generated")) {
            return;
        }
        checkAvroSchemaExist(project, sourceSet);
        // the idl Generator input options will be inside the PegasusOptions class. Users of the
        // plugin can set the inputOptions in their build.gradle
        @SuppressWarnings("unchecked") Map<String, PegasusOptions> pegasusOptions = (Map<String, PegasusOptions>) project.getExtensions().getExtraProperties().get("pegasus");
        pegasusOptions.put(sourceSet.getName(), new PegasusOptions());
        // rest model generation could fail on incompatibility
        // if it can fail, fail it early
        configureRestModelGeneration(project, sourceSet);
        // Do compatibility check for schemas under "pegasus" directory if the configuration property is provided.
        if (isPropertyTrue(project, ENABLE_PEGASUS_SCHEMA_COMPATIBILITY_CHECK)) {
            configurePegasusSchemaSnapshotGeneration(project, sourceSet, false);
        }
        configurePegasusSchemaSnapshotGeneration(project, sourceSet, true);
        configureConversionUtilities(project, sourceSet);
        GenerateDataTemplateTask generateDataTemplateTask = configureDataTemplateGeneration(project, sourceSet);
        configureAvroSchemaGeneration(project, sourceSet);
        configureRestClientGeneration(project, sourceSet);
        if (!isPropertyTrue(project, DISABLE_SCHEMA_ANNOTATION_VALIDATION)) {
            configureSchemaAnnotationValidation(project, sourceSet, generateDataTemplateTask);
        }
        Task cleanGeneratedDirTask = project.task(sourceSet.getTaskName("clean", "GeneratedDir"));
        cleanGeneratedDirTask.doLast(new CacheableAction<>(task -> {
            deleteGeneratedDir(project, sourceSet, REST_GEN_TYPE);
            deleteGeneratedDir(project, sourceSet, AVRO_SCHEMA_GEN_TYPE);
            deleteGeneratedDir(project, sourceSet, DATA_TEMPLATE_GEN_TYPE);
        }));
        // make clean depends on deleting the generated directories
        project.getTasks().getByName("clean").dependsOn(cleanGeneratedDirTask);
        // Set data schema directories as resource roots
        configureDataSchemaResourcesRoot(project, sourceSet);
    });
    project.getExtensions().getExtraProperties().set(GENERATOR_CLASSLOADER_NAME, getClass().getClassLoader());
}