Examples with RunningIndexJob com.searchcode.app.dto.RunningIndexJob used on opensource projects

Example 1 with RunningIndexJob

use of com.searchcode.app.dto.RunningIndexJob in project searchcode-server by boyter.

the class AdminRouteService method getStat.

public String getStat(String statname) {
    if (statname == null) {
        return Values.EMPTYSTRING;
    }
    switch(statname.toLowerCase()) {
        case "memoryusage":
            return this.statsService.getMemoryUsage("<br>");
        case "loadaverage":
            return this.statsService.getLoadAverage();
        case "uptime":
            return this.statsService.getUptime();
        case "searchcount":
            return Values.EMPTYSTRING + this.statsService.getSearchCount();
        case "runningjobs":
            StringBuilder stringBuffer = new StringBuilder();
            for (String key : Singleton.getRunningIndexRepoJobs().keySet()) {
                RunningIndexJob indexJob = Singleton.getRunningIndexRepoJobs().get(key);
                if (indexJob != null) {
                    int runningTime = Singleton.getHelpers().getCurrentTimeSeconds() - indexJob.startTime;
                    stringBuffer.append(key).append(" <small>(").append(runningTime).append(" seconds)</small>").append(" ");
                } else {
                    stringBuffer.append(key).append(" ");
                }
            }
            return stringBuffer.toString();
        case "spellingcount":
            return Values.EMPTYSTRING + Singleton.getSpellingCorrector().getWordCount();
        case "repocount":
            return Values.EMPTYSTRING + Singleton.getRepo().getRepoCount();
        case "numdocs":
            CodeSearcher codeSearcher = new CodeSearcher();
            return Values.EMPTYSTRING + codeSearcher.getTotalNumberDocumentsIndexed();
        case "servertime":
            return new Date().toString();
        case "deletionqueue":
            return Values.EMPTYSTRING + Singleton.getDataService().getPersistentDelete().size();
        case "alllogs":
            return StringUtils.join(Singleton.getLogger().getAllLogs(), System.lineSeparator());
        case "infologs":
            return StringUtils.join(Singleton.getLogger().getInfoLogs(), System.lineSeparator());
        case "warninglogs":
            return StringUtils.join(Singleton.getLogger().getWarningLogs(), System.lineSeparator());
        case "severelogs":
            return StringUtils.join(Singleton.getLogger().getSevereLogs(), System.lineSeparator());
        case "searchlogs":
            return StringUtils.join(Singleton.getLogger().getSearchLogs(), System.lineSeparator());
        case "apilogs":
            return StringUtils.join(Singleton.getLogger().getApiLogs(), System.lineSeparator());
        case "threads":
            return "" + java.lang.Thread.activeCount();
        case "paused":
            return this.sharedService.getPauseBackgroundJobs() ? "paused" : "running";
    }
    return Values.EMPTYSTRING;
}

Example 2 with RunningIndexJob

use of com.searchcode.app.dto.RunningIndexJob in project searchcode-server by boyter.

the class IndexFileRepoJob method execute.

/**
 * The main method used for finding jobs to index and actually doing the work
 */
@Override
public void execute(JobExecutionContext context) {
    if (!isEnabled()) {
        return;
    }
    if (this.indexService.shouldPause(IIndexService.JobType.REPO_PARSER)) {
        this.logger.info("8fe60701::pausing parser");
        return;
    }
    Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
    // Pull the next repo to index from the queue
    UniqueRepoQueue repoQueue = this.getNextQueuedRepo();
    RepoResult repoResult = repoQueue.poll();
    if (repoResult != null && !Singleton.getRunningIndexRepoJobs().containsKey(repoResult.getName())) {
        this.logger.info(String.format("7aec9dd0::file indexer indexing repository %s", repoResult.getName()));
        repoResult.getData().indexStatus = "indexing";
        Singleton.getRepo().saveRepo(repoResult);
        try {
            Singleton.getRunningIndexRepoJobs().put(repoResult.getName(), new RunningIndexJob("Indexing", Singleton.getHelpers().getCurrentTimeSeconds()));
            JobDataMap data = context.getJobDetail().getJobDataMap();
            this.repoName = repoResult.getName();
            String repoRemoteLocation = repoResult.getUrl();
            String repoLocations = data.get("REPOLOCATIONS").toString();
            this.LOWMEMORY = Boolean.parseBoolean(data.get("LOWMEMORY").toString());
            Path docDir = Paths.get(repoRemoteLocation);
            this.indexDocsByPath(docDir, repoResult, repoLocations, repoRemoteLocation, true);
            int runningTime = Singleton.getHelpers().getCurrentTimeSeconds() - Singleton.getRunningIndexRepoJobs().get(repoResult.getName()).startTime;
            repoResult.getData().averageIndexTimeSeconds = (repoResult.getData().averageIndexTimeSeconds + runningTime) / 2;
            repoResult.getData().indexStatus = "success";
            repoResult.getData().jobRunTime = Instant.now();
            Singleton.getRepo().saveRepo(repoResult);
        } catch (Exception ex) {
            this.logger.severe(String.format("05aa777b::error in class %s exception %s repository %s", ex.getClass(), ex.getMessage(), repoResult.getName()));
        } finally {
            // Clean up the job
            // Mark that this job is finished
            // TODO ensure that this line is covered by tests
            this.indexService.decrementRepoJobsCount();
            Singleton.getRunningIndexRepoJobs().remove(repoResult.getName());
        }
    }
}

Example 3 with RunningIndexJob

use of com.searchcode.app.dto.RunningIndexJob in project searchcode-server by boyter.

the class IndexBaseRepoJob method execute.

/**
 * The main method used for finding jobs to index and actually doing the work
 */
public void execute(JobExecutionContext context) throws JobExecutionException {
    Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
    var jobDataMap = context.getJobDetail().getJobDataMap();
    this.LOWMEMORY = Boolean.parseBoolean(jobDataMap.get("LOWMEMORY").toString());
    var repoLocations = jobDataMap.get("REPOLOCATIONS").toString();
    if (!isEnabled() || this.indexService.shouldPause(IIndexService.JobType.REPO_PARSER)) {
        this.logger.info("080364f1::pausing parser as requested");
        return;
    }
    // Pull the next repo to index from the queue
    var repoResult = this.getNextQueuedRepo().poll();
    if (repoResult != null && !Singleton.getRunningIndexRepoJobs().containsKey(repoResult.getName())) {
        this.haveRepoResult = true;
        this.logger.info(String.format("1d980f51::indexing repository %s", repoResult.getName()));
        if (repoResult.getData() != null) {
            repoResult.getData().indexStatus = "indexing";
            repoResult.getData().indexError = Values.EMPTYSTRING;
            Singleton.getRepo().saveRepo(repoResult);
        }
        try {
            Singleton.getRunningIndexRepoJobs().put(repoResult.getName(), new RunningIndexJob("Indexing", Singleton.getHelpers().getCurrentTimeSeconds()));
            this.checkCloneSuccess(repoResult.getDirectoryName(), repoLocations);
            var repoGitLocation = repoLocations + "/" + repoResult.getDirectoryName() + "/.git/";
            var file = new File(repoGitLocation);
            // TODO this assumes git every time? Correct????
            var existingRepo = file.exists();
            var useCredentials = repoResult.getUsername() != null && !repoResult.getUsername().isEmpty();
            RepositoryChanged repositoryChanged;
            if (existingRepo) {
                repositoryChanged = this.updateExistingRepository(repoResult, repoLocations, useCredentials);
            } else {
                repositoryChanged = this.getNewRepository(repoResult, repoLocations, useCredentials);
            }
            // Write file indicating we have successfully cloned
            this.createCloneUpdateSuccess(repoLocations + "/" + repoResult.getDirectoryName());
            this.triggerIndex(repoResult, repoResult.getName(), repoResult.getUrl(), repoLocations, repoGitLocation, existingRepo, repositoryChanged);
            if (this.DELETEREPO) {
                Singleton.getHelpers().tryDelete(repoLocations + "/" + repoResult.getDirectoryName());
            }
        } catch (Exception ex) {
            this.logger.severe(String.format("f8026b97::error in class %s exception %s", ex.getClass(), ex.getMessage()));
        } finally {
            // Clean up the job
            // Mark that this job is finished
            // TODO ensure that this line is covered by tests
            this.indexService.decrementRepoJobsCount();
            Singleton.getRunningIndexRepoJobs().remove(repoResult.getName());
        }
    }
}