Examples with GitLog org.craftercms.studio.api.v2.dal.GitLog used on opensource projects

Example 1 with GitLog

use of org.craftercms.studio.api.v2.dal.GitLog in project studio by craftercms.

the class StudioAuditLogProcessingTask method processAuditLogFromRepo.

private void processAuditLogFromRepo(String siteId, int batchSize) throws SiteNotFoundException {
    List<GitLog> unauditedGitlogs = contentRepository.getUnauditedCommits(siteId, batchSize);
    if (unauditedGitlogs != null) {
        SiteFeed siteFeed = siteService.getSite(siteId);
        for (GitLog gl : unauditedGitlogs) {
            if (contentRepository.commitIdExists(siteId, gl.getCommitId())) {
                String prevCommitId = gl.getCommitId() + PREVIOUS_COMMIT_SUFFIX;
                List<RepoOperation> operations = contentRepository.getOperationsFromDelta(siteId, prevCommitId, gl.getCommitId());
                for (RepoOperation repoOperation : operations) {
                    Map<String, String> activityInfo = new HashMap<String, String>();
                    String contentClass;
                    AuditLog auditLog;
                    switch(repoOperation.getAction()) {
                        case CREATE:
                        case COPY:
                            contentClass = contentService.getContentTypeClass(siteId, repoOperation.getPath());
                            if (repoOperation.getPath().endsWith(DmConstants.XML_PATTERN)) {
                                activityInfo.put(DmConstants.KEY_CONTENT_TYPE, contentClass);
                            }
                            logger.debug("Insert audit log for site: " + siteId + " path: " + repoOperation.getPath());
                            auditLog = auditServiceInternal.createAuditLogEntry();
                            auditLog.setOperation(OPERATION_CREATE);
                            auditLog.setOperationTimestamp(repoOperation.getDateTime());
                            auditLog.setSiteId(siteFeed.getId());
                            auditLog.setActorId(repoOperation.getAuthor());
                            auditLog.setActorDetails(repoOperation.getAuthor());
                            auditLog.setPrimaryTargetId(siteId + ":" + repoOperation.getPath());
                            auditLog.setPrimaryTargetType(TARGET_TYPE_CONTENT_ITEM);
                            auditLog.setPrimaryTargetValue(repoOperation.getPath());
                            auditLog.setPrimaryTargetSubtype(contentService.getContentTypeClass(siteId, repoOperation.getPath()));
                            auditLog.setOrigin(ORIGIN_GIT);
                            auditServiceInternal.insertAuditLog(auditLog);
                            break;
                        case UPDATE:
                            contentClass = contentService.getContentTypeClass(siteId, repoOperation.getPath());
                            if (repoOperation.getPath().endsWith(DmConstants.XML_PATTERN)) {
                                activityInfo.put(DmConstants.KEY_CONTENT_TYPE, contentClass);
                            }
                            logger.debug("Insert audit log for site: " + siteId + " path: " + repoOperation.getPath());
                            auditLog = auditServiceInternal.createAuditLogEntry();
                            auditLog.setOperation(OPERATION_UPDATE);
                            auditLog.setOperationTimestamp(repoOperation.getDateTime());
                            auditLog.setSiteId(siteFeed.getId());
                            auditLog.setActorId(repoOperation.getAuthor());
                            auditLog.setActorDetails(repoOperation.getAuthor());
                            auditLog.setOrigin(ORIGIN_GIT);
                            auditLog.setPrimaryTargetId(siteId + ":" + repoOperation.getPath());
                            auditLog.setPrimaryTargetType(TARGET_TYPE_CONTENT_ITEM);
                            auditLog.setPrimaryTargetValue(repoOperation.getPath());
                            auditLog.setPrimaryTargetSubtype(contentService.getContentTypeClass(siteId, repoOperation.getPath()));
                            auditServiceInternal.insertAuditLog(auditLog);
                            break;
                        case DELETE:
                            contentClass = contentService.getContentTypeClass(siteId, repoOperation.getPath());
                            if (repoOperation.getPath().endsWith(DmConstants.XML_PATTERN)) {
                                activityInfo.put(DmConstants.KEY_CONTENT_TYPE, contentClass);
                            }
                            logger.debug("Insert audit log for site: " + siteId + " path: " + repoOperation.getPath());
                            auditLog = auditServiceInternal.createAuditLogEntry();
                            auditLog.setOperation(OPERATION_DELETE);
                            auditLog.setOperationTimestamp(repoOperation.getDateTime());
                            auditLog.setSiteId(siteFeed.getId());
                            auditLog.setOrigin(ORIGIN_GIT);
                            auditLog.setActorId(repoOperation.getAuthor());
                            auditLog.setActorDetails(repoOperation.getAuthor());
                            auditLog.setPrimaryTargetId(siteId + ":" + repoOperation.getPath());
                            auditLog.setPrimaryTargetType(TARGET_TYPE_CONTENT_ITEM);
                            auditLog.setPrimaryTargetValue(repoOperation.getPath());
                            auditLog.setPrimaryTargetSubtype(contentService.getContentTypeClass(siteId, repoOperation.getPath()));
                            auditServiceInternal.insertAuditLog(auditLog);
                            break;
                        case MOVE:
                            contentClass = contentService.getContentTypeClass(siteId, repoOperation.getMoveToPath());
                            if (repoOperation.getMoveToPath().endsWith(DmConstants.XML_PATTERN)) {
                                activityInfo.put(DmConstants.KEY_CONTENT_TYPE, contentClass);
                            }
                            logger.debug("Insert audit log for site: " + siteId + " path: " + repoOperation.getMoveToPath());
                            auditLog = auditServiceInternal.createAuditLogEntry();
                            auditLog.setOperation(OPERATION_MOVE);
                            auditLog.setOperationTimestamp(repoOperation.getDateTime());
                            auditLog.setSiteId(siteFeed.getId());
                            auditLog.setActorId(repoOperation.getAuthor());
                            auditLog.setActorDetails(repoOperation.getAuthor());
                            auditLog.setOrigin(ORIGIN_GIT);
                            auditLog.setPrimaryTargetId(siteId + ":" + repoOperation.getMoveToPath());
                            auditLog.setPrimaryTargetType(TARGET_TYPE_CONTENT_ITEM);
                            auditLog.setPrimaryTargetValue(repoOperation.getMoveToPath());
                            auditLog.setPrimaryTargetSubtype(contentService.getContentTypeClass(siteId, repoOperation.getMoveToPath()));
                            auditServiceInternal.insertAuditLog(auditLog);
                            break;
                        default:
                            logger.error("Error: Unknown repo operation for site " + siteId + " operation: " + repoOperation.getAction());
                            break;
                    }
                }
            }
            contentRepository.markGitLogAudited(siteId, gl.getCommitId());
        }
    }
}

Example 2 with GitLog

use of org.craftercms.studio.api.v2.dal.GitLog in project studio by craftercms.

the class GitContentRepository method insertFullGitLog.

@Override
public void insertFullGitLog(String siteId, int processed) {
    List<GitLog> gitLogs = new ArrayList<>();
    try {
        GitRepositoryHelper helper = GitRepositoryHelper.getHelper(studioConfiguration, securityService, userServiceInternal, encryptor, generalLockService, retryingRepositoryOperationFacade);
        synchronized (helper.getRepository(siteId, SANDBOX)) {
            Repository repo = helper.getRepository(siteId, SANDBOX);
            try (Git git = new Git(repo)) {
                Iterable<RevCommit> logs = git.log().call();
                for (RevCommit rev : logs) {
                    GitLog gitLog = new GitLog();
                    gitLog.setCommitId(rev.getId().getName());
                    gitLog.setProcessed(processed);
                    gitLog.setSiteId(siteId);
                    gitLogs.add(gitLog);
                }
            }
        }
    } catch (GitAPIException | CryptoException e) {
        logger.error("Error getting full git log for site " + siteId, e);
    }
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("siteId", siteId);
    params.put("gitLogs", gitLogs);
    params.put("processed", 1);
    gitLogDao.insertGitLogList(params);
}

Example 3 with GitLog

use of org.craftercms.studio.api.v2.dal.GitLog in project studio by craftercms.

the class SiteServiceImpl method syncDatabaseWithRepoUnprocessedCommits.

@Override
@ValidateParams
public boolean syncDatabaseWithRepoUnprocessedCommits(@ValidateStringParam(name = "site") String site, List<GitLog> commitIds) throws IOException {
    boolean toReturn = true;
    String repoLastCommitId = contentRepository.getRepoLastCommitId(site);
    String scriptFilename = "repoOperations_" + UUID.randomUUID();
    Path scriptPath = Files.createTempFile(scriptFilename, ".sql");
    boolean success = true;
    long startUpdateDBMark = 0;
    List<String> cIds = new ArrayList<String>();
    for (GitLog gitLog : commitIds) {
        String commitId = gitLog.getCommitId();
        long startGetOperationsFromDeltaMark = logger.isDebugEnabled() ? System.currentTimeMillis() : 0;
        List<RepoOperation> repoOperationsDelta = contentRepositoryV2.getOperationsFromDelta(site, commitId + PREVIOUS_COMMIT_SUFFIX, commitId);
        if (logger.isDebugEnabled()) {
            logger.debug("Get Repo Operations from Delta finished in " + (System.currentTimeMillis() - startGetOperationsFromDeltaMark) + " milliseconds");
            logger.debug("Number of Repo operations from delta " + repoOperationsDelta.size());
        }
        if (!CollectionUtils.isEmpty(repoOperationsDelta)) {
            logger.debug("Syncing database with repository for site: " + site + " commitId = " + commitId);
            logger.debug("Operations to sync: ");
            if (logger.isDebugEnabled()) {
                for (RepoOperation repoOperation : repoOperationsDelta) {
                    logger.debug("\tOperation: " + repoOperation.getAction().toString() + " " + repoOperation.getPath());
                }
            }
            startUpdateDBMark = logger.isDebugEnabled() ? System.currentTimeMillis() : 0;
            success = processRepoOperations(site, repoOperationsDelta, scriptPath);
        }
        toReturn = toReturn && success;
    }
    StudioDBScriptRunner studioDBScriptRunner = studioDBScriptRunnerFactory.getDBScriptRunner();
    studioDBScriptRunner.execute(scriptPath.toFile());
    // At this point we have attempted to process all operations, some may have failed
    // We will update the lastCommitId of the database ignoring errors if any
    logger.debug("Done syncing operations with a result of: " + success);
    logger.debug("Syncing database lastCommitId for site: " + site);
    // Update database
    if (success) {
        contentRepositoryV2.markGitLogVerifiedProcessedBulk(site, cIds);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Update DB finished in " + (System.currentTimeMillis() - startUpdateDBMark) + " milliseconds");
    }
    // Sync all preview deployers
    try {
        logger.debug("Sync preview for site " + site);
        deploymentService.syncAllContentToPreview(site, false);
    } catch (ServiceLayerException e) {
        logger.error("Error synchronizing preview with repository for site: " + site, e);
    }
    logger.info("Done syncing database with repository for site: " + site + " for unprocessed commits with a " + "final result of: " + toReturn);
    if (toReturn) {
        updateLastCommitId(site, repoLastCommitId);
        logger.info("Last commit ID for site: " + site + " is " + repoLastCommitId);
    } else {
        // Some operations failed during sync database from repo
        // Must log and make some noise here, this isn't great
        logger.error("Some operations failed to sync to database for site: " + site + " see previous error logs");
    }
    return toReturn;
}

Example 4 with GitLog

use of org.craftercms.studio.api.v2.dal.GitLog in project studio by craftercms.

the class RepositoryManagementServiceInternalImpl method processCommitId.

private Set<String> processCommitId(String siteId, RevCommit revCommit) {
    Set<String> toRet = new HashSet<String>();
    Queue<RevCommit> commitIdsQueue = new LinkedList<RevCommit>();
    commitIdsQueue.offer(revCommit);
    while (!commitIdsQueue.isEmpty()) {
        RevCommit rc = commitIdsQueue.poll();
        if (Objects.nonNull(rc)) {
            String cId = rc.getName();
            if (!toRet.contains(cId)) {
                GitLog gitLog = contentRepositoryV2.getGitLog(siteId, cId);
                if (Objects.isNull(gitLog)) {
                    RevCommit[] parents = rc.getParents();
                    if (Objects.nonNull(parents) && parents.length > 0) {
                        for (int i = 0; i < parents.length; i++) {
                            commitIdsQueue.offer(parents[i]);
                        }
                    }
                    toRet.add(cId);
                }
            }
        }
    }
    return toRet;
}

Example 5 with GitLog

use of org.craftercms.studio.api.v2.dal.GitLog in project studio by craftercms.

the class GitContentRepository method getOperations.

@Override
public List<RepoOperation> getOperations(String site, String commitIdFrom, String commitIdTo) {
    List<RepoOperation> operations = new ArrayList<>();
    try {
        GitRepositoryHelper helper = GitRepositoryHelper.getHelper(studioConfiguration, securityService, userServiceInternal, encryptor, generalLockService, retryingRepositoryOperationFacade);
        Repository repository = helper.getRepository(site, StringUtils.isEmpty(site) ? GLOBAL : SANDBOX);
        if (repository != null) {
            try {
                // Get the sandbox repo, and then get a reference to the commitId we received and another for head
                boolean fromEmptyRepo = StringUtils.isEmpty(commitIdFrom);
                String firstCommitId = getRepoFirstCommitId(site);
                if (fromEmptyRepo) {
                    commitIdFrom = firstCommitId;
                }
                Repository repo = helper.getRepository(site, SANDBOX);
                ObjectId objCommitIdFrom = repo.resolve(commitIdFrom);
                ObjectId objCommitIdTo = repo.resolve(commitIdTo);
                ObjectId objFirstCommitId = repo.resolve(firstCommitId);
                try (Git git = new Git(repo)) {
                    if (fromEmptyRepo) {
                        try (RevWalk walk = new RevWalk(repo)) {
                            RevCommit firstCommit = walk.parseCommit(objFirstCommitId);
                            try (ObjectReader reader = repo.newObjectReader()) {
                                CanonicalTreeParser firstCommitTreeParser = new CanonicalTreeParser();
                                // reset(reader, firstCommitTree.getId());
                                firstCommitTreeParser.reset();
                                // Diff the two commit Ids
                                long startDiffMark1 = logger.isDebugEnabled() ? System.currentTimeMillis() : 0;
                                List<DiffEntry> diffEntries = git.diff().setOldTree(firstCommitTreeParser).setNewTree(null).call();
                                if (logger.isDebugEnabled()) {
                                    logger.debug("Diff from " + objFirstCommitId.getName() + " to null " + " finished in " + ((System.currentTimeMillis() - startDiffMark1) / 1000) + " seconds");
                                    logger.debug("Number of diff entries " + diffEntries.size());
                                }
                                // Now that we have a diff, let's itemize the file changes, pack them into a TO
                                // and add them to the list of RepoOperations to return to the caller
                                // also include date/time of commit by taking number of seconds and multiply by 1000 and
                                // convert to java date before sending over
                                operations.addAll(processDiffEntry(git, diffEntries, firstCommit.getId()));
                            }
                        }
                    }
                    // let's do it
                    if (!objCommitIdFrom.equals(objCommitIdTo)) {
                        // Compare HEAD with commitId we're given
                        // Get list of commits between commitId and HEAD in chronological order
                        // Get the log of all the commits between commitId and head
                        Iterable<RevCommit> commits = git.log().addRange(objCommitIdFrom, objCommitIdTo).call();
                        // Loop through through the commits and diff one from the next util head
                        ObjectId prevCommitId = objCommitIdFrom;
                        ObjectId nextCommitId = objCommitIdFrom;
                        String author = EMPTY;
                        // Reverse orders of commits
                        // TODO: DB: try to find better algorithm
                        Iterator<RevCommit> iterator = commits.iterator();
                        List<RevCommit> revCommits = new ArrayList<RevCommit>();
                        while (iterator.hasNext()) {
                            RevCommit commit = iterator.next();
                            revCommits.add(commit);
                        }
                        ReverseListIterator<RevCommit> reverseIterator = new ReverseListIterator<RevCommit>(revCommits);
                        while (reverseIterator.hasNext()) {
                            RevCommit commit = reverseIterator.next();
                            if (StringUtils.contains(commit.getFullMessage(), studioConfiguration.getProperty(REPO_SYNC_DB_COMMIT_MESSAGE_NO_PROCESSING))) {
                                prevCommitId = commit.getId();
                                logger.debug("Skipping commitId: " + prevCommitId.getName() + " for site " + site + " because it is marked not to be processed.");
                                GitLog gitLog = getGitLog(site, prevCommitId.getName());
                                if (gitLog != null) {
                                    markGitLogVerifiedProcessed(site, prevCommitId.getName());
                                } else {
                                    insertGitLog(site, prevCommitId.getName(), 1);
                                }
                                updateLastVerifiedGitlogCommitId(site, prevCommitId.getName());
                            } else {
                                nextCommitId = commit.getId();
                                if (commit.getAuthorIdent() != null) {
                                    author = commit.getAuthorIdent().getName();
                                }
                                if (StringUtils.isEmpty(author)) {
                                    author = commit.getCommitterIdent().getName();
                                }
                                RevTree prevTree = helper.getTreeForCommit(repo, prevCommitId.getName());
                                RevTree nextTree = helper.getTreeForCommit(repo, nextCommitId.getName());
                                if (prevTree != null && nextTree != null) {
                                    try (ObjectReader reader = repo.newObjectReader()) {
                                        CanonicalTreeParser prevCommitTreeParser = new CanonicalTreeParser();
                                        CanonicalTreeParser nextCommitTreeParser = new CanonicalTreeParser();
                                        prevCommitTreeParser.reset(reader, prevTree.getId());
                                        nextCommitTreeParser.reset(reader, nextTree.getId());
                                        // Diff the two commit Ids
                                        long startDiffMark2 = logger.isDebugEnabled() ? System.currentTimeMillis() : 0;
                                        List<DiffEntry> diffEntries = git.diff().setOldTree(prevCommitTreeParser).setNewTree(nextCommitTreeParser).call();
                                        if (logger.isDebugEnabled()) {
                                            logger.debug("Diff from " + objCommitIdFrom.getName() + " to " + objCommitIdTo.getName() + " finished in " + ((System.currentTimeMillis() - startDiffMark2) / 1000) + " seconds");
                                            logger.debug("Number of diff entries " + diffEntries.size());
                                        }
                                        // Now that we have a diff, let's itemize the file changes, pack them into a TO
                                        // and add them to the list of RepoOperations to return to the caller
                                        // also include date/time of commit by taking number of seconds and multiply by 1000 and
                                        // convert to java date before sending over
                                        operations.addAll(processDiffEntry(git, diffEntries, nextCommitId));
                                        prevCommitId = nextCommitId;
                                    }
                                }
                            }
                        }
                    }
                } catch (GitAPIException e) {
                    logger.error("Error getting operations for site " + site + " from commit ID: " + commitIdFrom + " to commit ID: " + commitIdTo, e);
                }
            } catch (IOException e) {
                logger.error("Error getting operations for site " + site + " from commit ID: " + commitIdFrom + " to commit ID: " + commitIdTo, e);
            }
        }
    } catch (CryptoException e) {
        logger.error("Error getting operations for site " + site + " from commit ID: " + commitIdFrom + " to commit ID: " + commitIdTo, e);
    }
    return operations;
}