use of org.craftercms.studio.api.v2.dal.ClusterSiteRecord in project studio by craftercms.
the class StudioClusterPublishedRepoSyncTask method updateContent.
protected void updateContent(long sId, String siteId, List<ClusterMember> clusterNodes, List<ClusterSiteRecord> clusterSiteRecords) throws IOException, CryptoException, ServiceLayerException {
logger.debug("Update published repo for site " + siteId);
Path siteSandboxPath = buildRepoPath(siteId).resolve(GIT_ROOT);
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repo = builder.setGitDir(siteSandboxPath.toFile()).readEnvironment().findGitDir().build();
String gitLockKey = SITE_PUBLISHED_REPOSITORY_GIT_LOCK.replaceAll(PATTERN_SITE, siteId);
logger.debug("Git Lock Key: " + gitLockKey);
try (Git git = new Git(repo)) {
Set<String> environments = getAllPublishingEnvironments(siteId);
logger.debug("Update published repo from all active cluster members");
if (generalLockService.tryLock(gitLockKey)) {
try {
for (ClusterMember remoteNode : clusterNodes) {
ClusterSiteRecord csr = clusterDao.getClusterSiteRecord(remoteNode.getId(), sId);
if (Objects.nonNull(csr) && csr.getPublishedRepoCreated() > 0) {
try {
logger.debug("Fetch from cluster member " + remoteNode.getLocalAddress());
final Path tempKey = Files.createTempFile(UUID.randomUUID().toString(), ".tmp");
FetchCommand fetch = git.fetch().setRemote(remoteNode.getGitRemoteName());
fetch = studioClusterUtils.configureAuthenticationForCommand(remoteNode, fetch, tempKey);
fetch.call();
Files.delete(tempKey);
} catch (GitAPIException e) {
logger.error("Error while fetching published repo for site " + siteId + " from remote " + remoteNode.getGitRemoteName());
logger.error(e.getMessage());
}
}
}
for (String branch : environments) {
for (ClusterMember remoteNode : clusterNodes) {
ClusterSiteRecord csr = clusterDao.getClusterSiteRecord(remoteNode.getId(), sId);
if (Objects.nonNull(csr) && csr.getPublishedRepoCreated() > 0) {
try {
updatePublishedBranch(siteId, git, remoteNode, branch);
} catch (GitAPIException e) {
logger.error("Error while updating published repo for site " + siteId + " from remote " + remoteNode.getGitRemoteName() + " environment " + branch);
logger.error(e.getMessage());
}
}
}
}
} finally {
generalLockService.unlock(gitLockKey);
}
} else {
logger.debug("Failed to get lock " + gitLockKey);
}
}
}
use of org.craftercms.studio.api.v2.dal.ClusterSiteRecord in project studio by craftercms.
the class StudioClusterPublishedRepoSyncTask method executeInternal.
@Override
protected void executeInternal(String siteId) {
// Log start time
long startTime = System.currentTimeMillis();
logger.debug("Worker starts syncing cluster node published for site " + siteId);
try {
HierarchicalConfiguration<ImmutableNode> registrationData = studioClusterUtils.getClusterConfiguration();
if (registrationData != null && !registrationData.isEmpty()) {
String localAddress = studioClusterUtils.getClusterNodeLocalAddress();
ClusterMember localNode = clusterDao.getMemberByLocalAddress(localAddress);
List<ClusterMember> clusterNodes = studioClusterUtils.getClusterNodes(localAddress);
SiteFeed siteFeed = siteService.getSite(siteId);
List<ClusterSiteRecord> clusterSiteRecords = clusterDao.getSiteStateAcrossCluster(siteId);
Optional<ClusterSiteRecord> localNodeRecord = clusterSiteRecords.stream().filter(x -> x.getClusterNodeId() == localNode.getId() && StringUtils.equals(x.getState(), STATE_CREATED)).findFirst();
if (!localNodeRecord.isPresent()) {
return;
}
long nodesCreated = clusterSiteRecords.stream().filter(x -> StringUtils.equals(x.getState(), STATE_CREATED)).count();
if (nodesCreated < 1) {
return;
}
// Check if site exists
logger.debug("Check if site " + siteId + " exists in local repository");
boolean success = true;
int publishedReposCreated = clusterSiteRecords.stream().mapToInt(ClusterSiteRecord::getPublishedRepoCreated).sum();
if (publishedReposCreated > 0 || siteFeed.getPublishedRepoCreated() > 0) {
boolean siteCheck = checkIfSiteRepoExists(siteId);
if (!siteCheck) {
// Site doesn't exist locally, create it
success = createSite(localNode.getId(), siteFeed.getId(), siteId, siteFeed.getSandboxBranch());
} else {
clusterDao.setPublishedRepoCreated(localNode.getId(), siteFeed.getId());
}
} else {
success = false;
}
if (success) {
try {
// Add the remote repositories to the local repository to sync from if not added already
logger.debug("Add remotes for site " + siteId);
addRemotes(siteId, clusterNodes);
} catch (InvalidRemoteUrlException | ServiceLayerException | CryptoException e) {
logger.error("Error while adding remotes on cluster node for site " + siteId);
}
try {
// Sync with remote and update the local cache with the last commit ID to speed things up
logger.debug("Update content for site " + siteId);
updateContent(siteFeed.getId(), siteId, clusterNodes, clusterSiteRecords);
} catch (IOException | CryptoException | ServiceLayerException e) {
logger.error("Error while updating content for site " + siteId + " on cluster node.", e);
}
}
}
} catch (SiteNotFoundException e) {
logger.error("Error while executing Cluster Node Sync Published for site " + siteId, e);
}
// Compute execution duration and log it
long duration = System.currentTimeMillis() - startTime;
logger.debug("Worker finished syncing cluster node for site " + siteId);
logger.debug("Worker performed cluster node sync for site " + siteId + " in " + duration + "ms");
logger.debug("Finished Cluster Node Sync task for site " + siteId);
}
use of org.craftercms.studio.api.v2.dal.ClusterSiteRecord in project studio by craftercms.
the class StudioClusterSandboxRepoSyncTask method executeInternal.
@Override
protected void executeInternal(String siteId) {
// Log start time
long startTime = System.currentTimeMillis();
logger.debug("Worker starts syncing cluster node sandbox for site " + siteId);
try {
HierarchicalConfiguration<ImmutableNode> registrationData = studioClusterUtils.getClusterConfiguration();
if (registrationData != null && !registrationData.isEmpty()) {
String localAddress = studioClusterUtils.getClusterNodeLocalAddress();
ClusterMember localNode = clusterDao.getMemberByLocalAddress(localAddress);
List<ClusterMember> clusterNodes = studioClusterUtils.getClusterNodes(localAddress);
SiteFeed siteFeed = siteService.getSite(siteId);
List<ClusterSiteRecord> clusterSiteRecords = clusterDao.getSiteStateAcrossCluster(siteId);
long nodesCreated = clusterSiteRecords.stream().filter(x -> StringUtils.equals(x.getState(), STATE_CREATED)).count();
if (nodesCreated < 1 && !StringUtils.equals(siteFeed.getState(), STATE_CREATED)) {
return;
}
// Check if site exists
logger.debug("Check if site " + siteId + " exists in local repository");
boolean success = true;
boolean siteCheck = checkIfSiteRepoExists(siteId);
if (!siteCheck) {
// Site doesn't exist locally, create it
success = createSite(localNode.getId(), siteFeed.getId(), siteId, siteFeed.getSiteUuid(), siteFeed.getSearchEngine(), clusterNodes, clusterSiteRecords);
}
if (success && clusterDao.existsClusterSiteSyncRepo(localNode.getId(), siteFeed.getId()) < 1) {
String commitId = contentRepository.getRepoLastCommitId(siteId);
clusterDao.insertClusterSiteSyncRepo(localNode.getId(), siteFeed.getId(), commitId, commitId, siteFeed.getLastSyncedGitlogCommitId());
clusterDao.setSiteState(localNode.getId(), siteFeed.getId(), STATE_CREATED);
addSiteUuidFile(siteId, siteFeed.getSiteUuid());
}
if (success) {
syncRemoteRepositories(siteId, localAddress);
// Check if the site needs to be synced
boolean syncRequired = isSyncRequired(siteId, siteFeed.getLastCommitId());
if (syncRequired) {
try {
// Add the remote repositories to the local repository to sync from if not added already
logger.debug("Add remotes for site " + siteId);
addRemotes(siteId, clusterNodes);
} catch (InvalidRemoteUrlException | ServiceLayerException e) {
logger.error("Error while adding remotes on cluster node for site " + siteId);
}
try {
// Sync with remote and update the local cache with the last commit ID to speed things up
logger.debug("Update content for site " + siteId);
updateContent(localNode.getId(), siteFeed.getId(), siteId, siteFeed.getSandboxBranch(), clusterNodes);
} catch (IOException | CryptoException | ServiceLayerException e) {
logger.error("Error while updating content for site " + siteId + " on cluster node.", e);
}
}
}
}
} catch (SiteNotFoundException | IOException e) {
logger.error("Error while executing Cluster Node Sync Sandbox for site " + siteId, e);
}
// Compute execution duration and log it
long duration = System.currentTimeMillis() - startTime;
logger.debug("Worker finished syncing cluster node for site " + siteId);
logger.debug("Worker performed cluster node sync for site " + siteId + " in " + duration + "ms");
logger.debug("Finished Cluster Node Sync task for site " + siteId);
}
use of org.craftercms.studio.api.v2.dal.ClusterSiteRecord in project studio by craftercms.
the class StudioClusterSandboxRepoSyncTask method updateContent.
protected void updateContent(long localNodeId, long sId, String siteId, String sandboxBranchName, List<ClusterMember> clusterNodes) throws IOException, CryptoException, ServiceLayerException {
logger.debug("Update sandbox for site " + siteId);
Path siteSandboxPath = buildRepoPath(siteId).resolve(GIT_ROOT);
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repo = builder.setGitDir(siteSandboxPath.toFile()).readEnvironment().findGitDir().build();
Map<String, String> remoteLastSyncCommits = remotesMap.get(siteId);
if (remoteLastSyncCommits == null || remoteLastSyncCommits.isEmpty()) {
remoteLastSyncCommits = new HashMap<String, String>();
remotesMap.put(siteId, remoteLastSyncCommits);
}
try (Git git = new Git(repo)) {
logger.debug("Update content from each active cluster memeber");
for (ClusterMember remoteNode : clusterNodes) {
ClusterSiteRecord csr = clusterDao.getClusterSiteRecord(remoteNode.getId(), sId);
if (Objects.nonNull(csr) && StringUtils.equals(csr.getState(), STATE_CREATED)) {
updateBranch(siteId, git, remoteNode, sandboxBranchName);
}
}
String updatedCommitId = contentRepository.getRepoLastCommitId(siteId);
clusterDao.updateNodeLastCommitId(localNodeId, sId, updatedCommitId);
PreviewEventContext context = new PreviewEventContext();
context.setSite(siteId);
eventService.publish(EVENT_PREVIEW_SYNC, context);
} catch (GitAPIException e) {
logger.error("Error while syncing cluster node content for site " + siteId);
}
}
use of org.craftercms.studio.api.v2.dal.ClusterSiteRecord in project studio by craftercms.
the class StudioClusterSandboxRepoSyncTask method createSiteFromRemote.
protected boolean createSiteFromRemote(String siteId, long localNodeId, List<ClusterSiteRecord> clusterSiteRecords) throws CryptoException, ServiceLayerException {
// Clone from the first node in the cluster (it doesn't matter which one to clone from, so pick the first)
// we will eventually to catch up to the latest
boolean cloned = false;
int idx = 0;
String gitLockKey = SITE_SANDBOX_REPOSITORY_GIT_LOCK.replaceAll(PATTERN_SITE, siteId);
if (generalLockService.tryLock(gitLockKey)) {
try {
Optional<ClusterSiteRecord> csr = clusterSiteRecords.stream().filter(x -> StringUtils.equals(x.getState(), STATE_CREATED) && !(x.getClusterNodeId() == localNodeId)).findFirst();
if (csr.isPresent()) {
ClusterMember remoteNode = clusterDao.getMemberById(csr.get().getClusterNodeId());
logger.debug("Cloning " + SANDBOX + " repository for site " + siteId + " from " + remoteNode.getLocalAddress());
// prepare a new folder for the cloned repository
Path siteSandboxPath = buildRepoPath(siteId);
File localPath = siteSandboxPath.toFile();
// then clone
logger.debug("Cloning from " + remoteNode.getGitUrl() + " to " + localPath);
CloneCommand cloneCommand = Git.cloneRepository();
Git cloneResult = null;
try {
if (localPath.exists()) {
FileUtils.forceDelete(localPath);
}
final Path tempKey = Files.createTempFile(UUID.randomUUID().toString(), ".tmp");
logger.debug("Add user credentials if provided");
studioClusterUtils.configureAuthenticationForCommand(remoteNode, cloneCommand, tempKey);
String cloneUrl = remoteNode.getGitUrl().replace("{siteId}", siteId);
cloneUrl = cloneUrl + "/" + studioConfiguration.getProperty(SANDBOX_PATH);
logger.debug("Executing clone command");
Git gitClone = cloneResult = cloneCommand.setURI(cloneUrl).setRemote(remoteNode.getGitRemoteName()).setDirectory(localPath).setCloneAllBranches(true).call();
Files.deleteIfExists(tempKey);
cloned = validateRepository(gitClone.getRepository());
} catch (InvalidRemoteException e) {
logger.error("Invalid remote repository: " + remoteNode.getGitRemoteName() + " (" + remoteNode.getGitUrl() + ")", e);
} catch (TransportException e) {
if (StringUtils.endsWithIgnoreCase(e.getMessage(), "not authorized")) {
logger.error("Bad credentials or read only repository: " + remoteNode.getGitRemoteName() + " (" + remoteNode.getGitUrl() + ")", e);
} else {
logger.error("Remote repository not found: " + remoteNode.getGitRemoteName() + " (" + remoteNode.getGitUrl() + ")", e);
}
} catch (GitAPIException | IOException e) {
logger.error("Error while creating repository for site with path" + siteSandboxPath, e);
} finally {
if (cloneResult != null) {
cloneResult.close();
}
}
}
} finally {
generalLockService.unlock(gitLockKey);
}
} else {
logger.debug("Failed to get lock " + gitLockKey);
}
return cloned;
}
Aggregations