Examples with TemporaryPath com.walmartlabs.concord.common.TemporaryPath used on opensource projects

Example 16 with TemporaryPath

use of com.walmartlabs.concord.common.TemporaryPath in project concord by walmartlabs.

the class RepositoryRefresher method refresh.

public void refresh(String orgName, String projectName, String repositoryName, boolean sync) {
    UUID orgId = orgManager.assertAccess(orgName, true).getId();
    ProjectEntry projectEntry = assertProject(orgId, projectName, ResourceAccessLevel.READER, true);
    UUID projectId = projectEntry.getId();
    RepositoryEntry repositoryEntry = assertRepository(projectEntry, repositoryName);
    if (!sync) {
        Map<String, Object> event = new HashMap<>();
        event.put("event", "repositoryRefresh");
        event.put("org", orgName);
        event.put("project", projectName);
        event.put("repository", repositoryName);
        externalEventResource.event("concord", event);
        return;
    }
    try (TemporaryPath tmpRepoPath = IOUtils.tempDir("refreshRepo_")) {
        repositoryManager.withLock(repositoryEntry.getUrl(), () -> {
            Repository repo = repositoryManager.fetch(projectId, repositoryEntry);
            repo.export(tmpRepoPath.path());
            return null;
        });
        tx(tx -> {
            for (RepositoryRefreshListener l : listeners) {
                l.onRefresh(tx, repositoryEntry, tmpRepoPath.path());
            }
        });
    } catch (Exception e) {
        String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
        throw new ConcordApplicationException("Error while refreshing repository: \n" + errorMessage, e);
    }
}

Example 17 with TemporaryPath

use of com.walmartlabs.concord.common.TemporaryPath in project concord by walmartlabs.

the class GitClientFetchTest method testFetch1.

@Test
public void testFetch1() throws Exception {
    Path tmpDir = IOUtils.createTempDir("test");
    IOUtils.copy(resourceToPath("/test4"), tmpDir);
    // init repo
    Git repo = Git.init().setInitialBranch("master").setDirectory(tmpDir.toFile()).call();
    repo.add().addFilepattern(".").call();
    RevCommit initialCommit = commit(repo, "import");
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        // --- fetch master
        String actualCommitId = fetch(tmpDir.toUri().toString(), "master", null, null, repoPath.path());
        assertContent(repoPath, "concord.yml", "concord-init");
        assertEquals(initialCommit.name(), actualCommitId);
        // update file in repo
        Files.copy(tmpDir.resolve("new_concord.yml"), tmpDir.resolve("concord.yml"), StandardCopyOption.REPLACE_EXISTING);
        repo.add().addFilepattern(".").call();
        RevCommit commitAfterUpdate = commit(repo, "update");
        // --- fetch prev commit
        String prevCommit = fetch(tmpDir.toUri().toString(), "master", initialCommit.name(), null, repoPath.path());
        assertContent(repoPath, "concord.yml", "concord-init");
        assertEquals(initialCommit.name(), prevCommit);
        // --- fetch master again
        actualCommitId = fetch(tmpDir.toUri().toString(), "master", null, null, repoPath.path());
        assertContent(repoPath, "concord.yml", "new-concord-content");
        assertEquals(commitAfterUpdate.name(), actualCommitId);
    }
}

Example 18 with TemporaryPath

use of com.walmartlabs.concord.common.TemporaryPath in project concord by walmartlabs.

the class GitClientSpeedTest method testFetch.

@Test
public void testFetch() throws Exception {
    String url = "https://github.com/walmartlabs/concord";
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        // fetch master
        fetch(url, "master", null, null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.73.1-SNAPSHOT</version>");
        long size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#1: " + FileUtils.byteCountToDisplaySize(size));
        // fetch branch
        fetch(url, "1.70.x", null, null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.70.2-SNAPSHOT</version>");
        size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#2: " + FileUtils.byteCountToDisplaySize(size));
        // fetch tag
        String tagCommitId = fetch(url, "1.73.0", null, null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.73.0</version>");
        assertEquals("9c080a5e3a34dea8ae7c3b19b66a7c26e78a9c62", tagCommitId);
        size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#3: " + FileUtils.byteCountToDisplaySize(size));
        // fetch by commit
        fetch(url, null, "071b966c36f30047aba6e94b49564a836c26bbd5", null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.72.1-SNAPSHOT</version>");
        size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#4: " + FileUtils.byteCountToDisplaySize(size));
        fetch(url, null, "910dbf2830f67b04ccbfd6aa0f2fad4aa5b54834", null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.72.0</version>");
        size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#5: " + FileUtils.byteCountToDisplaySize(size));
    }
    // fetch by commit with clean repo
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        String result = fetch(url, "1.63.x", "64feb9fe2d518e71a5497ba43132f4dafa1c471f", null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.63.1</version>");
        assertEquals("64feb9fe2d518e71a5497ba43132f4dafa1c471f", result);
        long size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#6: " + FileUtils.byteCountToDisplaySize(size));
    }
    // fetch by commit with clean repo
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        String result = fetch(url, "1.66.0", "56e248cf2e6fa10d058e41aac005b5dee70526e4", null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.65.1-SNAPSHOT</version>");
        assertEquals("56e248cf2e6fa10d058e41aac005b5dee70526e4", result);
        long size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#7: " + FileUtils.byteCountToDisplaySize(size));
    }
    // fetch by commit with clean repo and without branch -> should  fetch all repo and checkout commit-id
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        String result = fetch(url, null, "56e248cf2e6fa10d058e41aac005b5dee70526e4", null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.65.1-SNAPSHOT</version>");
        assertEquals("56e248cf2e6fa10d058e41aac005b5dee70526e4", result);
        long size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#8: " + FileUtils.byteCountToDisplaySize(size));
    }
    // fetch same branch two times
    try (TemporaryPath repoPath = IOUtils.tempDir("git-client-test")) {
        // fetch branch
        fetch(url, "1.70.x", null, null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.70.2-SNAPSHOT</version>");
        long size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#9: " + FileUtils.byteCountToDisplaySize(size));
        // fetch branch
        fetch(url, "1.70.x", null, null, repoPath.path());
        assertContent(repoPath, "pom.xml", "<version>1.70.2-SNAPSHOT</version>");
        size = FileUtils.sizeOfDirectory(repoPath.path().toFile());
        System.out.println("#10: " + FileUtils.byteCountToDisplaySize(size));
    }
}

Example 19 with TemporaryPath

use of com.walmartlabs.concord.common.TemporaryPath in project concord by walmartlabs.

the class CheckpointManager method process.

public void process(UUID checkpointId, UUID correlationId, String checkpointName, Path baseDir) throws ExecutionException {
    try {
        Path checkpointDir = baseDir.resolve(Constants.Files.JOB_CHECKPOINTS_DIR_NAME);
        if (!Files.exists(checkpointDir)) {
            Files.createDirectories(checkpointDir);
        }
        Path checkpointMeta = baseDir.resolve(Constants.Files.CHECKPOINT_META_FILE_NAME);
        Files.write(checkpointMeta, checkpointName.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
        try (TemporaryPath checkpointFile = new TemporaryPath(checkpointDir.resolve(checkpointId + "_" + checkpointName + ".zip"))) {
            try (ZipArchiveOutputStream zip = new ZipArchiveOutputStream(Files.newOutputStream(checkpointFile.path()))) {
                IOUtils.zip(zip, InternalConstants.Files.JOB_ATTACHMENTS_DIR_NAME + "/", baseDir.resolve(InternalConstants.Files.JOB_ATTACHMENTS_DIR_NAME));
                IOUtils.zip(zip, InternalConstants.Files.CONCORD_SYSTEM_DIR_NAME + "/", baseDir.resolve(InternalConstants.Files.CONCORD_SYSTEM_DIR_NAME));
                IOUtils.zipFile(zip, checkpointMeta, Constants.Files.CHECKPOINT_META_FILE_NAME);
            }
            try (InputStream in = Files.newInputStream(checkpointFile.path())) {
                Map<String, Object> data = new HashMap<>();
                data.put("id", checkpointId);
                data.put("correlationId", correlationId);
                data.put("name", checkpointName);
                data.put("data", in);
                processApiClient.uploadCheckpoint(instanceId, data);
            }
        }
        Files.delete(checkpointMeta);
        // write resume event
        Path resumeEventFile = baseDir.resolve(InternalConstants.Files.JOB_ATTACHMENTS_DIR_NAME).resolve(InternalConstants.Files.JOB_STATE_DIR_NAME).resolve(InternalConstants.Files.RESUME_MARKER_FILE_NAME);
        Files.write(resumeEventFile, checkpointName.getBytes());
        log.info("checkpoint {} ('{}')", checkpointName, checkpointId);
    } catch (Exception e) {
        throw new ExecutionException("Checkpoint process error", e);
    }
}

Example 20 with TemporaryPath

use of com.walmartlabs.concord.common.TemporaryPath in project concord by walmartlabs.

the class DefaultCheckpointService method create.

@Override
public void create(ThreadId threadId, String name, Runtime runtime, ProcessSnapshot snapshot) {
    validate(threadId, snapshot);
    UUID checkpointId = UUID.randomUUID();
    try (StateArchive archive = new StateArchive()) {
        // the goal here is to create a process state snapshot with
        // a "synthetic" event that can be used to continue the process
        // after the checkpoint step
        String resumeEventRef = checkpointId.toString();
        State state = clone(snapshot.vmState(), classLoader);
        state.setEventRef(threadId, resumeEventRef);
        state.setStatus(threadId, ThreadStatus.SUSPENDED);
        archive.withResumeEvent(resumeEventRef).withProcessState(ProcessSnapshot.builder().from(snapshot).vmState(state).build()).withSystemDirectory(workingDirectory.getValue());
        try (TemporaryPath zip = archive.zip()) {
            checkpointUploader.upload(checkpointId, name, zip.path());
        }
    } catch (Exception e) {
        throw new RuntimeException("Checkpoint upload error", e);
    }
    log.info("Checkpoint '{}' created", name);
}