Examples with RepositoryXmxStorage jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage used on opensource projects

Example 1 with RepositoryXmxStorage

use of jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage in project teamcity-git by JetBrains.

the class FetchCommandImpl method fetchInSeparateProcess.

private void fetchInSeparateProcess(@NotNull Repository repository, @NotNull URIish uri, @NotNull FetchSettings settings) throws VcsException {
    final Collection<RefSpec> specs = settings.getRefSpecs();
    final String debugInfo = getDebugInfo(repository, uri, specs);
    final ProcessXmxProvider xmxProvider = new ProcessXmxProvider(new RepositoryXmxStorage(repository, "fetch"), myConfig, "fetch", debugInfo);
    Integer xmx = xmxProvider.getNextXmx();
    int attempt = 0;
    while (xmx != null) {
        attempt++;
        File gitPropertiesFile = null;
        File teamcityPrivateKey = null;
        GitProcessStuckMonitor processStuckMonitor = null;
        try {
            File gcDump = getDumpFile(repository, "gc");
            gitPropertiesFile = myFetcherProperties.getPropertiesFile();
            teamcityPrivateKey = getTeamCityPrivateKey(settings.getAuthSettings());
            final GeneralCommandLine cl = createFetcherCommandLine(repository, uri, xmx);
            final String commandLineString = cl.getCommandLineString();
            final GitProcessExecutor processExecutor = new GitProcessExecutor(cl);
            processStuckMonitor = new GitProcessStuckMonitor(gcDump, xmx.longValue(), commandLineString) {

                @Override
                protected void stuckDetected() {
                    processExecutor.interrupt();
                }
            };
            processStuckMonitor.start();
            final GitProcessExecutor.GitExecResult gitResult = processExecutor.runProcess(getFetchProcessInputBytes(getAuthSettings(settings, teamcityPrivateKey), repository.getDirectory(), uri, specs, getDumpFile(repository, null), gcDump, gitPropertiesFile), myConfig.getFetchTimeout(), settings.createStdoutBuffer(), new ByteArrayOutputStream(), new GitProcessExecutor.ProcessExecutorAdapter() {

                @Override
                public void processStarted() {
                    if (LOG.isDebugEnabled())
                        LOG.debug("git fetch process for " + debugInfo + " started in separate process with command line: " + commandLineString);
                    settings.getProgress().reportProgress("git fetch " + uri);
                }

                @Override
                public void processFinished() {
                    if (LOG.isDebugEnabled())
                        LOG.debug("git fetch process for " + debugInfo + " finished");
                    settings.getProgress().reportProgress("git fetch " + uri + " finished");
                }

                @Override
                public void processFailed(@NotNull final ExecutionException e) {
                    if (LOG.isDebugEnabled())
                        LOG.debug("git fetch process for " + debugInfo + " failed");
                    settings.getProgress().reportProgress("git fetch " + uri + " failed");
                }
            });
            final ExecResult result = gitResult.getExecResult();
            VcsException commandError = CommandLineUtil.getCommandLineError("git fetch", " (repository dir: <TeamCity data dir>/system/caches/git/" + repository.getDirectory().getName() + ")", result, true, true);
            if (commandError != null) {
                commandError.setRecoverable(isRecoverable(commandError));
                /* if the process had not enough memory or we killed it because gc */
                if (gitResult.isOutOfMemoryError() || gitResult.isInterrupted()) {
                    final Integer nextXmx = xmxProvider.getNextXmx();
                    if (nextXmx != null) {
                        xmx = nextXmx;
                        clean(repository);
                        continue;
                    }
                    commandError = new VcsException("There is not enough memory for git fetch (last attempted -Xmx" + xmx + "M). Please contact your system administrator", commandError);
                } else if (attempt == 1 && settings.getAuthSettings().doesTokenNeedRefresh()) {
                    LOG.debug("git fetch process failed due to suspected token expiration for \"" + uri + "\" in directory \"" + repository.getDirectory() + "\", took " + TimePrinter.createMillisecondsFormatter().formatTime(gitResult.getDuration()) + ". Retrying with token refresh.");
                    continue;
                }
                LOG.info("git fetch process failed for \"" + uri + "\" in directory \"" + repository.getDirectory() + "\", took " + TimePrinter.createMillisecondsFormatter().formatTime(gitResult.getDuration()));
                if (gitResult.isTimeout()) {
                    logTimeout(debugInfo, getDumpFile(repository, null));
                }
                clean(repository);
                throw commandError;
            }
            LOG.info("git fetch process finished for: " + uri + " in directory: " + repository.getDirectory() + ", took " + gitResult.getDuration() + "ms");
            if (result.getStderr().length() > 0) {
                LOG.warn("Error output produced by git fetch:\n" + result.getStderr());
            }
            LOG.debug("git fetch process output:\n" + result.getStdout());
            break;
        } finally {
            if (teamcityPrivateKey != null) {
                FileUtil.delete(teamcityPrivateKey);
            }
            if (gitPropertiesFile != null) {
                FileUtil.delete(gitPropertiesFile);
            }
            if (processStuckMonitor != null) {
                processStuckMonitor.finish();
            }
        }
    }
}

Example 2 with RepositoryXmxStorage

use of jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage in project teamcity-git by JetBrains.

the class GitPatchBuilderDispatcher method buildPatchInSeparateProcess.

private void buildPatchInSeparateProcess() throws Exception {
    final String rootStr = LogUtil.describe(myGitRoot);
    final ProcessXmxProvider xmxProvider = new ProcessXmxProvider(new RepositoryXmxStorage(myContext.getRepository(), "patch"), myConfig, "patch", "(root: " + rootStr + ")");
    Integer xmx = xmxProvider.getNextXmx();
    while (xmx != null) {
        final GeneralCommandLine patchCmd = createPatchCommandLine(xmx);
        final File patchFile = FileUtil.createTempFile("git", "patch");
        final File internalProperties = getPatchPropertiesFile();
        try {
            final ByteArrayOutputStream stdout = new LineAwareByteArrayOutputStream(Charset.forName("UTF-8"), new NoOpLineListener(), false);
            final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
            final GitProcessExecutor.GitExecResult gitResult = new GitProcessExecutor(patchCmd).runProcess(getInput(patchFile, internalProperties), myConfig.getPatchProcessIdleTimeoutSeconds(), stdout, stderr, new GitProcessExecutor.ProcessExecutorAdapter());
            VcsException patchError = CommandLineUtil.getCommandLineError("build patch", gitResult.getExecResult());
            if (patchError != null) {
                if (gitResult.isOutOfMemoryError()) {
                    final Integer nextXmx = xmxProvider.getNextXmx();
                    if (nextXmx != null) {
                        xmx = nextXmx;
                        FileUtil.delete(patchFile);
                        continue;
                    }
                    throw new VcsException("There is not enough memory for git patch (last attempted -Xmx" + xmx + "M). Please contact your system administrator", patchError);
                } else if (gitResult.isTimeout()) {
                    throw new VcsException("git patch for root " + rootStr + " was idle for more than " + myConfig.getFetchTimeout() + " second(s), try increasing the timeout using the " + PluginConfigImpl.TEAMCITY_GIT_IDLE_TIMEOUT_SECONDS + " property", patchError);
                }
                throw patchError;
            }
            new LowLevelPatcher(new FileInputStream(patchFile)).applyPatch(new NoExitLowLevelPatchTranslator(((PatchBuilderEx) myBuilder).getLowLevelBuilder()));
            break;
        } finally {
            FileUtil.delete(patchFile);
            FileUtil.delete(internalProperties);
        }
    }
}

Example 3 with RepositoryXmxStorage

use of jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage in project teamcity-git by JetBrains.

the class RepositoryXmxStorageTest method test_read_write_delete.

@Test
public void test_read_write_delete() throws Throwable {
    final RepositoryXmxStorage storage = create("fetch");
    then(storage.read()).isNull();
    storage.write(1024);
    then(storage.read()).isEqualTo(1024);
    storage.write(2056);
    then(storage.read()).isEqualTo(2056);
    storage.write(null);
    then(storage.read()).isNull();
}

Example 4 with RepositoryXmxStorage

use of jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage in project teamcity-git by JetBrains.

the class RepositoryXmxStorageTest method test_several_processes.

@Test
public void test_several_processes() throws Throwable {
    final RepositoryXmxStorage fetch = create("fetch");
    then(fetch.read()).isNull();
    final RepositoryXmxStorage patch = create("patch");
    then(patch.read()).isNull();
    fetch.write(1024);
    then(fetch.read()).isEqualTo(1024);
    then(patch.read()).isNull();
    patch.write(2056);
    then(fetch.read()).isEqualTo(1024);
    then(patch.read()).isEqualTo(2056);
    {
        final List<String> lines = new ArrayList<>(FileUtil.readFile(fetch.getStorage()));
        lines.removeIf(s -> s.startsWith("#"));
        then(lines).containsExactly("fetch.Xmx=1024M", "patch.Xmx=2056M");
    }
    fetch.write(2056);
    then(fetch.read()).isEqualTo(2056);
    then(patch.read()).isEqualTo(2056);
    patch.write(1024);
    then(fetch.read()).isEqualTo(2056);
    then(patch.read()).isEqualTo(1024);
    {
        final List<String> lines = new ArrayList<>(FileUtil.readFile(fetch.getStorage()));
        lines.removeIf(s -> s.startsWith("#"));
        then(lines).containsExactly("fetch.Xmx=2056M", "patch.Xmx=1024M");
    }
    fetch.write(null);
    then(fetch.read()).isNull();
    then(patch.read()).isEqualTo(1024);
    patch.write(null);
    then(fetch.read()).isNull();
    then(patch.read()).isNull();
    then(fetch.getStorage()).doesNotExist();
}

Example 5 with RepositoryXmxStorage

use of jetbrains.buildServer.buildTriggers.vcs.git.process.RepositoryXmxStorage in project teamcity-git by JetBrains.

the class RepositoryXmxStorageTest method test_write.

@Test
public void test_write() throws Throwable {
    final RepositoryXmxStorage storage = create("fetch");
    storage.write(12345);
    final List<String> lines = new ArrayList<>(FileUtil.readFile(storage.getStorage()));
    lines.removeIf(s -> s.startsWith("#"));
    then(lines).containsExactly("fetch.Xmx=12345M");
}