Examples with Imports com.walmartlabs.concord.imports.Imports used on opensource projects

Example 1 with Imports

use of com.walmartlabs.concord.imports.Imports in project concord by walmartlabs.

the class ProjectLoaderV2 method export.

public void export(Path baseDir, Path destDir, ImportsNormalizer importsNormalizer, ImportsListener listener, CopyOption... options) throws Exception {
    YamlParserV2 parser = new YamlParserV2();
    ProcessDefinition root = loadRoot(parser, baseDir);
    Resources resources = root != null ? root.resources() : Resources.builder().build();
    boolean hasImports = root != null && root.imports() != null && !root.imports().isEmpty();
    if (!hasImports) {
        copyResources(baseDir, resources, destDir, options);
        return;
    }
    Path tmpDir = null;
    try {
        tmpDir = IOUtils.createTempDir("concord-export");
        copyResources(baseDir, resources, tmpDir, options);
        Imports imports = importsNormalizer.normalize(root.imports());
        importManager.process(imports, tmpDir, listener);
        copyResources(tmpDir, resources, destDir, options);
    } finally {
        if (tmpDir != null) {
            IOUtils.deleteRecursively(tmpDir);
        }
    }
}

Example 2 with Imports

use of com.walmartlabs.concord.imports.Imports in project concord by walmartlabs.

the class ProjectLoaderV2 method load.

public Result load(Path baseDir, ImportsNormalizer importsNormalizer, ImportsListener listener) throws Exception {
    YamlParserV2 parser = new YamlParserV2();
    // load the initial ProcessDefinition from the root concord.yml file
    // it will be used to determine whether we need to load other resources (e.g. imports)
    ProcessDefinition root = loadRoot(parser, baseDir);
    List<Snapshot> snapshots = Collections.emptyList();
    if (root != null) {
        Imports imports = importsNormalizer.normalize(root.imports());
        snapshots = importManager.process(imports, baseDir, listener);
    }
    List<Path> files = loadResources(baseDir, root != null ? root.resources() : Resources.builder().build());
    Collections.sort(files);
    List<ProcessDefinition> definitions = new ArrayList<>();
    for (Path p : files) {
        definitions.add(parser.parse(baseDir, p));
    }
    if (root != null) {
        definitions.add(root);
    }
    if (definitions.isEmpty()) {
        throw new IllegalStateException("Can't find any Concord process definition files in '" + baseDir + "'");
    }
    return new Result(snapshots, merge(definitions));
}

Example 3 with Imports

use of com.walmartlabs.concord.imports.Imports in project concord by walmartlabs.

the class ProjectLoader method loadProject.

/**
 * Loads the project definition using the supplied path. Processes the configured
 * imports, templates and extra directories with project files.
 *
 * @param workDir       the directory containing the project files. If {@code imports} are
 *                      configured, the directory will be used as a target for repository
 *                      checkouts.
 */
public Result loadProject(Path workDir, ImportsNormalizer importsNormalizer, ImportsListener listener) throws Exception {
    workDir = workDir.normalize().toAbsolutePath();
    ProjectDefinition initial = initialLoad(workDir);
    Resources resources = initial.getResources();
    Imports imports = importsNormalizer.normalize(initial.getImports());
    List<Snapshot> snapshots = importManager.process(imports, workDir, listener);
    ProjectDefinitionBuilder b = new ProjectDefinitionBuilder(parser);
    List<String> projectPaths = resources.getProjectFilePaths();
    if (projectPaths != null) {
        for (String n : projectPaths) {
            Path p = assertLocal(workDir, workDir.resolve(n));
            if (Files.exists(p)) {
                b.addProjects(p);
            }
        }
    }
    for (String n : Constants.Files.PROJECT_ROOT_FILE_NAMES) {
        Path p = assertLocal(workDir, workDir.resolve(n));
        if (Files.exists(p)) {
            b.addProjectFile(workDir, p);
            break;
        }
    }
    List<String> definitionPaths = resources.getDefinitionPaths();
    if (definitionPaths != null) {
        for (String n : definitionPaths) {
            Path p = assertLocal(workDir, workDir.resolve(n));
            if (Files.exists(p)) {
                b.addDefinitions(p);
            }
        }
    }
    List<String> profilesPaths = resources.getProfilesPaths();
    if (profilesPaths != null) {
        for (String n : profilesPaths) {
            Path p = assertLocal(workDir, workDir.resolve(n));
            if (Files.exists(p)) {
                b.addProfiles(p);
            }
        }
    }
    ProjectDefinition pd = b.build();
    // save the normalized imports, so the exact same workDir structure
    // can be re-created later (e.g. by the Agent)
    pd = new ProjectDefinition(pd, imports);
    return new Result(snapshots, pd);
}

Example 4 with Imports

use of com.walmartlabs.concord.imports.Imports in project concord by walmartlabs.

the class Dispatcher method sendResponse.

private void sendResponse(Match match) {
    WebSocketChannel channel = match.request.channel;
    long correlationId = match.request.request.getCorrelationId();
    ProcessQueueEntry item = match.response;
    try {
        SecretReference secret = null;
        if (item.repoId() != null) {
            secret = dao.getSecretReference(item.repoId());
        }
        // backward compatibility with old process queue entries that are not normalized
        Imports imports = importsNormalizerFactory.forProject(item.projectId()).normalize(item.imports());
        ProcessResponse resp = new ProcessResponse(correlationId, sessionTokenCreator.create(item.key()), item.key().getInstanceId(), secret != null ? secret.orgName : null, item.repoUrl(), item.repoPath(), item.commitId(), item.commitBranch(), secret != null ? secret.secretName : null, imports);
        if (!channelManager.sendResponse(channel.getChannelId(), resp)) {
            log.warn("sendResponse ['{}'] -> failed", correlationId);
        }
        logManager.info(item.key(), "Acquired by: " + channel.getUserAgent());
    } catch (Exception e) {
        log.error("sendResponse ['{}'] -> failed (instanceId: {})", correlationId, item.key().getInstanceId());
    }
}

Example 5 with Imports

use of com.walmartlabs.concord.imports.Imports in project concord by walmartlabs.

the class ProcessQueueManager method enqueue.

/**
 * Updates an existing record, moving the process into the ENQUEUED status.
 */
public boolean enqueue(Payload payload) {
    ProcessKey processKey = payload.getProcessKey();
    ProcessStatus s = queueDao.getStatus(processKey);
    if (s == null) {
        throw new ProcessException(processKey, "Process not found: " + processKey);
    }
    if (s == ProcessStatus.CANCELLED) {
        // (e.g. it was a process in an "exclusive" group)
        return false;
    }
    if (!TO_ENQUEUED_STATUSES.contains(s)) {
        // something's wrong (e.g. someone tried to change the process' status directly in the DB and was unlucky)
        throw new ProcessException(processKey, "Invalid process status: " + s);
    }
    Set<String> tags = payload.getHeader(Payload.PROCESS_TAGS);
    OffsetDateTime startAt = PayloadUtils.getStartAt(payload);
    Map<String, Object> requirements = PayloadUtils.getRequirements(payload);
    Long processTimeout = getProcessTimeout(payload);
    Long suspendTimeout = getSuspendTimeout(payload);
    Set<String> handlers = payload.getHeader(Payload.PROCESS_HANDLERS);
    Map<String, Object> meta = getMeta(getCfg(payload));
    Imports imports = payload.getHeader(Payload.IMPORTS);
    ExclusiveMode exclusive = PayloadUtils.getExclusive(payload);
    String runtime = payload.getHeader(Payload.RUNTIME);
    List<String> dependencies = payload.getHeader(Payload.DEPENDENCIES);
    return queueDao.txResult(tx -> {
        boolean updated = queueDao.enqueue(tx, processKey, tags, startAt, requirements, processTimeout, handlers, meta, imports, exclusive, runtime, dependencies, suspendTimeout, TO_ENQUEUED_STATUSES);
        if (updated) {
            notifyStatusChange(tx, processKey, ProcessStatus.ENQUEUED);
        }
        return updated;
    });
}