Examples with Id io.cdap.cdap.common.id.Id used on opensource projects

Example 11 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class DefaultArtifactRepository method addSystemArtifacts.

@Override
public void addSystemArtifacts() throws Exception {
    // scan the directory for artifact .jar files and config files for those artifacts
    Map<Id.Artifact, SystemArtifactInfo> systemArtifacts = new HashMap<>();
    for (File systemArtifactDir : systemArtifactDirs) {
        for (File jarFile : DirUtils.listFiles(systemArtifactDir, "jar")) {
            // parse id from filename
            Id.Artifact artifactId;
            try {
                artifactId = Id.Artifact.parse(Id.Namespace.SYSTEM, jarFile.getName());
            } catch (IllegalArgumentException e) {
                LOG.warn(String.format("Skipping system artifact '%s' because the name is invalid: ", e.getMessage()));
                continue;
            }
            // check for a corresponding .json config file
            String artifactFileName = jarFile.getName();
            String configFileName = artifactFileName.substring(0, artifactFileName.length() - ".jar".length()) + ".json";
            File configFile = new File(systemArtifactDir, configFileName);
            try {
                // read and parse the config file if it exists. Otherwise use an empty config with the artifact filename
                ArtifactConfig artifactConfig = configFile.isFile() ? configReader.read(artifactId.getNamespace(), configFile) : new ArtifactConfig();
                validateParentSet(artifactId, artifactConfig.getParents());
                validatePluginSet(artifactConfig.getPlugins());
                systemArtifacts.put(artifactId, new SystemArtifactInfo(artifactId, jarFile, artifactConfig));
            } catch (InvalidArtifactException e) {
                LOG.warn(String.format("Could not add system artifact '%s' because it is invalid.", artifactFileName), e);
            }
        }
    }
    // child -> parents
    Multimap<Id.Artifact, Id.Artifact> childToParents = HashMultimap.create();
    // parent -> children
    Multimap<Id.Artifact, Id.Artifact> parentToChildren = HashMultimap.create();
    Set<Id.Artifact> remainingArtifacts = new HashSet<>();
    // build mapping from child to parents and from parents to children
    for (SystemArtifactInfo child : systemArtifacts.values()) {
        Id.Artifact childId = child.getArtifactId();
        remainingArtifacts.add(childId);
        for (SystemArtifactInfo potentialParent : systemArtifacts.values()) {
            Id.Artifact potentialParentId = potentialParent.getArtifactId();
            // skip if we're looking at ourselves
            if (childId.equals(potentialParentId)) {
                continue;
            }
            if (child.getConfig().hasParent(potentialParentId)) {
                childToParents.put(childId, potentialParentId);
                parentToChildren.put(potentialParentId, childId);
            }
        }
    }
    if (!remainingArtifacts.isEmpty()) {
        ExecutorService executorService = Executors.newFixedThreadPool(Math.min(maxArtifactLoadParallelism, remainingArtifacts.size()), Threads.createDaemonThreadFactory("system-artifact-loader-%d"));
        try {
            // loop until there is no change
            boolean artifactsAdded = true;
            while (!remainingArtifacts.isEmpty() && artifactsAdded) {
                artifactsAdded = loadSystemArtifacts(executorService, systemArtifacts, remainingArtifacts, parentToChildren, childToParents);
            }
        } finally {
            executorService.shutdownNow();
        }
        if (!remainingArtifacts.isEmpty()) {
            LOG.warn("Unable to add system artifacts {} due to cyclic dependencies", Joiner.on(",").join(remainingArtifacts));
        }
    }
}

Example 12 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class SystemAppManagementServiceTest method testSystemAppManagementServiceE2E.

/**
 * Tests SystemAppManagementService's upgrade method end to end by running this scenario:
 * 1. Creates a system app config for an application into corresponding directory with artifact version VERSION1.
 * 2. Successfully read and load the config.
 * 3. Runs all steps to enable a system app , tests SystemAppEnableExecutor.
 * 4. Deploys the VERSION1 app and runs all programs corresponding to the app.
 * 5. Checks status of a continuously running program, i.e a service program.
 * 6. Updates system app config with app version upgraded to VERSION2.
 * 7. On restart of SystemAppManagementService, app should kill old running programs and start program again.
 */
@Test
public void testSystemAppManagementServiceE2E() throws Exception {
    systemConfigDir = TEMPORARY_FOLDER.newFolder("demo-sys-app-config-dir");
    cConf.set(Constants.SYSTEM_APP_CONFIG_DIR, systemConfigDir.getAbsolutePath());
    systemAppManagementService = new SystemAppManagementService(cConf, applicationLifecycleService, programLifecycleService);
    Id.Artifact artifactId1 = Id.Artifact.from(Id.Namespace.DEFAULT, "App", VERSION1);
    addAppArtifact(artifactId1, AllProgramsApp.class);
    createEnableSysAppConfigFile(artifactId1, "demo.json");
    systemAppManagementService.startUp();
    ApplicationId appId1 = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    ProgramId serviceId1 = appId1.program(ProgramType.SERVICE, AllProgramsApp.NoOpService.NAME);
    waitState(serviceId1, RUNNING);
    Assert.assertEquals(RUNNING, getProgramStatus(serviceId1));
    // Program shouldn't be killed first time it is started.
    assertProgramRuns(serviceId1, ProgramRunStatus.KILLED, 0);
    systemAppManagementService.shutDown();
    // New system app config with newer artifact version.
    Id.Artifact artifactId2 = Id.Artifact.from(Id.Namespace.DEFAULT, "App", VERSION2);
    addAppArtifact(artifactId2, AllProgramsApp.class);
    createEnableSysAppConfigFile(artifactId2, "demo.json");
    // SystemAppManagement restarts again.
    systemAppManagementService.startUp();
    // Program ID still stays the same.
    waitState(serviceId1, RUNNING);
    Assert.assertEquals(RUNNING, getProgramStatus(serviceId1));
    assertProgramRuns(serviceId1, ProgramRunStatus.KILLED, 1);
}

Example 13 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class ProgramLifecycleHttpHandler method stopPrograms.

/**
 * Stops all programs that are passed into the data. The data is an array of JSON objects
 * where each object must contain the following three elements: appId, programType, and programId
 * (flow name, service name, etc.).
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "MapReduce2"}]
 * </code></pre>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as a "statusCode" field which maps to the status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, App2 does not exist), then all JsonObjects for which the
 * parameters have a valid status will have the status field but all JsonObjects for which the parameters do not have
 * a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no App2 in the data above, then the response would be 200 OK with following possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1", "statusCode": 200},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 200}]
 * </code></pre>
 */
@POST
@Path("/stop")
@AuditPolicy({ AuditDetail.REQUEST_BODY, AuditDetail.RESPONSE_BODY })
public void stopPrograms(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ListenableFuture<BatchProgramResult>> issuedStops = new ArrayList<>(programs.size());
    for (final BatchProgram program : programs) {
        ProgramId programId = new ProgramId(namespaceId, program.getAppId(), program.getProgramType(), program.getProgramId());
        try {
            List<ListenableFuture<ProgramRunId>> stops = lifecycleService.issueStop(programId, null);
            for (ListenableFuture<ProgramRunId> stop : stops) {
                ListenableFuture<BatchProgramResult> issuedStop = Futures.transform(stop, (Function<ProgramRunId, BatchProgramResult>) input -> new BatchProgramResult(program, HttpResponseStatus.OK.code(), null, input.getRun()));
                issuedStops.add(issuedStop);
            }
        } catch (NotFoundException e) {
            issuedStops.add(Futures.immediateFuture(new BatchProgramResult(program, HttpResponseStatus.NOT_FOUND.code(), e.getMessage())));
        } catch (BadRequestException e) {
            issuedStops.add(Futures.immediateFuture(new BatchProgramResult(program, HttpResponseStatus.BAD_REQUEST.code(), e.getMessage())));
        }
    }
    List<BatchProgramResult> output = new ArrayList<>(programs.size());
    // need to keep this index in case there is an exception getting the future, since we won't have the program
    // information in that scenario
    int i = 0;
    for (ListenableFuture<BatchProgramResult> issuedStop : issuedStops) {
        try {
            output.add(issuedStop.get());
        } catch (Throwable t) {
            LOG.warn(t.getMessage(), t);
            output.add(new BatchProgramResult(programs.get(i), HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), t.getMessage()));
        }
        i++;
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(output));
}

Example 14 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class AppLifecycleHttpHandler method getAllApps.

/**
 * Returns a list of applications associated with a namespace.
 */
@GET
@Path("/apps")
public void getAllApps(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @QueryParam("artifactName") String artifactName, @QueryParam("artifactVersion") String artifactVersion, @QueryParam("pageToken") String pageToken, @QueryParam("pageSize") Integer pageSize, @QueryParam("orderBy") SortOrder orderBy, @QueryParam("nameFilter") String nameFilter) throws Exception {
    NamespaceId namespace = validateNamespace(namespaceId);
    Set<String> names = new HashSet<>();
    if (!Strings.isNullOrEmpty(artifactName)) {
        for (String name : Splitter.on(',').split(artifactName)) {
            names.add(name);
        }
    }
    if (Optional.ofNullable(pageSize).orElse(0) != 0) {
        JsonPaginatedListResponder.respond(GSON, responder, APP_LIST_PAGINATED_KEY, jsonListResponder -> {
            AtomicReference<ApplicationRecord> lastRecord = new AtomicReference<>(null);
            ScanApplicationsRequest scanRequest = getScanRequest(namespaceId, artifactVersion, pageToken, pageSize, orderBy, nameFilter, names);
            boolean pageLimitReached = applicationLifecycleService.scanApplications(scanRequest, appDetail -> {
                ApplicationRecord record = new ApplicationRecord(appDetail);
                jsonListResponder.send(record);
                lastRecord.set(record);
            });
            ApplicationRecord record = lastRecord.get();
            return !pageLimitReached || record == null ? null : record.getName() + EntityId.IDSTRING_PART_SEPARATOR + record.getAppVersion();
        });
    } else {
        ScanApplicationsRequest scanRequest = getScanRequest(namespaceId, artifactVersion, pageToken, null, orderBy, nameFilter, names);
        JsonWholeListResponder.respond(GSON, responder, jsonListResponder -> applicationLifecycleService.scanApplications(scanRequest, d -> jsonListResponder.send(new ApplicationRecord(d))));
    }
}

Example 15 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class ProgramLifecycleHttpHandlerTest method testProgramStartStopStatusErrors.

@Category(XSlowTests.class)
@Test
public void testProgramStartStopStatusErrors() throws Exception {
    // deploy, check the status
    deploy(AllProgramsApp.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    String appName = AllProgramsApp.NAME;
    String serviceName = AllProgramsApp.NoOpService.NAME;
    String mrName = AllProgramsApp.NoOpMR.NAME;
    // start unknown program
    startProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, "noexist"), 404);
    // start program in unknonw app
    startProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.SERVICE, serviceName), 404);
    // start program in unknown namespace
    startProgram(Id.Program.from("noexist", appName, ProgramType.SERVICE, serviceName), 404);
    // debug unknown program
    debugProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, "noexist"), 404);
    // debug a program that does not support it
    debugProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.MAPREDUCE, mrName), // not implemented
    501);
    // status for unknown program
    programStatus(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, "noexist"), 404);
    // status for program in unknonw app
    programStatus(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.SERVICE, serviceName), 404);
    // status for program in unknown namespace
    programStatus(Id.Program.from("noexist", appName, ProgramType.SERVICE, serviceName), 404);
    // stop unknown program
    stopProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, "noexist"), 404);
    // stop program in unknonw app
    stopProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.SERVICE, serviceName), 404);
    // stop program in unknown namespace
    stopProgram(Id.Program.from("noexist", appName, ProgramType.SERVICE, serviceName), 404);
    // stop program that is not running
    stopProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), 400);
    // stop run of a program with ill-formed run id
    stopProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), "norunid", 400);
    // start program twice
    startProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName));
    verifyProgramRuns(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), ProgramRunStatus.RUNNING);
    startProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), // conflict
    409);
    // get run records for later use
    List<RunRecord> runs = getProgramRuns(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), ProgramRunStatus.RUNNING);
    Assert.assertEquals(1, runs.size());
    String runId = runs.get(0).getPid();
    // stop program
    stopProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), 200);
    waitState(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), "STOPPED");
    // get run records again, should be empty now
    Tasks.waitFor(true, () -> {
        Id.Program id = Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName);
        return getProgramRuns(id, ProgramRunStatus.RUNNING).isEmpty();
    }, 10, TimeUnit.SECONDS);
    // stop run of the program that is not running
    stopProgram(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.SERVICE, serviceName), runId, // active run not found
    400);
    // cleanup
    HttpResponse response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1));
    Assert.assertEquals(200, response.getResponseCode());
}