Examples with ArtifactSummary io.cdap.cdap.api.artifact.ArtifactSummary used on opensource projects

Example 21 with ArtifactSummary

use of io.cdap.cdap.api.artifact.ArtifactSummary in project cdap by caskdata.

the class CoreSchedulerServiceTest method testRunScheduledJobs.

@Test
@Category(XSlowTests.class)
public void testRunScheduledJobs() throws Exception {
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    dataEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.Dataset.DATA_EVENT_TOPIC));
    // Deploy the app with version
    Id.Artifact appArtifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "appwithschedules", VERSION1);
    addAppArtifact(appArtifactId, AppWithFrequentScheduledWorkflows.class);
    AppRequest<? extends Config> appRequest = new AppRequest<>(new ArtifactSummary(appArtifactId.getName(), appArtifactId.getVersion().getVersion()));
    deploy(APP_ID, appRequest);
    // Resume the schedule because schedules are initialized as paused
    enableSchedule(AppWithFrequentScheduledWorkflows.TEN_SECOND_SCHEDULE_1);
    enableSchedule(AppWithFrequentScheduledWorkflows.TEN_SECOND_SCHEDULE_2);
    enableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_1);
    enableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    for (int i = 0; i < 5; i++) {
        testNewPartition(i + 1);
    }
    // Enable COMPOSITE_SCHEDULE before publishing events to DATASET_NAME2
    enableSchedule(AppWithFrequentScheduledWorkflows.COMPOSITE_SCHEDULE);
    // disable the two partition schedules, send them notifications (but they should not trigger)
    int runs1 = getRuns(WORKFLOW_1, ProgramRunStatus.ALL);
    int runs2 = getRuns(WORKFLOW_2, ProgramRunStatus.ALL);
    disableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_1);
    // ensure schedule 2 is disabled after schedule 1
    Thread.sleep(BUFFER);
    long disableBeforeTime = System.currentTimeMillis();
    disableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    long disableAfterTime = System.currentTimeMillis() + 1;
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME1);
    long minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    // This would make sure the subscriber has processed the data event
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // Both workflows must run at least once.
    // If the testNewPartition() loop took longer than expected, it may be more (quartz fired multiple times)
    Tasks.waitFor(true, () -> getRuns(SCHEDULED_WORKFLOW_1, ProgramRunStatus.COMPLETED) > 0 && getRuns(SCHEDULED_WORKFLOW_2, ProgramRunStatus.COMPLETED) > 0, 10, TimeUnit.SECONDS);
    // There shouldn't be any partition trigger in the job queue
    Assert.assertFalse(Iterables.any(getAllJobs(), job -> job.getSchedule().getTrigger() instanceof ProtoTrigger.PartitionTrigger));
    ProgramId compositeWorkflow = APP_ID.workflow(AppWithFrequentScheduledWorkflows.COMPOSITE_WORKFLOW);
    // Workflow scheduled with the composite trigger has never been started
    Assert.assertEquals(0, getRuns(compositeWorkflow, ProgramRunStatus.ALL));
    // Publish two more new partition notifications to satisfy the partition trigger in the composite trigger,
    // and thus the whole composite trigger will be satisfied
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    // This would make sure the subscriber has processed the data event
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // Wait for 1 run to complete for compositeWorkflow
    waitForCompleteRuns(1, compositeWorkflow);
    for (RunRecordDetail runRecordMeta : store.getRuns(SCHEDULED_WORKFLOW_1, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).values()) {
        Map<String, String> sysArgs = runRecordMeta.getSystemArgs();
        Assert.assertNotNull(sysArgs);
        TriggeringScheduleInfo scheduleInfo = GSON.fromJson(sysArgs.get(ProgramOptionConstants.TRIGGERING_SCHEDULE_INFO), TriggeringScheduleInfo.class);
        Assert.assertEquals(AppWithFrequentScheduledWorkflows.TEN_SECOND_SCHEDULE_1, scheduleInfo.getName());
        List<TriggerInfo> triggerInfos = scheduleInfo.getTriggerInfos();
        // Only one notification is enough to satisfy Time Trigger
        Assert.assertEquals(1, triggerInfos.size());
        Assert.assertEquals(TriggerInfo.Type.TIME, triggerInfos.get(0).getType());
    }
    // Also verify that the two partition schedules did not trigger
    Assert.assertEquals(runs1, getRuns(WORKFLOW_1, ProgramRunStatus.ALL));
    Assert.assertEquals(runs2, getRuns(WORKFLOW_2, ProgramRunStatus.ALL));
    // enable partition schedule 2 and test reEnableSchedules
    scheduler.reEnableSchedules(NamespaceId.DEFAULT, disableBeforeTime, disableAfterTime);
    Assert.assertEquals(ProgramScheduleStatus.SCHEDULED, scheduler.getScheduleStatus(APP_ID.schedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2)));
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED, scheduler.getScheduleStatus(APP_ID.schedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_1)));
    testScheduleUpdate("disable");
    testScheduleUpdate("update");
    testScheduleUpdate("delete");
}

Example 22 with ArtifactSummary

use of io.cdap.cdap.api.artifact.ArtifactSummary in project cdap by caskdata.

the class DraftStore method fromRow.

@SuppressWarnings("ConstantConditions")
private Draft fromRow(StructuredRow row, boolean includeConfig) {
    String id = row.getString(ID_COL);
    String name = row.getString(NAME_COL);
    String description = row.getString(DESCRIPTION_COL);
    long createTime = row.getLong(CREATED_COL);
    long updateTime = row.getLong(UPDATED_COL);
    String artifactStr = row.getString(ARTIFACT_COL);
    ArtifactSummary artifact = GSON.fromJson(artifactStr, ArtifactSummary.class);
    String configStr = row.getString(PIPELINE_COL);
    ETLConfig config = null;
    if (includeConfig) {
        if (StudioUtil.isBatchPipeline(artifact)) {
            config = GSON.fromJson(configStr, ETLBatchConfig.class);
        } else if (StudioUtil.isStreamingPipeline(artifact)) {
            config = GSON.fromJson(configStr, DataStreamsConfig.class);
        } else {
            throw new IllegalArgumentException(String.format("Failed to parse pipeline config string: %s is not a supported pipeline type", artifact.getName()));
        }
    }
    return new Draft(config, name, description, artifact, id, createTime, updateTime);
}

Example 23 with ArtifactSummary

use of io.cdap.cdap.api.artifact.ArtifactSummary in project cdap by caskdata.

the class SystemProgramManagementServiceTest method deployTestApp.

private void deployTestApp() throws Exception {
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.SYSTEM, APP_NAME, VERSION);
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, APP_CLASS);
    File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
    Locations.linkOrCopyOverwrite(appJar, appJarFile);
    appJar.delete();
    artifactRepository.addArtifact(artifactId, appJarFile);
    ArtifactSummary summary = new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion(), ArtifactScope.SYSTEM);
    applicationLifecycleService.deployApp(NamespaceId.SYSTEM, APP_NAME, VERSION, summary, null, programId -> {
    // no-op
    }, null, false, false, Collections.emptyMap());
}

Example 24 with ArtifactSummary

use of io.cdap.cdap.api.artifact.ArtifactSummary in project cdap by caskdata.

the class AppLifecycleHttpHandlerTest method testDeployUsingNonexistantArtifact404.

@Test
public void testDeployUsingNonexistantArtifact404() throws Exception {
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, "badapp");
    AppRequest<Config> appRequest = new AppRequest<>(new ArtifactSummary("something", "1.0.0"), null);
    HttpResponse response = deploy(appId, appRequest);
    Assert.assertEquals(404, response.getResponseCode());
}

Example 25 with ArtifactSummary

use of io.cdap.cdap.api.artifact.ArtifactSummary in project cdap by caskdata.

the class AppLifecycleHttpHandlerTest method testDeployUsingArtifact.

@Test
public void testDeployUsingArtifact() throws Exception {
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "configapp", "1.0.0");
    addAppArtifact(artifactId, ConfigTestApp.class);
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, "cfgApp");
    ConfigTestApp.ConfigClass config = new ConfigTestApp.ConfigClass("abc", "def");
    AppRequest<ConfigTestApp.ConfigClass> request = new AppRequest<>(new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion()), config);
    Assert.assertEquals(200, deploy(appId, request).getResponseCode());
    JsonObject appDetails = getAppDetails(Id.Namespace.DEFAULT.getId(), appId.getId());
    Assert.assertEquals(GSON.toJson(config), appDetails.get("configuration").getAsString());
    Assert.assertEquals(200, doDelete(getVersionedAPIPath("apps/" + appId.getId(), appId.getNamespaceId())).getResponseCode());
}