Search in sources :

Example 1 with ProgramScheduleRecord

use of co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord in project cdap by caskdata.

the class ProgramScheduleStoreDataset method getScheduleRecord.

/**
 * Read all information about a schedule from the store.
 *
 * @param scheduleId the id of the schedule to read
 * @return the schedule record from the store
 * @throws NotFoundException if the schedule does not exist in the store
 */
public ProgramScheduleRecord getScheduleRecord(ScheduleId scheduleId) throws NotFoundException {
    Row row = store.get(new Get(rowKeyForSchedule(scheduleId)));
    byte[] serialized = row.get(SCHEDULE_COLUMN_BYTES);
    if (serialized == null) {
        throw new NotFoundException(scheduleId);
    }
    ProgramSchedule schedule = GSON.fromJson(Bytes.toString(serialized), ProgramSchedule.class);
    ProgramScheduleMeta meta = extractMetaFromRow(scheduleId, row);
    return new ProgramScheduleRecord(schedule, meta);
}
Also used : ProgramScheduleMeta(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleMeta) ProgramSchedule(co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule) Get(co.cask.cdap.api.dataset.table.Get) NotFoundException(co.cask.cdap.common.NotFoundException) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) Row(co.cask.cdap.api.dataset.table.Row)

Example 2 with ProgramScheduleRecord

use of co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord in project cdap by caskdata.

the class ProgramScheduleStoreDataset method modifySchedulesTriggeredByDeletedProgram.

/**
 * Update all schedules that can be triggered by the given deleted program. A schedule will be removed if
 * the only {@link ProgramStatusTrigger} in it is triggered by the deleted program. Schedules with composite triggers
 * will be updated if the composite trigger can still be satisfied after the program is deleted, otherwise the
 * schedules will be deleted.
 *
 * @param programId the program id for which to delete the schedules
 * @return the IDs of the schedules that were deleted
 */
public List<ScheduleId> modifySchedulesTriggeredByDeletedProgram(ProgramId programId) {
    List<ScheduleId> deleted = new ArrayList<>();
    Set<ProgramScheduleRecord> scheduleRecords = new HashSet<>();
    for (ProgramStatus status : ProgramStatus.values()) {
        scheduleRecords.addAll(findSchedules(Schedulers.triggerKeyForProgramStatus(programId, status)));
    }
    for (ProgramScheduleRecord scheduleRecord : scheduleRecords) {
        ProgramSchedule schedule = scheduleRecord.getSchedule();
        try {
            deleteSchedule(schedule.getScheduleId());
        } catch (NotFoundException e) {
            // this should never happen
            LOG.warn("Failed to delete the schedule '{}' triggered by '{}', skip this schedule.", schedule.getScheduleId(), programId, e);
            continue;
        }
        if (schedule.getTrigger() instanceof AbstractSatisfiableCompositeTrigger) {
            // get the updated composite trigger by removing the program status trigger of the given program
            Trigger updatedTrigger = ((AbstractSatisfiableCompositeTrigger) schedule.getTrigger()).getTriggerWithDeletedProgram(programId);
            if (updatedTrigger == null) {
                deleted.add(schedule.getScheduleId());
                continue;
            }
            // if the updated composite trigger is not null, add the schedule back with updated composite trigger
            try {
                addScheduleWithStatus(new ProgramSchedule(schedule.getName(), schedule.getDescription(), schedule.getProgramId(), schedule.getProperties(), updatedTrigger, schedule.getConstraints(), schedule.getTimeoutMillis()), scheduleRecord.getMeta().getStatus(), System.currentTimeMillis());
            } catch (AlreadyExistsException e) {
                // this should never happen
                LOG.warn("Failed to add the schedule '{}' triggered by '{}' with updated trigger '{}', " + "skip adding this schedule.", schedule.getScheduleId(), programId, updatedTrigger, e);
            }
        } else {
            deleted.add(schedule.getScheduleId());
        }
    }
    return deleted;
}
Also used : SatisfiableTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger) Trigger(co.cask.cdap.api.schedule.Trigger) AbstractSatisfiableCompositeTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) ProgramStatusTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger) AlreadyExistsException(co.cask.cdap.common.AlreadyExistsException) ProgramSchedule(co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule) ArrayList(java.util.ArrayList) AbstractSatisfiableCompositeTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) NotFoundException(co.cask.cdap.common.NotFoundException) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) ScheduleId(co.cask.cdap.proto.id.ScheduleId) HashSet(java.util.HashSet) ProgramStatus(co.cask.cdap.api.ProgramStatus)

Example 3 with ProgramScheduleRecord

use of co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method doGetSchedules.

protected void doGetSchedules(HttpResponder responder, ApplicationId applicationId, @Nullable String workflow, @Nullable String triggerTypeStr, @Nullable String statusStr) throws Exception {
    ApplicationSpecification appSpec = store.getApplication(applicationId);
    if (appSpec == null) {
        throw new NotFoundException(applicationId);
    }
    ProgramScheduleStatus status;
    try {
        status = statusStr == null ? null : ProgramScheduleStatus.valueOf(statusStr);
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(String.format("Invalid schedule status '%s'. Must be one of %s.", statusStr, Joiner.on(',').join(ProgramScheduleStatus.values())), e);
    }
    ProtoTrigger.Type triggerType;
    try {
        triggerType = triggerTypeStr == null ? null : ProtoTrigger.Type.valueOfCategoryName(triggerTypeStr);
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e.getMessage(), e);
    }
    Predicate<ProgramScheduleRecord> predicate = record -> true;
    if (status != null) {
        predicate = predicate.and(record -> record.getMeta().getStatus().equals(status));
    }
    if (triggerType != null) {
        predicate = predicate.and(record -> record.getSchedule().getTrigger().getType().equals(triggerType));
    }
    Collection<ProgramScheduleRecord> schedules;
    if (workflow != null) {
        WorkflowId workflowId = applicationId.workflow(workflow);
        if (appSpec.getWorkflows().get(workflow) == null) {
            throw new NotFoundException(workflowId);
        }
        schedules = programScheduleService.list(workflowId, predicate);
    } else {
        schedules = programScheduleService.list(applicationId, predicate);
    }
    List<ScheduleDetail> details = schedules.stream().map(ProgramScheduleRecord::toScheduleDetail).collect(Collectors.toList());
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(details, Schedulers.SCHEDULE_DETAILS_TYPE));
}
Also used : BatchProgramResult(co.cask.cdap.proto.BatchProgramResult) TypeToken(com.google.gson.reflect.TypeToken) RunRecordMeta(co.cask.cdap.internal.app.store.RunRecordMeta) GsonBuilder(com.google.gson.GsonBuilder) ProgramType(co.cask.cdap.proto.ProgramType) Map(java.util.Map) ConstraintCodec(co.cask.cdap.internal.app.runtime.schedule.constraint.ConstraintCodec) Constraint(co.cask.cdap.internal.schedule.constraint.Constraint) ProgramId(co.cask.cdap.proto.id.ProgramId) EnumSet(java.util.EnumSet) Schedulers(co.cask.cdap.internal.app.runtime.schedule.store.Schedulers) HttpRequest(io.netty.handler.codec.http.HttpRequest) BatchRunnable(co.cask.cdap.proto.BatchRunnable) Set(java.util.Set) FlowSpecification(co.cask.cdap.api.flow.FlowSpecification) Reader(java.io.Reader) FullHttpRequest(io.netty.handler.codec.http.FullHttpRequest) ConflictException(co.cask.cdap.common.ConflictException) StandardCharsets(java.nio.charset.StandardCharsets) ProgramSpecification(co.cask.cdap.api.ProgramSpecification) FlowUtils(co.cask.cdap.internal.app.runtime.flow.FlowUtils) ProgramSchedule(co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule) Containers(co.cask.cdap.proto.Containers) BadRequestException(co.cask.cdap.common.BadRequestException) UnauthorizedException(co.cask.cdap.security.spi.authorization.UnauthorizedException) Joiner(com.google.common.base.Joiner) Singleton(com.google.inject.Singleton) BatchRunnableInstances(co.cask.cdap.proto.BatchRunnableInstances) NotRunningProgramLiveInfo(co.cask.cdap.proto.NotRunningProgramLiveInfo) CaseInsensitiveEnumTypeAdapterFactory(co.cask.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory) NamespaceId(co.cask.cdap.proto.id.NamespaceId) GET(javax.ws.rs.GET) ProgramScheduleStatus(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleStatus) SatisfiableTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger) ArrayList(java.util.ArrayList) AuditPolicy(co.cask.cdap.common.security.AuditPolicy) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) ProgramRunStatus(co.cask.cdap.proto.ProgramRunStatus) Store(co.cask.cdap.app.store.Store) ProgramStatus(co.cask.cdap.proto.ProgramStatus) Constants(co.cask.cdap.common.conf.Constants) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) MRJobInfoFetcher(co.cask.cdap.app.mapreduce.MRJobInfoFetcher) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) AbstractAppFabricHttpHandler(co.cask.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler) ApplicationId(co.cask.cdap.proto.id.ApplicationId) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) InputStreamReader(java.io.InputStreamReader) Futures(com.google.common.util.concurrent.Futures) Instances(co.cask.cdap.proto.Instances) FlowletDefinition(co.cask.cdap.api.flow.FlowletDefinition) HttpResponder(co.cask.http.HttpResponder) AuditDetail(co.cask.cdap.common.security.AuditDetail) JsonObject(com.google.gson.JsonObject) WorkflowId(co.cask.cdap.proto.id.WorkflowId) MethodNotAllowedException(co.cask.cdap.common.MethodNotAllowedException) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) NotImplementedException(co.cask.cdap.common.NotImplementedException) Trigger(co.cask.cdap.api.schedule.Trigger) ProtoTrigger(co.cask.cdap.proto.ProtoTrigger) QueryParam(javax.ws.rs.QueryParam) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) Gson(com.google.gson.Gson) DefaultValue(javax.ws.rs.DefaultValue) ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) EndpointStrategy(co.cask.cdap.common.discovery.EndpointStrategy) FlowId(co.cask.cdap.proto.id.FlowId) MetricStore(co.cask.cdap.api.metrics.MetricStore) Objects(com.google.common.base.Objects) ProgramStatusTrigger(co.cask.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger) ProgramRuntimeService(co.cask.cdap.app.runtime.ProgramRuntimeService) DELETE(javax.ws.rs.DELETE) NamespaceNotFoundException(co.cask.cdap.common.NamespaceNotFoundException) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) Function(com.google.common.base.Function) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) HttpResponseStatus(io.netty.handler.codec.http.HttpResponseStatus) ServiceDiscoverable(co.cask.cdap.common.service.ServiceDiscoverable) ProgramRecord(co.cask.cdap.proto.ProgramRecord) Collectors(java.util.stream.Collectors) Id(co.cask.cdap.common.id.Id) MRJobInfo(co.cask.cdap.proto.MRJobInfo) List(java.util.List) Type(java.lang.reflect.Type) ServiceInstances(co.cask.cdap.proto.ServiceInstances) ProgramScheduleService(co.cask.cdap.scheduler.ProgramScheduleService) BatchProgramStatus(co.cask.cdap.proto.BatchProgramStatus) BatchProgramStart(co.cask.cdap.proto.BatchProgramStart) PathParam(javax.ws.rs.PathParam) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) ProgramController(co.cask.cdap.app.runtime.ProgramController) BatchProgram(co.cask.cdap.proto.BatchProgram) ProgramLiveInfo(co.cask.cdap.proto.ProgramLiveInfo) HashMap(java.util.HashMap) ProgramLifecycleService(co.cask.cdap.internal.app.services.ProgramLifecycleService) JsonElement(com.google.gson.JsonElement) HashSet(java.util.HashSet) QueueAdmin(co.cask.cdap.data2.transaction.queue.QueueAdmin) TriggerCodec(co.cask.cdap.internal.app.runtime.schedule.trigger.TriggerCodec) ScheduleId(co.cask.cdap.proto.id.ScheduleId) ScheduleDetail(co.cask.cdap.proto.ScheduleDetail) ByteBufInputStream(io.netty.buffer.ByteBufInputStream) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) JsonSyntaxException(com.google.gson.JsonSyntaxException) RunRecord(co.cask.cdap.proto.RunRecord) TimeUnit(java.util.concurrent.TimeUnit) ApplicationSpecificationAdapter(co.cask.cdap.internal.app.ApplicationSpecificationAdapter) NotFoundException(co.cask.cdap.common.NotFoundException) PUT(javax.ws.rs.PUT) Collections(java.util.Collections) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) NamespaceNotFoundException(co.cask.cdap.common.NamespaceNotFoundException) NotFoundException(co.cask.cdap.common.NotFoundException) WorkflowId(co.cask.cdap.proto.id.WorkflowId) ProgramScheduleStatus(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleStatus) ProtoTrigger(co.cask.cdap.proto.ProtoTrigger) BadRequestException(co.cask.cdap.common.BadRequestException) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) ScheduleDetail(co.cask.cdap.proto.ScheduleDetail)

Example 4 with ProgramScheduleRecord

use of co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord in project cdap by caskdata.

the class ProgramScheduleStoreDataset method findSchedules.

/**
 * Find all schedules that have a trigger with a given trigger key.
 *
 * @param triggerKey the trigger key to look up
 * @return a list of all schedules that are triggered by this key; never null
 */
public Collection<ProgramScheduleRecord> findSchedules(String triggerKey) {
    Map<ScheduleId, ProgramScheduleRecord> schedulesFound = new HashMap<>();
    try (Scanner scanner = store.readByIndex(TRIGGER_KEY_COLUMN_BYTES, Bytes.toBytes(triggerKey))) {
        Row triggerRow;
        while ((triggerRow = scanner.next()) != null) {
            String triggerRowKey = Bytes.toString(triggerRow.getRow());
            try {
                ScheduleId scheduleId = extractScheduleIdFromTriggerKey(triggerRowKey);
                if (schedulesFound.containsKey(scheduleId)) {
                    continue;
                }
                Row row = store.get(new Get(rowKeyForSchedule(scheduleId)));
                byte[] serialized = row.get(SCHEDULE_COLUMN_BYTES);
                if (serialized == null) {
                    throw new NotFoundException(scheduleId);
                }
                ProgramSchedule schedule = GSON.fromJson(Bytes.toString(serialized), ProgramSchedule.class);
                ProgramScheduleMeta meta = extractMetaFromRow(scheduleId, row);
                ProgramScheduleRecord record = new ProgramScheduleRecord(schedule, meta);
                schedulesFound.put(scheduleId, record);
            } catch (IllegalArgumentException | NotFoundException e) {
                // the only exceptions we know to be thrown here are IllegalArgumentException (ill-formed key) or
                // NotFoundException (if the schedule does not exist). Both should never happen, so we warn and ignore.
                // we will let any other exception propagate up, because it would be a DataSetException or similarly serious.
                LOG.warn("Problem with trigger id '{}' found for trigger key '{}': {}. Skipping entry.", triggerRowKey, triggerKey, e.getMessage());
            }
        }
    }
    return schedulesFound.values();
}
Also used : Scanner(co.cask.cdap.api.dataset.table.Scanner) ProgramScheduleMeta(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleMeta) HashMap(java.util.HashMap) NotFoundException(co.cask.cdap.common.NotFoundException) ScheduleId(co.cask.cdap.proto.id.ScheduleId) ProgramSchedule(co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule) Get(co.cask.cdap.api.dataset.table.Get) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) Row(co.cask.cdap.api.dataset.table.Row)

Example 5 with ProgramScheduleRecord

use of co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord in project cdap by caskdata.

the class ProgramScheduleStoreDataset method listScheduleRecords.

/**
 * List schedule records in a given application and if the programId is not null, only return the schedules
 * which can launch the given program
 */
private List<ProgramScheduleRecord> listScheduleRecords(ApplicationId appId, @Nullable ProgramId programId) {
    List<ProgramScheduleRecord> result = new ArrayList<>();
    byte[] prefix = keyPrefixForApplicationScan(appId);
    try (Scanner scanner = store.scan(new Scan(prefix, Bytes.stopKeyForPrefix(prefix)))) {
        Row row;
        while ((row = scanner.next()) != null) {
            byte[] serialized = row.get(SCHEDULE_COLUMN_BYTES);
            if (serialized != null) {
                ProgramSchedule schedule = GSON.fromJson(Bytes.toString(serialized), ProgramSchedule.class);
                if (programId == null || programId.equals(schedule.getProgramId())) {
                    result.add(new ProgramScheduleRecord(schedule, extractMetaFromRow(schedule.getScheduleId(), row)));
                }
            }
        }
    }
    return result;
}
Also used : Scanner(co.cask.cdap.api.dataset.table.Scanner) ProgramSchedule(co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule) ArrayList(java.util.ArrayList) ProgramScheduleRecord(co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) Scan(co.cask.cdap.api.dataset.table.Scan) Row(co.cask.cdap.api.dataset.table.Row)

Aggregations

ProgramSchedule (co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule)5 ProgramScheduleRecord (co.cask.cdap.internal.app.runtime.schedule.ProgramScheduleRecord)5 NotFoundException (co.cask.cdap.common.NotFoundException)4 Row (co.cask.cdap.api.dataset.table.Row)3 ScheduleId (co.cask.cdap.proto.id.ScheduleId)3 ArrayList (java.util.ArrayList)3 Get (co.cask.cdap.api.dataset.table.Get)2 Scanner (co.cask.cdap.api.dataset.table.Scanner)2 Trigger (co.cask.cdap.api.schedule.Trigger)2 ProgramStatusTrigger (co.cask.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger)2 SatisfiableTrigger (co.cask.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger)2 HashSet (java.util.HashSet)2 ProgramSpecification (co.cask.cdap.api.ProgramSpecification)1 ProgramStatus (co.cask.cdap.api.ProgramStatus)1 ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)1 Scan (co.cask.cdap.api.dataset.table.Scan)1 FlowSpecification (co.cask.cdap.api.flow.FlowSpecification)1 FlowletDefinition (co.cask.cdap.api.flow.FlowletDefinition)1 MetricStore (co.cask.cdap.api.metrics.MetricStore)1 MRJobInfoFetcher (co.cask.cdap.app.mapreduce.MRJobInfoFetcher)1