use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.
the class MapReduceRuntimeService method setDecoderForStream.
private void setDecoderForStream(StreamInputFormatProvider streamProvider, Job job, Map<String, String> inputFormatConfiguration, Class<? extends Mapper> mapperClass) {
// For stream, we need to do two extra steps.
// 1. stream usage registration since it only happens on client side.
// 2. Infer the stream event decoder from Mapper/Reducer
TypeToken<?> mapperTypeToken = mapperClass == null ? null : resolveClass(mapperClass, Mapper.class);
Type inputValueType = getInputValueType(job.getConfiguration(), StreamEvent.class, mapperTypeToken);
streamProvider.setDecoderType(inputFormatConfiguration, inputValueType);
StreamId streamId = streamProvider.getStreamId();
try {
streamAdmin.register(ImmutableList.of(context.getProgram().getId()), streamId);
streamAdmin.addAccess(context.getProgram().getId().run(context.getRunId().getId()), streamId, AccessType.READ);
} catch (Exception e) {
LOG.warn("Failed to register usage {} -> {}", context.getProgram().getId(), streamId, e);
}
}
use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.
the class LineageAdmin method getRollupRelations.
private Multimap<RelationKey, Relation> getRollupRelations(Multimap<RelationKey, Relation> relations, Map<ProgramRunId, RunRecordMeta> runRecordMap, Map<String, ProgramRunId> workflowIdMap) throws NotFoundException {
Multimap<RelationKey, Relation> relationsNew = HashMultimap.create();
for (Map.Entry<RelationKey, Collection<Relation>> entry : relations.asMap().entrySet()) {
for (Relation relation : entry.getValue()) {
ProgramRunId workflowProgramRunId = getWorkflowProgramRunid(relation, runRecordMap, workflowIdMap);
if (workflowProgramRunId == null) {
relationsNew.put(entry.getKey(), relation);
} else {
ProgramId workflowProgramId = new ProgramId(workflowProgramRunId.getNamespace(), workflowProgramRunId.getApplication(), workflowProgramRunId.getType(), workflowProgramRunId.getProgram());
Relation workflowRelation;
NamespacedEntityId data = relation.getData();
if (data instanceof DatasetId) {
workflowRelation = new Relation((DatasetId) data, workflowProgramId, relation.getAccess(), RunIds.fromString(workflowProgramRunId.getRun()));
} else {
workflowRelation = new Relation((StreamId) data, workflowProgramId, relation.getAccess(), RunIds.fromString(workflowProgramRunId.getRun()));
}
relationsNew.put(entry.getKey(), workflowRelation);
}
}
}
return relationsNew;
}
use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.
the class LineageHandler method streamLineage.
@GET
@Path("/namespaces/{namespace-id}/streams/{stream-id}/lineage")
public void streamLineage(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("stream-id") String stream, @QueryParam("start") String startStr, @QueryParam("end") String endStr, @QueryParam("levels") @DefaultValue("10") int levels, @QueryParam("collapse") List<String> collapse, @QueryParam("rollup") String rollup) throws Exception {
checkLevels(levels);
TimeRange range = parseRange(startStr, endStr);
StreamId streamId = new StreamId(namespaceId, stream);
Lineage lineage = lineageAdmin.computeLineage(streamId, range.getStart(), range.getEnd(), levels, rollup);
responder.sendJson(HttpResponseStatus.OK, LineageSerializer.toLineageRecord(TimeUnit.MILLISECONDS.toSeconds(range.getStart()), TimeUnit.MILLISECONDS.toSeconds(range.getEnd()), lineage, getCollapseTypes(collapse)), LineageRecord.class, GSON);
}
use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.
the class MetadataHttpHandler method addStreamTags.
@POST
@Path("/namespaces/{namespace-id}/streams/{stream-id}/metadata/tags")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void addStreamTags(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("stream-id") String streamId) throws BadRequestException, NotFoundException {
StreamId stream = new StreamId(namespaceId, streamId);
metadataAdmin.addTags(stream, readArray(request));
responder.sendString(HttpResponseStatus.OK, String.format("Added tags to stream %s successfully.", stream));
}
use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.
the class MetadataHttpHandler method removeStreamMetadata.
@DELETE
@Path("/namespaces/{namespace-id}/streams/{stream-id}/metadata")
public void removeStreamMetadata(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("stream-id") String streamId) throws NotFoundException {
StreamId stream = new StreamId(namespaceId, streamId);
metadataAdmin.removeMetadata(stream);
responder.sendString(HttpResponseStatus.OK, String.format("Metadata for stream %s deleted successfully.", stream));
}
Aggregations