Search in sources :

Example 1 with ProgramContext

use of co.cask.cdap.data.ProgramContext in project cdap by caskdata.

the class LineageWriterDatasetFramework method doWriteLineage.

private void doWriteLineage(DatasetId datasetInstanceId, AccessType accessType) {
    ProgramContext programContext = this.programContext;
    if (programContext != null) {
        ProgramRunId programRunId = programContext.getProgramRunId();
        NamespacedEntityId componentId = programContext.getComponentId();
        try {
            lineageWriter.addAccess(programRunId, datasetInstanceId, accessType, componentId);
        } catch (Throwable t) {
            // Failure to write to lineage shouldn't cause dataset operation failure
            LOG.warn("Failed to write lineage information for dataset {} with access type {} from {},{}", datasetInstanceId, accessType, programRunId, componentId);
            // Log the stacktrace as debug to not polluting the log
            LOG.debug("Cause for lineage writing failure for {} {} {} {}", datasetInstanceId, accessType, programRunId, componentId, t);
        }
    }
}
Also used : NamespacedEntityId(co.cask.cdap.proto.id.NamespacedEntityId) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) ProgramContext(co.cask.cdap.data.ProgramContext)

Example 2 with ProgramContext

use of co.cask.cdap.data.ProgramContext in project cdap by caskdata.

the class LineageWriterDataFabricFacade method createStreamConsumer.

@Override
public StreamConsumer createStreamConsumer(StreamId streamName, ConsumerConfig consumerConfig) throws IOException {
    String namespace = String.format("%s.%s", programId.getApplication(), programId.getProgram());
    final StreamConsumer consumer = streamConsumerFactory.create(streamName, namespace, consumerConfig);
    datasetCache.addExtraTransactionAware(consumer);
    ProgramContext programContext = this.programContext;
    if (programContext != null) {
        lineageWriter.addAccess(programContext.getProgramRunId(), streamName, AccessType.READ, programContext.getComponentId());
    }
    return new ForwardingStreamConsumer(consumer) {

        @Override
        public void close() throws IOException {
            super.close();
            datasetCache.removeExtraTransactionAware(consumer);
        }
    };
}
Also used : ForwardingStreamConsumer(co.cask.cdap.data2.transaction.stream.ForwardingStreamConsumer) StreamConsumer(co.cask.cdap.data2.transaction.stream.StreamConsumer) ProgramContext(co.cask.cdap.data.ProgramContext) ForwardingStreamConsumer(co.cask.cdap.data2.transaction.stream.ForwardingStreamConsumer)

Example 3 with ProgramContext

use of co.cask.cdap.data.ProgramContext in project cdap by caskdata.

the class FlowletProgramRunner method run.

@SuppressWarnings("unchecked")
@Override
public ProgramController run(Program program, ProgramOptions options) {
    BasicFlowletContext flowletContext = null;
    try {
        // Extract and verify parameters
        String flowletName = options.getProgramId().getProgram();
        int instanceId = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
        Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
        int instanceCount = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
        Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
        RunId runId = ProgramRunners.getRunId(options);
        ApplicationSpecification appSpec = program.getApplicationSpecification();
        Preconditions.checkNotNull(appSpec, "Missing application specification.");
        ProgramType processorType = program.getType();
        Preconditions.checkNotNull(processorType, "Missing processor type.");
        Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
        String processorName = program.getName();
        Preconditions.checkNotNull(processorName, "Missing processor name.");
        FlowSpecification flowSpec = appSpec.getFlows().get(processorName);
        FlowletDefinition flowletDef = flowSpec.getFlowlets().get(flowletName);
        Preconditions.checkNotNull(flowletDef, "Definition missing for flowlet \"%s\"", flowletName);
        Class<?> clz = Class.forName(flowletDef.getFlowletSpec().getClassName(), true, program.getClassLoader());
        Preconditions.checkArgument(Flowlet.class.isAssignableFrom(clz), "%s is not a Flowlet.", clz);
        // Setup dataset framework context, if required
        ProgramId programId = program.getId();
        FlowletId flowletId = programId.flowlet(flowletName);
        ProgramRunId run = programId.run(runId);
        ProgramContext programContext = new BasicProgramContext(run, flowletId);
        if (dsFramework instanceof ProgramContextAware) {
            ((ProgramContextAware) dsFramework).setContext(programContext);
        }
        Class<? extends Flowlet> flowletClass = (Class<? extends Flowlet>) clz;
        // Creates flowlet context
        flowletContext = new BasicFlowletContext(program, options, flowletId, instanceId, instanceCount, flowletDef.getDatasets(), flowletDef.getFlowletSpec(), metricsCollectionService, discoveryServiceClient, txClient, dsFramework, secureStore, secureStoreManager, messageService, cConf);
        // Creates tx related objects
        DataFabricFacade dataFabricFacade = dataFabricFacadeFactory.create(program, flowletContext.getDatasetCache());
        if (dataFabricFacade instanceof ProgramContextAware) {
            ((ProgramContextAware) dataFabricFacade).setContext(programContext);
        }
        // Creates QueueSpecification
        Table<Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(new ApplicationId(program.getNamespaceId(), program.getApplicationId())).create(flowSpec);
        Flowlet flowlet = new InstantiatorFactory(false).get(TypeToken.of(flowletClass)).create();
        TypeToken<? extends Flowlet> flowletType = TypeToken.of(flowletClass);
        // Set the context classloader to the cdap classloader. It is needed for the DatumWriterFactory be able
        // to load cdap classes
        Thread.currentThread().setContextClassLoader(FlowletProgramRunner.class.getClassLoader());
        // Inject DataSet, OutputEmitter, Metric fields
        ImmutableList.Builder<ProducerSupplier> queueProducerSupplierBuilder = ImmutableList.builder();
        Reflections.visit(flowlet, flowlet.getClass(), new PropertyFieldSetter(flowletDef.getFlowletSpec().getProperties()), new DataSetFieldSetter(flowletContext), new MetricsFieldSetter(flowletContext.getMetrics()), new OutputEmitterFieldSetter(outputEmitterFactory(flowletContext, flowletName, dataFabricFacade, queueProducerSupplierBuilder, queueSpecs)));
        ImmutableList.Builder<ConsumerSupplier<?>> queueConsumerSupplierBuilder = ImmutableList.builder();
        Collection<ProcessSpecification<?>> processSpecs = createProcessSpecification(flowletContext, flowletType, processMethodFactory(flowlet), processSpecificationFactory(flowletContext, dataFabricFacade, queueReaderFactory, flowletName, queueSpecs, queueConsumerSupplierBuilder, createSchemaCache(program)), Lists.<ProcessSpecification<?>>newLinkedList());
        List<ConsumerSupplier<?>> consumerSuppliers = queueConsumerSupplierBuilder.build();
        // Create the flowlet driver
        AtomicReference<FlowletProgramController> controllerRef = new AtomicReference<>();
        Service serviceHook = createServiceHook(flowletName, consumerSuppliers, controllerRef);
        FlowletRuntimeService driver = new FlowletRuntimeService(flowlet, flowletContext, processSpecs, createCallback(flowlet, flowletDef.getFlowletSpec()), dataFabricFacade, serviceHook);
        FlowletProgramController controller = new FlowletProgramController(flowletName, flowletContext, driver, queueProducerSupplierBuilder.build(), consumerSuppliers);
        controllerRef.set(controller);
        LOG.info("Starting flowlet: {}", flowletContext);
        driver.start();
        LOG.info("Flowlet started: {}", flowletContext);
        return controller;
    } catch (Exception e) {
        // of the flowlet context.
        if (flowletContext != null) {
            flowletContext.close();
        }
        throw Throwables.propagate(e);
    }
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) Set(java.util.Set) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableList(com.google.common.collect.ImmutableList) Node(co.cask.cdap.app.queue.QueueSpecificationGenerator.Node) BasicProgramContext(co.cask.cdap.internal.app.runtime.BasicProgramContext) ProgramContext(co.cask.cdap.data.ProgramContext) InstantiatorFactory(co.cask.cdap.common.lang.InstantiatorFactory) FlowletDefinition(co.cask.cdap.api.flow.FlowletDefinition) FlowSpecification(co.cask.cdap.api.flow.FlowSpecification) MetricsFieldSetter(co.cask.cdap.internal.app.runtime.MetricsFieldSetter) ProgramType(co.cask.cdap.proto.ProgramType) RunId(org.apache.twill.api.RunId) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) Flowlet(co.cask.cdap.api.flow.flowlet.Flowlet) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) Service(com.google.common.util.concurrent.Service) MessagingService(co.cask.cdap.messaging.MessagingService) AbstractService(com.google.common.util.concurrent.AbstractService) AtomicReference(java.util.concurrent.atomic.AtomicReference) ProgramId(co.cask.cdap.proto.id.ProgramId) BasicProgramContext(co.cask.cdap.internal.app.runtime.BasicProgramContext) DataSetFieldSetter(co.cask.cdap.internal.app.runtime.DataSetFieldSetter) UnsupportedTypeException(co.cask.cdap.api.data.schema.UnsupportedTypeException) IOException(java.io.IOException) SimpleQueueSpecificationGenerator(co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator) PropertyFieldSetter(co.cask.cdap.common.lang.PropertyFieldSetter) FlowletId(co.cask.cdap.proto.id.FlowletId) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) DataFabricFacade(co.cask.cdap.internal.app.runtime.DataFabricFacade) ApplicationId(co.cask.cdap.proto.id.ApplicationId) ProgramContextAware(co.cask.cdap.data.ProgramContextAware)

Aggregations

ProgramContext (co.cask.cdap.data.ProgramContext)3 ProgramRunId (co.cask.cdap.proto.id.ProgramRunId)2 ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)1 UnsupportedTypeException (co.cask.cdap.api.data.schema.UnsupportedTypeException)1 FlowSpecification (co.cask.cdap.api.flow.FlowSpecification)1 FlowletDefinition (co.cask.cdap.api.flow.FlowletDefinition)1 Flowlet (co.cask.cdap.api.flow.flowlet.Flowlet)1 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)1 Node (co.cask.cdap.app.queue.QueueSpecificationGenerator.Node)1 InstantiatorFactory (co.cask.cdap.common.lang.InstantiatorFactory)1 PropertyFieldSetter (co.cask.cdap.common.lang.PropertyFieldSetter)1 ProgramContextAware (co.cask.cdap.data.ProgramContextAware)1 ForwardingStreamConsumer (co.cask.cdap.data2.transaction.stream.ForwardingStreamConsumer)1 StreamConsumer (co.cask.cdap.data2.transaction.stream.StreamConsumer)1 SimpleQueueSpecificationGenerator (co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator)1 BasicProgramContext (co.cask.cdap.internal.app.runtime.BasicProgramContext)1 DataFabricFacade (co.cask.cdap.internal.app.runtime.DataFabricFacade)1 DataSetFieldSetter (co.cask.cdap.internal.app.runtime.DataSetFieldSetter)1 MetricsFieldSetter (co.cask.cdap.internal.app.runtime.MetricsFieldSetter)1 MessagingService (co.cask.cdap.messaging.MessagingService)1