Examples with ProgramId co.cask.cdap.proto.id.ProgramId used on opensource projects

Example 96 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LineageCollapser method collapseRelations.

/**
   * Collapse {@link Relation}s based on {@link CollapseType}
   * @param relations lineage relations
   * @param collapseTypes fields to collapse relations on
   * @return collapsed relations
   */
public static Set<CollapsedRelation> collapseRelations(Iterable<Relation> relations, Set<CollapseType> collapseTypes) {
    Set<CollapsedRelation> collapsedRelations = new HashSet<>();
    Multimap<CollapseKey, Relation> multimap = HashMultimap.create();
    for (Relation relation : relations) {
        multimap.put(getCollapseKey(relation, collapseTypes), relation);
    }
    LOG.trace("Collapsed relations: {}", multimap.asMap());
    for (Map.Entry<CollapseKey, Collection<Relation>> collapsedEntry : multimap.asMap().entrySet()) {
        NamespacedEntityId data = collapsedEntry.getKey().data;
        ProgramId program = collapsedEntry.getKey().program;
        Set<AccessType> accessTypes = new HashSet<>();
        Set<RunId> runs = new HashSet<>();
        Set<NamespacedEntityId> components = new HashSet<>();
        for (Relation relation : collapsedEntry.getValue()) {
            accessTypes.add(relation.getAccess());
            runs.add(relation.getRun());
            components.addAll(relation.getComponents());
        }
        collapsedRelations.add(toCollapsedRelation(data, program, accessTypes, runs, components));
    }
    return collapsedRelations;
}

Example 97 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LineageDataset method parseRow.

private RowKey parseRow(Row row) {
    ProgramId program;
    NamespacedEntityId data;
    RunId runId;
    MDSKey.Splitter splitter = new MDSKey(row.getRow()).split();
    char marker = (char) splitter.getInt();
    LOG.trace("Got marker {}", marker);
    switch(marker) {
        case PROGRAM_MARKER:
            program = (ProgramId) toEntityId(splitter, marker);
            // inverted start time
            splitter.skipLong();
            marker = (char) splitter.getInt();
            // data
            data = toEntityId(splitter, marker);
            runId = RunIds.fromString(splitter.getString());
            return new RowKey(program, data, runId);
        case DATASET_MARKER:
        case STREAM_MARKER:
            data = toEntityId(splitter, marker);
            // inverted start time
            splitter.skipLong();
            marker = (char) splitter.getInt();
            // program
            program = (ProgramId) toEntityId(splitter, marker);
            runId = RunIds.fromString(splitter.getString());
            return new RowKey(program, data, runId);
        default:
            throw new IllegalStateException("Invalid row with marker " + marker);
    }
}

Example 98 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class AbstractProgramTwillRunnable method initialize.

@Override
public void initialize(TwillContext context) {
    name = context.getSpecification().getName();
    LOG.info("Initializing runnable: " + name);
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new RunnableSecurityManager(System.getSecurityManager()));
    // Install the JUL to SLF4J Bridge
    SLF4JBridgeHandler.install();
    runLatch = new CountDownLatch(1);
    coreServices = new ArrayList<>();
    try {
        CommandLine cmdLine = parseArgs(context.getApplicationArguments());
        // Loads configurations
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(cmdLine.getOptionValue(RunnableOptions.HADOOP_CONF_FILE)).toURI().toURL());
        UserGroupInformation.setConfiguration(hConf);
        cConf = CConfiguration.create(new File(cmdLine.getOptionValue(RunnableOptions.CDAP_CONF_FILE)));
        programOpts = createProgramOptions(cmdLine, context, context.getSpecification().getConfigs());
        // This impersonation info is added in PropertiesResolver#getSystemProperties
        // if kerberos is enabled we expect the principal to be provided in the program options as we
        // need it to be used later in ExploreClient to make request. If kerberos is disabled this will be null
        String principal = programOpts.getArguments().getOption(ProgramOptionConstants.PRINCIPAL);
        ProgramId programId = GSON.fromJson(cmdLine.getOptionValue(RunnableOptions.PROGRAM_ID), ProgramId.class);
        String instanceId = programOpts.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID);
        String runId = programOpts.getArguments().getOption(ProgramOptionConstants.RUN_ID);
        Injector injector = Guice.createInjector(createModule(context, programId, runId, instanceId, principal));
        coreServices.add(injector.getInstance(ZKClientService.class));
        coreServices.add(injector.getInstance(KafkaClientService.class));
        coreServices.add(injector.getInstance(BrokerService.class));
        coreServices.add(injector.getInstance(MetricsCollectionService.class));
        coreServices.add(injector.getInstance(StreamCoordinatorClient.class));
        // Initialize log appender
        logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();
        // Create the ProgramRunner
        programRunner = createProgramRunner(injector);
        try {
            Location programJarLocation = Locations.toLocation(new File(cmdLine.getOptionValue(RunnableOptions.JAR)));
            ApplicationSpecification appSpec = readAppSpec(new File(cmdLine.getOptionValue(RunnableOptions.APP_SPEC_FILE)));
            program = Programs.create(cConf, programRunner, new ProgramDescriptor(programId, appSpec), programJarLocation, new File(cmdLine.getOptionValue(RunnableOptions.EXPANDED_JAR)));
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
        coreServices.add(new ProgramRunnableResourceReporter(program.getId(), injector.getInstance(MetricsCollectionService.class), context));
        LOG.info("Runnable initialized: {}", name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

Example 99 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class ScheduleTaskRunner method launch.

public void launch(Job job) throws Exception {
    ProgramSchedule schedule = job.getSchedule();
    ProgramId programId = schedule.getProgramId();
    Map<String, String> userArgs = Maps.newHashMap();
    Map<String, String> systemArgs = Maps.newHashMap();
    // notificationProperties is present only in jobs containing schedules with TimeTrigger and StreamSizeTrigger.
    // Since both triggers are satisfied by the first notification, there can be only one notification in in the job
    Map<String, String> notificationProperties = job.getNotifications().get(0).getProperties();
    userArgs.putAll(schedule.getProperties());
    userArgs.putAll(propertiesResolver.getUserProperties(programId.toId()));
    String userOverridesString = notificationProperties.get(ProgramOptionConstants.USER_OVERRIDES);
    if (userOverridesString != null) {
        Map<String, String> userOverrides = GSON.fromJson(userOverridesString, STRING_STRING_MAP);
        userArgs.putAll(userOverrides);
    }
    systemArgs.putAll(propertiesResolver.getSystemProperties(programId.toId()));
    String systemOverridesString = notificationProperties.get(ProgramOptionConstants.SYSTEM_OVERRIDES);
    if (systemOverridesString != null) {
        Map<String, String> systemOverrides = GSON.fromJson(systemOverridesString, STRING_STRING_MAP);
        systemArgs.putAll(systemOverrides);
    }
    execute(programId, systemArgs, userArgs);
    LOG.info("Successfully started program {} in schedule {}.", schedule.getProgramId(), schedule.getName());
}

Example 100 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class ServiceProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    int instanceId = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
    Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
    int instanceCount = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
    Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
    RunId runId = ProgramRunners.getRunId(options);
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType programType = program.getType();
    Preconditions.checkNotNull(programType, "Missing processor type.");
    Preconditions.checkArgument(programType == ProgramType.SERVICE, "Only Service process type is supported.");
    ServiceSpecification spec = appSpec.getServices().get(program.getName());
    String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
    Preconditions.checkArgument(host != null, "No hostname is provided");
    // Setup dataset framework context, if required
    if (datasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) datasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    final PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
    try {
        ServiceHttpServer component = new ServiceHttpServer(host, program, options, cConf, spec, instanceId, instanceCount, serviceAnnouncer, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, pluginInstantiator, secureStore, secureStoreManager, messagingService, defaultArtifactManager);
        // Add a service listener to make sure the plugin instantiator is closed when the http server is finished.
        component.addListener(new ServiceListenerAdapter() {

            @Override
            public void terminated(Service.State from) {
                Closeables.closeQuietly(pluginInstantiator);
            }

            @Override
            public void failed(Service.State from, Throwable failure) {
                Closeables.closeQuietly(pluginInstantiator);
            }
        }, Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new ServiceProgramControllerAdapter(component, program.getId(), runId, spec.getName() + "-" + instanceId);
        component.start();
        return controller;
    } catch (Throwable t) {
        Closeables.closeQuietly(pluginInstantiator);
        throw t;
    }
}