Examples with CConfiguration co.cask.cdap.common.conf.CConfiguration used on opensource projects

Example 76 with CConfiguration

use of co.cask.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class LevelDBStreamConsumerStateTest method init.

@BeforeClass
public static void init() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new DataFabricLevelDBModule(), new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    streamAdmin = injector.getInstance(StreamAdmin.class);
    stateStoreFactory = injector.getInstance(StreamConsumerStateStoreFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    streamCoordinatorClient.startAndWait();
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}

Example 77 with CConfiguration

use of co.cask.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class StreamTailer method main.

public static void main(String[] args) throws Exception {
    if (args.length < 1) {
        System.out.println(String.format("Usage: java %s [streamName]", StreamTailer.class.getName()));
        return;
    }
    String streamName = args[0];
    CConfiguration cConf = CConfiguration.create();
    Configuration hConf = new Configuration();
    String txClientId = StreamTailer.class.getName();
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new DataFabricModules(txClientId).getDistributedModules(), new DataSetsModules().getDistributedModules(), new LocationRuntimeModule().getDistributedModules(), new ExploreClientModule(), new ViewAdminModules().getDistributedModules(), new StreamAdminModules().getDistributedModules(), new AuthorizationEnforcementModule().getDistributedModules(), new AuthenticationContextModules().getMasterModule(), new NotificationFeedClientModule());
    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    //TODO: get namespace from commandline arguments
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    Location streamLocation = streamConfig.getLocation();
    List<Location> eventFiles = Lists.newArrayList();
    for (Location partition : streamLocation.list()) {
        if (!partition.isDirectory()) {
            continue;
        }
        for (Location file : partition.list()) {
            if (StreamFileType.EVENT.isMatched(file.getName())) {
                eventFiles.add(file);
            }
        }
    }
    int generation = StreamUtils.getGeneration(streamConfig);
    MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, ImmutableList.copyOf(Iterables.transform(eventFiles, createOffsetConverter(generation))));
    List<StreamEvent> events = Lists.newArrayList();
    while (reader.read(events, 10, 100, TimeUnit.MILLISECONDS) >= 0) {
        for (StreamEvent event : events) {
            System.out.println(event.getTimestamp() + " " + Charsets.UTF_8.decode(event.getBody()));
        }
        events.clear();
    }
    reader.close();
}

Example 78 with CConfiguration

use of co.cask.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class MapReduceRuntimeService method startUp.

@Override
protected void startUp() throws Exception {
    // Creates a temporary directory locally for storing all generated files.
    File tempDir = createTempDirectory();
    cleanupTask = createCleanupTask(tempDir);
    try {
        Job job = createJob(new File(tempDir, "mapreduce"));
        Configuration mapredConf = job.getConfiguration();
        classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getApplicationSpecification().getPlugins(), context.getPluginInstantiator());
        cleanupTask = createCleanupTask(cleanupTask, classLoader);
        mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader));
        ClassLoaders.setContextClassLoader(mapredConf.getClassLoader());
        context.setJob(job);
        beforeSubmit(job);
        // Localize additional resources that users have requested via BasicMapReduceContext.localize methods
        Map<String, String> localizedUserResources = localizeUserResources(job, tempDir);
        // Override user-defined job name, since we set it and depend on the name.
        // https://issues.cask.co/browse/CDAP-2441
        String jobName = job.getJobName();
        if (!jobName.isEmpty()) {
            LOG.warn("Job name {} is being overridden.", jobName);
        }
        job.setJobName(getJobName(context));
        // Create a temporary location for storing all generated files through the LocationFactory.
        Location tempLocation = createTempLocationDirectory();
        cleanupTask = createCleanupTask(cleanupTask, tempLocation);
        // For local mode, everything is in the configuration classloader already, hence no need to create new jar
        if (!MapReduceTaskContextProvider.isLocal(mapredConf)) {
            // After calling initialize, we know what plugins are needed for the program, hence construct the proper
            // ClassLoader from here and use it for setting up the job
            Location pluginArchive = createPluginArchive(tempLocation);
            if (pluginArchive != null) {
                job.addCacheArchive(pluginArchive.toURI());
                mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName());
            }
        }
        // set resources for the job
        TaskType.MAP.setResources(mapredConf, context.getMapperResources());
        TaskType.REDUCE.setResources(mapredConf, context.getReducerResources());
        // replace user's Mapper, Reducer, Partitioner, and Comparator classes with our wrappers in job config
        MapperWrapper.wrap(job);
        ReducerWrapper.wrap(job);
        PartitionerWrapper.wrap(job);
        RawComparatorWrapper.CombinerGroupComparatorWrapper.wrap(job);
        RawComparatorWrapper.GroupComparatorWrapper.wrap(job);
        RawComparatorWrapper.KeyComparatorWrapper.wrap(job);
        // packaging job jar which includes cdap classes with dependencies
        File jobJar = buildJobJar(job, tempDir);
        job.setJar(jobJar.toURI().toString());
        Location programJar = programJarLocation;
        if (!MapReduceTaskContextProvider.isLocal(mapredConf)) {
            // Copy and localize the program jar in distributed mode
            programJar = copyProgramJar(tempLocation);
            job.addCacheFile(programJar.toURI());
            // Generate and localize the launcher jar to control the classloader of MapReduce containers processes
            Location launcherJar = createLauncherJar(tempLocation);
            job.addCacheFile(launcherJar.toURI());
            // Launcher.jar should be the first one in the classpath
            List<String> classpath = new ArrayList<>();
            classpath.add(launcherJar.getName());
            // Localize logback.xml
            Location logbackLocation = ProgramRunners.createLogbackJar(tempLocation.append("logback.xml.jar"));
            if (logbackLocation != null) {
                job.addCacheFile(logbackLocation.toURI());
                classpath.add(logbackLocation.getName());
                mapredConf.set("yarn.app.mapreduce.am.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
                mapredConf.set("mapreduce.map.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
                mapredConf.set("mapreduce.reduce.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
            }
            // Get all the jars in jobJar and sort them lexically before adding to the classpath
            // This allows CDAP classes to be picked up first before the Twill classes
            List<String> jarFiles = new ArrayList<>();
            try (JarFile jobJarFile = new JarFile(jobJar)) {
                Enumeration<JarEntry> entries = jobJarFile.entries();
                while (entries.hasMoreElements()) {
                    JarEntry entry = entries.nextElement();
                    if (entry.getName().startsWith("lib/") && entry.getName().endsWith(".jar")) {
                        jarFiles.add("job.jar/" + entry.getName());
                    }
                }
            }
            Collections.sort(jarFiles);
            classpath.addAll(jarFiles);
            classpath.add("job.jar/classes");
            // Add extra jars set in cConf
            for (URI jarURI : CConfigurationUtil.getExtraJars(cConf)) {
                if ("file".equals(jarURI.getScheme())) {
                    Location extraJarLocation = copyFileToLocation(new File(jarURI.getPath()), tempLocation);
                    job.addCacheFile(extraJarLocation.toURI());
                } else {
                    job.addCacheFile(jarURI);
                }
                classpath.add(LocalizationUtils.getLocalizedName(jarURI));
            }
            // Add the mapreduce application classpath at last
            MapReduceContainerHelper.addMapReduceClassPath(mapredConf, classpath);
            mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, Joiner.on(",").join(classpath));
            mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(classpath));
        }
        MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf);
        // We start long-running tx to be used by mapreduce job tasks.
        Transaction tx = txClient.startLong();
        try {
            // We remember tx, so that we can re-use it in mapreduce tasks
            CConfiguration cConfCopy = cConf;
            contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources);
            // submits job and returns immediately. Shouldn't need to set context ClassLoader.
            job.submit();
            // log after the job.submit(), because the jobId is not assigned before then
            LOG.debug("Submitted MapReduce Job: {}.", context);
            this.job = job;
            this.transaction = tx;
        } catch (Throwable t) {
            Transactions.invalidateQuietly(txClient, tx);
            throw t;
        }
    } catch (LinkageError e) {
        // of the user program is missing dependencies (CDAP-2543)
        throw new Exception(e.getMessage(), e);
    } catch (Throwable t) {
        cleanupTask.run();
        // don't log the error. It will be logged by the ProgramControllerServiceAdapter.failed()
        if (t instanceof TransactionFailureException) {
            throw Transactions.propagate((TransactionFailureException) t, Exception.class);
        }
        throw t;
    }
}

Example 79 with CConfiguration

use of co.cask.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class MapReduceTaskContextProvider method createCacheLoader.

/**
   * Creates a {@link CacheLoader} for the task context cache.
   */
private CacheLoader<ContextCacheKey, BasicMapReduceTaskContext> createCacheLoader(final Injector injector) {
    final DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    final DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    final SecureStore secureStore = injector.getInstance(SecureStore.class);
    final SecureStoreManager secureStoreManager = injector.getInstance(SecureStoreManager.class);
    final MessagingService messagingService = injector.getInstance(MessagingService.class);
    // Multiple instances of BasicMapReduceTaskContext can shares the same program.
    final AtomicReference<Program> programRef = new AtomicReference<>();
    return new CacheLoader<ContextCacheKey, BasicMapReduceTaskContext>() {

        @Override
        public BasicMapReduceTaskContext load(ContextCacheKey key) throws Exception {
            MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration());
            MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
            Program program = programRef.get();
            if (program == null) {
                // Creation of program is relatively cheap, so just create and do compare and set.
                programRef.compareAndSet(null, createProgram(contextConfig, classLoader.getProgramClassLoader()));
                program = programRef.get();
            }
            WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
            DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, program.getApplicationSpecification());
            // Setup dataset framework context, if required
            if (programDatasetFramework instanceof ProgramContextAware) {
                ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
                ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
            }
            MapReduceSpecification spec = program.getApplicationSpecification().getMapReduce().get(program.getName());
            MetricsCollectionService metricsCollectionService = null;
            MapReduceMetrics.TaskType taskType = null;
            String taskId = null;
            TaskAttemptID taskAttemptId = key.getTaskAttemptID();
            // from a org.apache.hadoop.io.RawComparator
            if (taskAttemptId != null) {
                taskId = taskAttemptId.getTaskID().toString();
                if (MapReduceMetrics.TaskType.hasType(taskAttemptId.getTaskType())) {
                    taskType = MapReduceMetrics.TaskType.from(taskAttemptId.getTaskType());
                    // if this is not for a mapper or a reducer, we don't need the metrics collection service
                    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
                }
            }
            CConfiguration cConf = injector.getInstance(CConfiguration.class);
            TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
            return new BasicMapReduceTaskContext(program, contextConfig.getProgramOptions(), cConf, taskType, taskId, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txClient, contextConfig.getTx(), programDatasetFramework, classLoader.getPluginInstantiator(), contextConfig.getLocalizedResources(), secureStore, secureStoreManager, authorizationEnforcer, authenticationContext, messagingService);
        }
    };
}

Example 80 with CConfiguration

use of co.cask.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class LocalMRJobInfoFetcherTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    CConfiguration conf = CConfiguration.create();
    conf.set(Constants.AppFabric.OUTPUT_DIR, System.getProperty("java.io.tmpdir"));
    conf.set(Constants.AppFabric.TEMP_DIR, System.getProperty("java.io.tmpdir"));
    injector = startMetricsService(conf);
    metricStore = injector.getInstance(MetricStore.class);
}