Examples with ConsumerEx com.hazelcast.function.ConsumerEx used on opensource projects

Example 1 with ConsumerEx

use of com.hazelcast.function.ConsumerEx in project hazelcast by hazelcast.

the class HadoopFileSourceFactory method configureFn.

private static <T> ConsumerEx<Configuration> configureFn(FileSourceConfiguration<T> fsc, JobConfigurer configurer, FileFormat<T> fileFormat) {
    return new ConsumerEx<Configuration>() {

        @Override
        public void acceptEx(Configuration configuration) throws Exception {
            try {
                configuration.setBoolean(FileInputFormat.INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS, true);
                configuration.setBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, false);
                configuration.setBoolean(HadoopSources.SHARED_LOCAL_FS, fsc.isSharedFileSystem());
                configuration.setBoolean(HadoopSources.IGNORE_FILE_NOT_FOUND, fsc.isIgnoreFileNotFound());
                for (Entry<String, String> option : fsc.getOptions().entrySet()) {
                    configuration.set(option.getKey(), option.getValue());
                }
                // Some methods we use to configure actually take a Job
                Job job = Job.getInstance(configuration);
                Path inputPath = getInputPath(fsc, configuration);
                FileInputFormat.addInputPath(job, inputPath);
                configurer.configure(job, fileFormat);
                // original configuration instance
                for (Entry<String, String> entry : job.getConfiguration()) {
                    configuration.set(entry.getKey(), entry.getValue());
                }
            } catch (IOException e) {
                throw new JetException("Could not create a source", e);
            }
        }

        @Override
        public List<Permission> permissions() {
            String keyFile = fsc.getOptions().get("google.cloud.auth.service.account.json.keyfile");
            if (keyFile != null) {
                return asList(ConnectorPermission.file(keyFile, ACTION_READ), ConnectorPermission.file(fsc.getPath(), ACTION_READ));
            }
            return singletonList(ConnectorPermission.file(fsc.getPath(), ACTION_READ));
        }
    };
}