use of org.apache.commons.cli.HelpFormatter in project Cloud9 by lintool.
the class EncodeBfsGraph method run.
@SuppressWarnings("static-access")
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("XML dump file").create(INPUT_OPTION));
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION));
options.addOption(OptionBuilder.withArgName("nodeid").hasArg().withDescription("source node").create(SRC_OPTION));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(OUTPUT_OPTION) || !cmdline.hasOption(SRC_OPTION)) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String inputPath = cmdline.getOptionValue(INPUT_OPTION);
String outputPath = cmdline.getOptionValue(OUTPUT_OPTION);
int src = Integer.parseInt(cmdline.getOptionValue(SRC_OPTION));
LOG.info("Tool name: " + this.getClass().getName());
LOG.info(" - inputDir: " + inputPath);
LOG.info(" - outputDir: " + outputPath);
LOG.info(" - src: " + src);
Job job = Job.getInstance(getConf());
job.setJobName(String.format("EncodeBfsGraph[%s: %s, %s: %s, %s: %d]", INPUT_OPTION, inputPath, OUTPUT_OPTION, outputPath, SRC_OPTION, src));
job.setJarByClass(EncodeBfsGraph.class);
job.setNumReduceTasks(0);
job.getConfiguration().setInt(SRC_OPTION, src);
job.getConfiguration().setInt("mapred.min.split.size", 1024 * 1024 * 1024);
FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(BfsNode.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(BfsNode.class);
job.setMapperClass(MyMapper.class);
// Delete the output directory if it exists already.
FileSystem.get(job.getConfiguration()).delete(new Path(outputPath), true);
job.waitForCompletion(true);
return 0;
}
use of org.apache.commons.cli.HelpFormatter in project Cloud9 by lintool.
the class FindReachableNodes method run.
@SuppressWarnings("static-access")
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("XML dump file").create(INPUT_OPTION));
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(OUTPUT_OPTION)) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String inputPath = cmdline.getOptionValue(INPUT_OPTION);
String outputPath = cmdline.getOptionValue(OUTPUT_OPTION);
LOG.info("Tool name: " + this.getClass().getName());
LOG.info(" - inputDir: " + inputPath);
LOG.info(" - outputDir: " + outputPath);
Job job = Job.getInstance(getConf());
job.setJobName(String.format("FindReachableNodes:[%s: %s, %s: %s]", INPUT_OPTION, inputPath, OUTPUT_OPTION, outputPath));
job.setJarByClass(FindReachableNodes.class);
job.setNumReduceTasks(0);
job.getConfiguration().setInt("mapred.min.split.size", 1024 * 1024 * 1024);
FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(BfsNode.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(BfsNode.class);
job.setMapperClass(MyMapper.class);
// Delete the output directory if it exists already.
FileSystem.get(job.getConfiguration()).delete(new Path(outputPath), true);
job.waitForCompletion(true);
return 0;
}
use of org.apache.commons.cli.HelpFormatter in project Cloud9 by lintool.
the class IterateBfs method run.
@SuppressWarnings("static-access")
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("XML dump file").create(INPUT_OPTION));
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION));
options.addOption(OptionBuilder.withArgName("num").hasArg().withDescription("number of partitions").create(NUM_PARTITIONS_OPTION));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(OUTPUT_OPTION) || !cmdline.hasOption(NUM_PARTITIONS_OPTION)) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String inputPath = cmdline.getOptionValue(INPUT_OPTION);
String outputPath = cmdline.getOptionValue(OUTPUT_OPTION);
int n = Integer.parseInt(cmdline.getOptionValue(NUM_PARTITIONS_OPTION));
LOG.info("Tool name: " + this.getClass().getName());
LOG.info(" - inputDir: " + inputPath);
LOG.info(" - outputDir: " + outputPath);
LOG.info(" - numPartitions: " + n);
getConf().set("mapred.child.java.opts", "-Xmx2048m");
Job job = Job.getInstance(getConf());
job.setJobName(String.format("IterateBfs[%s: %s, %s: %s, %s: %d]", INPUT_OPTION, inputPath, OUTPUT_OPTION, outputPath, NUM_PARTITIONS_OPTION, n));
job.setJarByClass(EncodeBfsGraph.class);
job.setNumReduceTasks(n);
FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(BfsNode.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(BfsNode.class);
job.setMapperClass(MapClass.class);
job.setReducerClass(ReduceClass.class);
// Delete the output directory if it exists already.
FileSystem.get(job.getConfiguration()).delete(new Path(outputPath), true);
job.waitForCompletion(true);
return 0;
}
use of org.apache.commons.cli.HelpFormatter in project Cloud9 by lintool.
the class AnalyzeBigramCount method main.
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzeBigramCount.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
List<PairOfWritables<Text, IntWritable>> bigrams = SequenceFileUtils.readDirectory(new Path(inputPath));
Collections.sort(bigrams, new Comparator<PairOfWritables<Text, IntWritable>>() {
public int compare(PairOfWritables<Text, IntWritable> e1, PairOfWritables<Text, IntWritable> e2) {
if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
int singletons = 0;
int sum = 0;
for (PairOfWritables<Text, IntWritable> bigram : bigrams) {
sum += bigram.getRightElement().get();
if (bigram.getRightElement().get() == 1) {
singletons++;
}
}
System.out.println("total number of unique bigrams: " + bigrams.size());
System.out.println("total number of bigrams: " + sum);
System.out.println("number of bigrams that appear only once: " + singletons);
System.out.println("\nten most frequent bigrams: ");
Iterator<PairOfWritables<Text, IntWritable>> iter = Iterators.limit(bigrams.iterator(), 10);
while (iter.hasNext()) {
PairOfWritables<Text, IntWritable> bigram = iter.next();
System.out.println(bigram.getLeftElement() + "\t" + bigram.getRightElement());
}
}
use of org.apache.commons.cli.HelpFormatter in project Cloud9 by lintool.
the class AnalyzeBigramRelativeFrequencyJson method main.
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzeBigramRelativeFrequencyJson.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> pairs = SequenceFileUtils.readDirectory(new Path(inputPath));
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> list1 = Lists.newArrayList();
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> list2 = Lists.newArrayList();
for (PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p : pairs) {
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
if (bigram.getJsonObject().get("Left").getAsString().equals("light")) {
list1.add(p);
}
if (bigram.getJsonObject().get("Left").getAsString().equals("contain")) {
list2.add(p);
}
}
Collections.sort(list1, new Comparator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>>() {
public int compare(PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e1, PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> iter1 = Iterators.limit(list1.iterator(), 10);
while (iter1.hasNext()) {
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p = iter1.next();
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
Collections.sort(list2, new Comparator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>>() {
public int compare(PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e1, PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> iter2 = Iterators.limit(list2.iterator(), 10);
while (iter2.hasNext()) {
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p = iter2.next();
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
}
Aggregations