hadoop如何自定义GroupComparator实现求最大值

小编给大家分享一下hadoop如何自定义GroupComparator实现求最大值，相信大部分人都还不怎么了解，因此分享这篇文章给大家参考一下，希望大家阅读完这篇文章后大有收获，下面让我们一起去了解一下吧！

import java.io.Datainput;import java.io.DataOutput;import java.io.IOException;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.io.WritableComparator;import org.apache.hadoop.mapReduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFORMat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class GroupComparatorMian {static final Log LOG = LogFactory.getLog(GroupComparatorMian.class);public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = Job.getInstance(conf);job.setjarByClass(GroupComparatorMian.class);job.setMapperClass(GroupComparatorMapper.class);job.setReducerClass(GroupComparatorReducer.class);job.setOutpuTKEyClass(OrderBean.class);job.setOutputValueClass(NullWritable.class);job.setGroupinGComparatorClass(CustGroupComparator.class);String jobName = "'Customize groupcomparator test'";job.setJobName(jobName);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));boolean bb = job.waitForCompletion(true);if(bb) {LOG.info("Job "+ jobName +" is done.");}else {LOG.info("Job "+ jobName +"is Going wrong,now exit.");System.exit(0);}}}class CustGroupComparator extends WritableComparator{public CustGroupComparator() {super(OrderBean.class,true);}@Overridepublic int compare(WritableComparable a, WritableComparable b) {OrderBean oa = (OrderBean)a;OrderBean ob = (OrderBean)b;return oa.getOrder_id().compareTo(ob.getOrder_id());}}class OrderBean implements WritableComparable<OrderBean>{private String order_id;private String id ;private double prise;public OrderBean() {}public OrderBean(String order_id,String id,double prise) {this.order_id = order_id ;this.id = id;this.prise = prise;}public String getOrder_id() {return order_id;}public void setOrder_id(String order_id) {this.order_id = order_id;}public String getId() {return id;}public void setId(String id) {this.id = id;}public double getPrise() {return prise;}public void setPrise(double prise) {this.prise = prise;}@Overridepublic void write(DataOutput out) throws IOException {out.writeUTF(order_id);out.writeUTF(id);out.writeDouble(prise);}@Overridepublic void readFields(DataInput in) throws IOException {this.order_id = in.readUTF();this.id = in.readUTF();this.prise = in.readDouble();}@Overridepublic int compareTo(OrderBean o) {int cnt = this.order_id.compareTo(o.getOrder_id());if(cnt==0) {cnt = (int) (-(this.prise- o.getPrise()));}return cnt;}@Overridepublic String toString() {return  order_id + "\t" + id + "\t" + prise ;}}class GroupComparatorMapper extends Mapper<LongWritable, Text, OrderBean, NullWritable>{NullWritable nul =  NullWritable.get();OrderBean ob = new OrderBean();@Overrideprotected void map(LongWritable key, Text value,Mapper<LongWritable, Text, OrderBean, NullWritable>.Context context)throws IOException, InterruptedException {String[] split =  value.toString().split("\t");ob.setOrder_id(split[0]);ob.setId(split[1]);ob.setPrise(Double.parseDouble(split[2]));context.write(ob, nul);}}class GroupComparatorReducer extends Reducer<OrderBean, NullWritable, OrderBean, NullWritable>{NullWritable nul = NullWritable.get();@Overrideprotected void reduce(OrderBean bean, Iterable<NullWritable> iter,Reducer<OrderBean, NullWritable, OrderBean, NullWritable>.Context context)throws IOException, InterruptedException {context.write(bean, nul);}}

以上是“hadoop如何自定义GroupComparator实现求最大值”这篇文章的所有内容，感谢各位的阅读！相信大家都有了一定的了解，希望分享的内容对大家有所帮助，如果还想学习更多知识，欢迎关注编程网精选频道！