作者winest (winest)
看板C_and_CPP
标题Re: [问题] c语言程式求助
时间Wed Oct 14 13:53:00 2009
※ 引述《costbook (没$结婚干嘛,有$干嘛结婚)》之铭言：
: 这算是程式人的啦滴赛大会吗？？？
: 有没有人要用Hadoop的MapReduce class来写？
: 改成云端运算的版本....XD
Hadoop版本，支援1~N台电脑平行运算
自动切割来源档案，并输出依体重排序所有人的名字
public class Training
{
public static void main( String[] args )
{
Path inputPath = new Path( "input" );
Path outputPath = new Path( "output" );
if ( args.length == 2 ) //if specify input and output path, use it
{
inputPath = new Path( args[0] );
outputPath = new Path( args[1] );
}
else{}
JobClient client = new JobClient();
JobConf conf = new JobConf( Training.class );
conf.setJobName( "Homework" );
conf.setNumMapTasks( 100 );
conf.setNumReduceTasks( 10 );
//mapper to reducer's <key,value> type
conf.setOutputKeyClass( Text.class );
conf.setOutputValueClass( Text.class );
//specify input and output DIRECTORIES (not files)
FileInputFormat.setInputPaths( conf , inputPath );
FileOutputFormat.setOutputPath( conf , outputPath );
conf.setMapperClass( TrainingMapper.class );
conf.setCombinerClass( TrainingCombiner.class );
conf.setReducerClass( TrainingReducer.class );
//setting output to different directories
conf.setOutputFormat( MyTextOutputFormat.class );
client.setConf( conf );
try
{
JobClient.runJob( conf );
}
catch ( Exception e )
{
e.printStackTrace();
}
} //end of main
}
//Mapper
public class TrainingMapper extends MapReduceBase
implements Mapper<LongWritable , Text , Text , Text>
{
private static int i;
private static String splitter = "\t";
public void map( LongWritable key , Text value ,
OutputCollector<Text , Text> output , Reporter reporter ) throws
IOException
{
String lineBuf = value.toString();
String [] data = lineBuf.split( splitter );
//if key or value is null, skip
if ( data[0].length() > 0 && data[1].length() > 0 )
{
System.out.println( data[1] + "的体重是" + data[0] );
output.collect( data[1] , new Text( data[0] ) );
}
}
}
//Reducer
public class TrainingReducer extends MapReduceBase
implements Reducer<Text , Text , Text , Text>
{
public void reduce( Text key , Iterator<Text> values ,
OutputCollector<Text , Text> output , Reporter reporter ) throws
IOException
{
while ( values.hasNext() )
{
output.collect( key , new Text( values.next() ) );
}
}
}
--
※ 发信站: 批踢踢实业坊(ptt.cc)
◆ From: 140.115.53.26
1F：推 archon:这是许愿池吗... (￣▽￣＃)﹏﹏ 10/14 13:58
2F：推 costbook:XD 10/14 14:03
3F：推 xpsteven:推Hadoop 10/14 14:20
4F：推 VictorTom:原来附加功能这麽多, 我想说怎麽code这麽长XD 10/14 14:26
5F：推 james732:推Hadoop XDDDD 10/14 14:27