作者winest (winest)
看板C_and_CPP
標題Re: [問題] c語言程式求助
時間Wed Oct 14 13:53:00 2009
※ 引述《costbook (沒$結婚幹嘛,有$幹嘛結婚)》之銘言：
: 這算是程式人的啦滴賽大會嗎？？？
: 有沒有人要用Hadoop的MapReduce class來寫？
: 改成雲端運算的版本....XD
Hadoop版本，支援1~N台電腦平行運算
自動切割來源檔案，並輸出依體重排序所有人的名字
public class Training
{
public static void main( String[] args )
{
Path inputPath = new Path( "input" );
Path outputPath = new Path( "output" );
if ( args.length == 2 ) //if specify input and output path, use it
{
inputPath = new Path( args[0] );
outputPath = new Path( args[1] );
}
else{}
JobClient client = new JobClient();
JobConf conf = new JobConf( Training.class );
conf.setJobName( "Homework" );
conf.setNumMapTasks( 100 );
conf.setNumReduceTasks( 10 );
//mapper to reducer's <key,value> type
conf.setOutputKeyClass( Text.class );
conf.setOutputValueClass( Text.class );
//specify input and output DIRECTORIES (not files)
FileInputFormat.setInputPaths( conf , inputPath );
FileOutputFormat.setOutputPath( conf , outputPath );
conf.setMapperClass( TrainingMapper.class );
conf.setCombinerClass( TrainingCombiner.class );
conf.setReducerClass( TrainingReducer.class );
//setting output to different directories
conf.setOutputFormat( MyTextOutputFormat.class );
client.setConf( conf );
try
{
JobClient.runJob( conf );
}
catch ( Exception e )
{
e.printStackTrace();
}
} //end of main
}
//Mapper
public class TrainingMapper extends MapReduceBase
implements Mapper<LongWritable , Text , Text , Text>
{
private static int i;
private static String splitter = "\t";
public void map( LongWritable key , Text value ,
OutputCollector<Text , Text> output , Reporter reporter ) throws
IOException
{
String lineBuf = value.toString();
String [] data = lineBuf.split( splitter );
//if key or value is null, skip
if ( data[0].length() > 0 && data[1].length() > 0 )
{
System.out.println( data[1] + "的體重是" + data[0] );
output.collect( data[1] , new Text( data[0] ) );
}
}
}
//Reducer
public class TrainingReducer extends MapReduceBase
implements Reducer<Text , Text , Text , Text>
{
public void reduce( Text key , Iterator<Text> values ,
OutputCollector<Text , Text> output , Reporter reporter ) throws
IOException
{
while ( values.hasNext() )
{
output.collect( key , new Text( values.next() ) );
}
}
}
--
※ 發信站: 批踢踢實業坊(ptt.cc)
◆ From: 140.115.53.26
1F：推 archon:這是許願池嗎... (￣▽￣＃)﹏﹏ 10/14 13:58
2F：推 costbook:XD 10/14 14:03
3F：推 xpsteven:推Hadoop 10/14 14:20
4F：推 VictorTom:原來附加功能這麼多, 我想說怎麼code這麼長XD 10/14 14:26
5F：推 james732:推Hadoop XDDDD 10/14 14:27