版權聲明：本文為部落客原創文章，未經部落客允許不得轉載。 https://blog.csdn.net/SunnyYoona/article/details/72912058
1. Maven依賴
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency> https://note.youdao.com/md/preview/preview.html?file=%2Fyws%2Fapi%2Fpersonal%2Ffile%2FWEB22197732237269ddf496c6b1057c9399%3Fmethod%3Ddownload%26read%3Dtrue#2-%E8%B0%83%E8%AF%95%E8%AE%BE%E7%BD%AE 2. 調試設定
本地調試需要設定兩點：
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local"); 這樣MapReduce程式就不會讀取HDFS目錄上的檔案，而是讀取本地檔案，如果不設定，就會找不到對應的檔案而報錯：
org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: hdfs://localhost:9000/home/xiaosi/test/input/maxTemperature
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:323)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
...
at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:119)
at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:42)
at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:234)
at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) https://note.youdao.com/md/preview/preview.html?file=%2Fyws%2Fapi%2Fpersonal%2Ffile%2FWEB22197732237269ddf496c6b1057c9399%3Fmethod%3Ddownload%26read%3Dtrue#3-%E7%A8%8B%E5%BA%8F%E8%AE%BE%E7%BD%AE 3. 程式設定
我們在調試代碼中對配置檔案進行更改，傳給MapReduce程式：
maxTemperature.setConf(conf); 建立Job作業時必須使用如下方式指定我們設定的配置檔案，否則讀取不到我們更改的設定：
Configuration conf = this.getConf();
Job job = Job.getInstance(conf); 而不能使用：
Job job = Job.getInstance(); 這樣是不會讀取我們更改的配置，隻是讀取預設配置，程式會去HDFS目錄上找我們的輸入檔案而找不到報錯．
https://note.youdao.com/md/preview/preview.html?file=%2Fyws%2Fapi%2Fpersonal%2Ffile%2FWEB22197732237269ddf496c6b1057c9399%3Fmethod%3Ddownload%26read%3Dtrue#4-%E7%A4%BA%E4%BE%8B 4. 示例
https://note.youdao.com/md/preview/preview.html?file=%2Fyws%2Fapi%2Fpersonal%2Ffile%2FWEB22197732237269ddf496c6b1057c9399%3Fmethod%3Ddownload%26read%3Dtrue#41-%E6%9C%AC%E5%9C%B0%E8%B0%83%E8%AF%95%E4%BB%A3%E7%A0%81 4.1 本地調試代碼
package com.sjf.open;
import com.sjf.open.maxTemperature.MaxTemperature;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import org.junit.Before;
import org.junit.Test;
/**
* Created by xiaosi on 17-6-8.
*/
public class MapReduceTest {
private Configuration conf = new Configuration();
private Path inputPath;
private Path outputPath;
private String baseInputStr = "/home/xiaosi/test/input";
private String outputStr = "/home/xiaosi/test/output";
@Before
public void setUp() throws IOException {
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local");
outputPath = new Path(outputStr);
FileSystem fileSystem = FileSystem.getLocal(conf);
fileSystem.delete(outputPath, true);
}
@Test
public void MapReduceTest() throws Exception {
String path = baseInputStr + "/maxTemperature";
inputPath = new Path(path);
outputPath = new Path(outputStr);
MaxTemperature maxTemperature = new MaxTemperature();
maxTemperature.setConf(conf);
int exitCode = maxTemperature.run(new String[] {inputPath.toString(), outputPath.toString()});
assertThat(exitCode, is(0));
}
} https://note.youdao.com/md/preview/preview.html?file=%2Fyws%2Fapi%2Fpersonal%2Ffile%2FWEB22197732237269ddf496c6b1057c9399%3Fmethod%3Ddownload%26read%3Dtrue#42-%E7%A8%8B%E5%BA%8Fmain%E4%BB%A3%E7%A0%81 4.2 程式Main代碼
package com.sjf.open.maxTemperature;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
/**
* Created by xiaosi on 16-7-27.
*/
public class MaxTemperature extends Configured implements Tool{
public static void main(String[] args) throws Exception {
int status = ToolRunner.run(new MaxTemperature(), args);
System.exit(status);
}
@Override
public int run(String[] args) throws Exception {
if(args.length != 2){
System.err.println("Usage: MaxTemperature <input path> <output path>");
System.exit(-1);
}
String inputPath = args[0];
String outputPath = args[1];
Configuration conf = this.getConf();
conf.setBoolean("mapred.output.compress", true);
conf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
Job job = Job.getInstance(conf);
job.setJarByClass(MaxTemperature.class);
job.setMapperClass(MaxTemperatureMapper.class);
job.setReducerClass(MaxTemperatureReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
} ![Java String.format方法的簡單使用[圖]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)