Hadoop HDFS Java程式設計

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.net.URI;

import org.apache.commons.io.IOUtils;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.LocatedFileStatus;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.RemoteIterator;

import org.junit.Before;

import org.junit.Test;

/**

 * HDFS java程式設計

 * 一般用hdfs shell 來操作,如下：

 * hadoop fs -put aa.txt /

 * hadoop fs -get /aa.txt

 * hadoop fs -ls / 

 * hadoop fs -cat /aa.txt

 *

 */

public class HdfsUtil {

FileSystem fs = null;

@Before

public void init() throws Exception{

//讀取classpath下的xxx-site.xml 配置檔案，并解析其内容，封裝到conf對象中

Configuration conf = new Configuration();

//也可以在代碼中對conf中的配置資訊進行手動設定，會覆寫掉配置檔案中的讀取的值

conf.set("fs.defaultFS", "hdfs://node1:9000/");

//根據配置資訊，去擷取一個具體檔案系統的用戶端操作執行個體對象

fs = FileSystem.get(new URI("hdfs://node1:9000/"),conf,"username");

}

* 上傳檔案，比較底層的寫法

* 

* @throws Exception

*/

@Test

public void upload() throws Exception {

FileSystem fs = FileSystem.get(new URI("hdfs://node1:9000/"),conf,"username");

Path dst = new Path("hdfs://node1:9000/aa/aa2.txt");

FSDataOutputStream os = fs.create(dst);

FileInputStream is = new FileInputStream("d:/aa.txt");

IOUtils.copy(is, os);

* 上傳檔案，封裝好的寫法

* @throws IOException

public void upload2() throws Exception, IOException{

fs.copyFromLocalFile(new Path("d:/aa.txt"), new Path("hdfs://node1:9000/bb.txt"));

* 下載下傳檔案

* @throws Exception 

* @throws IllegalArgumentException 

public void download() throws Exception {

fs.copyToLocalFile(new Path("hdfs://node1:9000/aa/bb.txt"), new Path("d:/aa_down.txt"));

* 檢視檔案資訊

* @throws IOException 

* @throws FileNotFoundException 

public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {

// listFiles列出的是檔案資訊，而且提供遞歸周遊

RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);

while(files.hasNext()){

LocatedFileStatus file = files.next();

Path filePath = file.getPath();

String fileName = filePath.getName();

System.out.println(fileName);

System.out.println("---------------------------------");

//listStatus 可以列出檔案和檔案夾的資訊，但是不提供自帶的遞歸周遊

FileStatus[] listStatus = fs.listStatus(new Path("/"));

for(FileStatus status: listStatus){

String name = status.getPath().getName();

System.out.println(name + (status.isDirectory()?" is dir":" is file"));

* 建立檔案夾

public void mkdir() throws IllegalArgumentException, Exception {

fs.mkdirs(new Path("/aaa/bbb/ccc"));

* 删除檔案或檔案夾

public void rm() throws IllegalArgumentException, IOException {

fs.delete(new Path("/aa"), true);

public static void main(String[] args) throws Exception {

FileSystem fs = FileSystem.get(conf);

FSDataInputStream is = fs.open(new Path("/jdk-7u65-linux-i586.tar.gz"));

FileOutputStream os = new FileOutputStream("c:/jdk7.tgz");

本文轉自lzf0530377451CTO部落格，原文連結： http://blog.51cto.com/8757576/1837212，如需轉載請自行聯系原作者