HDFS API入門

@author wxy

@date 6/17/2015 8:54:09 AM

一般流程

static FileSystem get(Configuration conf)
operator()
{
    //step1
    得到Configuration對象
    //step2
    得到FileSystem對象
    //step3
    進行檔案操作
}
           

初始化

将hadoop/conf下的core-site.xml,hdfs-site.xml,mapred-site.xml複制到項目的根目錄下。運作下面程式對conf初始化，獲得fs

private static String CORE_SITE_PATH = "core-site.xml";
private static String HDFS_SITE_PATH = "hdfs-site.xml";
private static String MAPRED_SITE_PATH = "mapred-site.xml";

public hdfsUtils(){
    if(conf == null){
        conf = new Configuration();
        conf.addResource(new Path(CORE_SITE_PATH));
        conf.addResource(new Path(HDFS_SITE_PATH));
        conf.addResource(new Path(MAPRED_SITE_PATH));
        try {
            fs = FileSystem.get(conf);
        } catch (Exception e) {
            // TODO: handle exception
            e.printStackTrace();
        }
    }
}
           

示例

//将本地檔案上傳到hadoop檔案系統中
public void copyLocalFileToHadoop(String src, String dst){
    Path srcPath = new Path(src);
    Path dstPath = new Path(dst);
    try {
        fs.copyFromLocalFile(srcPath,dstPath);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}