Hadoop hdfs 基礎操作 client Api(1)

1. 初始化連接配接

@Before
public void setup() throws Exception {
    configuration = new Configuration();
    System.out.println();

    configuration.set("dfs.replication", "1");
    // configuration.set("hadoop.tmp.dir","/home/hadoop0/data/tmp");
    Iterator iterator = configuration.iterator();
    while(iterator.hasNext()
         ){
        System.out.println(iterator.next()); //列印所有配置或預設參數
    }
    fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, HDFS_USER);
    System.out.println("--------------setup-------------");
}

@After
public void teardown(){
    try {
        fileSystem.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    configuration = null;
    System.out.println("--------------teardown-------------");
}           

2. 新增

2.1 建立檔案夾

@Test
public void mkDir() throws Exception{
    System.out.println(fileSystem.mkdirs(new Path("/Complete")));
}           

2.2 建立檔案并寫入

@Test
public void create() throws Exception{
    FSDataOutputStream out = fileSystem.create(new Path("/HDFSApp/write/2.txt"));
    out.writeUTF("hello! first wirte to HDFS file!");
    out.flush();
    out.close();
}           

2.3 拷貝本地檔案到遠端

@Test
public void copyFromLocal() throws IOException {
    Path localFile =  new Path("F:/校驗大師_2.7.5.1632.zip");
    Path remoteFile = new Path("/HDFSApp/write/校驗大師_2.7.5.1632.zip");
    fileSystem.copyFromLocalFile(localFile, remoteFile);
}           

2.4 拷貝大檔案，帶進度條

@Test
public void copyFromLocalLargeFile(){
    String localFile = "F:/校驗大師_2.7.5.1632.zip";
    Path remoteFile = new Path("/HDFSApp/write/校驗大師_2.7.5.1633.zip");
    InputStream in = null;
    FSDataOutputStream out = null;

    try {
        in = new BufferedInputStream(new FileInputStream(localFile));
        out = fileSystem.create(remoteFile, new Progressable() {
            @Override
            public void progress() {
                System.out.println("!");
            }
        });
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("建立輸入輸出流失敗！");
    }

    if(in != null && out != null){
        try {
            IOUtils.copyBytes(in, out,2048, true);
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("上傳過程中失敗！");
        }
    }
}           

3. 查詢檢索

3.1 讀檔案内容

@Test
public void text() throws Exception{
    FSDataInputStream in = fileSystem.open(new Path("/output/part-r-00000"));
    IOUtils.copyBytes(in, System.out, 1024);
}           

3.2 下載下傳hdfs檔案到本地

@Test
public void copyToLocal() throws IOException {
    Path localFile =  new Path("F:/");
    Path remoteFile = new Path("/HDFSApp/write/校驗大師_2.7.5.1632.zip");
    fileSystem.copyToLocalFile(remoteFile, localFile);
}           

3.3 擷取路徑下内容

@Test
public void listFiles() throws IOException {
    Path path =  new Path("/HDFSApp/write");
    FileStatus[] files = fileSystem.listStatus(path);
    for (FileStatus file:files) {
        String type = file.isDirectory() ? "檔案夾" : "檔案";
        String permission = file.getPermission().toString();
        short replication = file.getReplication();
        String filePath = file.getPath().toString();

        System.out.println(type + "\t" + permission + "\t" + replication + "\t" + filePath);
    }
}           

3.4 深層周遊路徑下檔案

@Test
public void listFilesRecursive() throws IOException {
    Path path =  new Path("/");
    RemoteIterator<LocatedFileStatus> remoteIterator = fileSystem.listFiles(path, true);
    while(remoteIterator.hasNext()){
        LocatedFileStatus file = remoteIterator.next();

        String type = file.isDirectory() ? "檔案夾" : "檔案";
        String permission = file.getPermission().toString();
        short replication = file.getReplication();
        String filePath = file.getPath().toString();

        System.out.println(type + "\t" + permission + "\t" + replication + "\t" + filePath);
    }
}           

3.5 擷取檔案block資訊

@Test
public void getBlocksInfo() throws IOException {
    Path path =  new Path("/HDFSApp/write/校驗大師_2.7.5.1632.zip");
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    BlockLocation[] blockLocations =  fileSystem.getFileBlockLocations(fileStatus, 0 , fileStatus.getLen());

    for (BlockLocation block: blockLocations) {
        for (String name :block.getNames()) {
            System.out.println(name + ":" + block.getOffset() +"," + block.getLength());
        }
    }
}           

4. 修改

4.1 修改檔案名

@Test
public void rename() throws IOException {
    Path oPath =  new Path("/HDFSApp/write/0.txt");
    Path nPath = new Path("/HDFSApp/write/3.txt");
    boolean res = fileSystem.rename(oPath, nPath);
    System.out.println(res);
}           

5. 删除

5.1 删除檔案或目錄

public void delete() throws IOException {
    Path path =  new Path("/HDFSApp/write/校驗大師_2.7.5.1632.zip");
    boolean res = fileSystem.delete(path, false);
    System.out.println(res);
}