java多线程下载美女图片

Main.java

主线程，控制两个线程池，待任务处理完成后关闭线程池

package photo;

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;

public class Main {

    static final String IMG_ROOT = "D:/meinv/";

    public static void main(String[] args) {
        main2(args);
    }

    public static void main1(String[] args) {
        long begin = System.currentTimeMillis();
        Spider s = new Spider();
        String url = "http://me2-sex.lofter.com/tag/%E7%BE%8E%E5%A5%B3%E6%91%84%E5%BD%B1";
        for (int page = ; page <= ; page++) {
            String content = s.getContent(url, page);
            List<String> urlList = s.parseContent(content);
            for (String picUrl : urlList) {
                int index = picUrl.lastIndexOf("=");
                String file = IMG_ROOT + picUrl.substring(index + );
                s.downPhoto(picUrl, file);
            }
        }
        long end = System.currentTimeMillis();
        System.out.println("total cost time:" + (end - begin) /  + "s");
    }

    public static void main2(String[] args) {
        LinkedBlockingDeque<String> photoUrlQue = new LinkedBlockingDeque<String>();
        ExecutorService executor = Executors.newFixedThreadPool();
        for (int page = ; page <= ; page++) {
            executor.execute(new DownPage(photoUrlQue, page));
        }
        ExecutorService photoexecutor = Executors.newFixedThreadPool();
        Status status = new Status();
        for (int i = ; i < ; i++) {
            photoexecutor.execute(new DownPhoto(photoUrlQue,status));
        }
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                Thread.sleep();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        status.doStop();
        photoexecutor.shutdown();

    }
}
           

DownPhoto.java

下载图片的线程，从队列中获得url，下载并保存到本地磁盘

package photo;

import java.util.concurrent.LinkedBlockingDeque;

public class DownPhoto implements Runnable {

    static final String         IMG_ROOT = "D:/meinv/";
    LinkedBlockingDeque<String> photoUrlQue;
    private Status status;
    public DownPhoto(LinkedBlockingDeque<String> photoUrlQue,Status status){
        this.photoUrlQue = photoUrlQue;
        this.status = status;
    }

    @Override
    public void run() {
        while (true) {
            try {
                if(photoUrlQue.isEmpty() && status.isStoped()){
                    break;
                }
                String picUrl = photoUrlQue.take();
                Spider s = new Spider();
                int index = picUrl.lastIndexOf("=");
                String file = IMG_ROOT + picUrl.substring(index + );
                s.downPhoto(picUrl, file);
            } catch (InterruptedException e) {
                break;
            }
        }
    }

}
           

DownPage.java

下载网页内容，并分析图片url，存放到队列中，以供另外一个线程使用

package photo;

import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;

public class DownPage implements Runnable {

    LinkedBlockingDeque<String> photoUrlQue;
    int                         page;
    String                      url = "http://me2-sex.lofter.com/tag/%E7%BE%8E%E5%A5%B3%E6%91%84%E5%BD%B1";

    public DownPage(LinkedBlockingDeque<String> photoUrlQue, int page){
        this.photoUrlQue = photoUrlQue;
        this.page = page;
    }

    @Override
    public void run() {
        Spider s = new Spider();
        String content = s.getContent(url, page);
        List<String> urlList = s.parseContent(content);
        for (String picUrl : urlList) {
            photoUrlQue.push(picUrl);
        }

    }
}
           

Spider.java

http通讯工具类

需要引入apache的commons-httpclient、commons-codec、commons-logging包

package photo;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;

public class Spider {

    public String getContent(String url, int page) {
        HttpClient client = new HttpClient();
        url += "?page=" + page;
        HttpMethod method = new PostMethod(url);
        method.setFollowRedirects(false);
        method.addRequestHeader("User-Agent",
                                "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31");
        try {
            int status = client.executeMethod(method);
            if (status != ) {
                return null;
            }
            return method.getResponseBodyAsString();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            method.releaseConnection();
        }
        return null;
    }

    public void downPhoto(String url, String fileName) {
        HttpClient client = new HttpClient();
        HttpMethod method = new GetMethod(url);
        method.setFollowRedirects(false);
        method.addRequestHeader("User-Agent",
                                "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31");
        try {
            int status = client.executeMethod(method);
            if (status != ) {
                return;
            }
            InputStream is = method.getResponseBodyAsStream();
            BufferedInputStream bis = new BufferedInputStream(is);
            File f = new File(fileName);
            OutputStream os = new FileOutputStream(f);
            BufferedOutputStream bos = new BufferedOutputStream(os);
            byte[] buffer = new byte[];
            int readLen = -;
            while ((readLen = bis.read(buffer)) != -) {
                bos.write(buffer, , readLen);
                bos.flush();
            }
            bos.close();
            bis.close();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            method.releaseConnection();
        }

    }

    public static void main(String[] args) {

    }

    public List<String> parseContent(String content) {
        List<String> urlList = new ArrayList<String>();
        // <img src="http://imglf0.ph.126.net/BKehcpizRueVO8BuzwT8Lg==/6630835768745140932.jpg" />
        Pattern p = Pattern.compile("<img\\s*src=\"(\\S+)\"\\s*/>");
        // String input
        Matcher m = p.matcher(content);
        while (m.find()) {
            String url = m.group();
            //System.out.println(url);
            urlList.add(url);
        }
        return urlList;
    }

}
           

Status

状态控制类

package photo;

public class Status {

    private volatile boolean stop;

    public void doStop() {
        stop = true;
    }

    public boolean isStoped() {
        return stop;
    }
}