class Manager(object):
def __init__(self):
self.old_urls=set()
self.new_urls=set()
def add_new_urls(self, root_url):
for url in root_url:
if(root_url not in self.old_urls and root_url not in self.new_urls):
self.new_urls.add(url)
def has_new_url(self):
if(len(self.new_urls) > 0):
return True
else:
return False
def get_new_url(self):
tmp=self.new_urls.pop()
self.old_urls.add(tmp)
return tmp
HTML下載下傳器
HtmlDownloader.py
import urllib.request
class Downloader(object):
def load(self, curr_url):
html = urllib.request.urlopen(curr_url)
return html.read().decode("UTF-8",'ignore')
HTML解析器
HtmlParser.py
import re
from urllib.parse import urljoin
from bs4 import BeautifulSoup
class Parser(object):
def __init__(self):
self.title = ''
self.urls = set()
self.lemmasummary = ''
def parse(self, curr_url, html_content):
soup = BeautifulSoup(html_content, 'html.parser',from_encoding="utf-8")
soupurls=soup.find_all('a',href=re.compile(r'/view'))
for url in soupurls:
self.urls.add(urljoin(curr_url,url['href']))
self.title = soup.title.string.split("_")[0]
self.lemmasummary = soup.find('div',class_='lemma-summary')
return self.title,self.urls,self.lemmasummary
結果展示器
HtmlOutputer.py
class Outputer(object):
def __init__(self):
self.table = []
def collect(self, data):
self.table.append(data)
def genhtml(self):
html = open('result.html','w')
html.write("<html>")
html.write("<head>")
html.write("</head>")
html.write("<body>")
html.write("<table>")
for data in self.table:
html.write("<tr>")
html.write("<td>%s</td>" % data['word'])
html.write("<td>%s</td>" % data['curr_url'])
html.write("<td>%s</td>" % data['lemmasummary'])
html.write("</tr>")
html.write("</table>")
html.write("</body>")
html.write("</html>")
html.close()
