复制代码 代码如下:

#!/usr/bin/python

#-*-coding:utf-8-*-

# JCrawler

# Author: Jam <810441377@qq.com>

import time

import urllib2

from bs4 import BeautifulSoup

# 目标站点

TargetHost = "http://adirectory.blog.com"

# User Agent

UserAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'

# 链接采集规则

# 目录链接采集规则

CategoryFind = [{'findMode':'find','findTag':'div','rule':{'id':'cat-nav'}},

{'findMode':'findAll','findTag':'a','rule':{}}]

# 文章链接采集规则

ArticleListFind = [{'findMode':'find','findTag':'div','rule':{'id':'content'}},

{'findMode':'findAll','findTag':'h2','rule':{'class':'title'}},

{'findMode':'findAll','findTag':'a','rule':{}}]

# 分页URL规则

PageUrl = 'page/#page/'

PageStart = 1

PageStep = 1

PageStopHtml = '404: Page Not Found'

def GetHtmlText(url):

request = urllib2.Request(url)

request.add_header('Accept', "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp")

request.add_header('Accept-Encoding', "*")

request.add_header('User-Agent', UserAgent)

return urllib2.urlopen(request).read()

def ArrToStr(varArr):

returnStr = ""

for s in varArr:

returnStr += str(s)

return returnStr

def GetHtmlFind(htmltext, findRule):

findReturn = BeautifulSoup(htmltext)

returnText = ""

for f in findRule:

if returnText != "":

findReturn = BeautifulSoup(returnText)

if f['findMode'] == 'find':

findReturn = findReturn.find(f['findTag'], f['rule'])

if f['findMode'] == 'findAll':

findReturn = findReturn.findAll(f['findTag'], f['rule'])

returnText = ArrToStr(findReturn)

return findReturn

def GetCategory():

categorys = [];

htmltext = GetHtmlText(TargetHost)

findReturn = GetHtmlFind(htmltext, CategoryFind)

for tag in findReturn:

print "[G]->Category:" + tag.string + "|Url:" + tag['href']

categorys.append({'name': tag.string, 'url': tag['href']})

return categorys;

def GetArticleList(categoryUrl):

articles = []

page = PageStart

#pageUrl = PageUrl

while True:

htmltext = ""

pageUrl = PageUrl.replace("#page", str(page))

print "[G]->PageUrl:" + categoryUrl + pageUrl

while True:

try:

htmltext = GetHtmlText(categoryUrl + pageUrl)

break

except urllib2.HTTPError,e:

print "[E]->HTTP Error:" + str(e.code)

if e.code == 404:

htmltext = PageStopHtml

break

if e.code == 504:

print "[E]->HTTP Error 504: Gateway Time-out, Wait"

time.sleep(5)

else:

break

if htmltext.find(PageStopHtml) >= 0:

print "End Page."

break

else:

findReturn = GetHtmlFind(htmltext, ArticleListFind)

for tag in findReturn:

if tag.string != None and tag['href'].find(TargetHost) >= 0:

print "[G]->Article:" + tag.string + "|Url:" + tag['href']

articles.append({'name': tag.string, 'url': tag['href']})

page += 1

return articles;

print "[G]->GetCategory"

Mycategorys = GetCategory();

print "[G]->GetCategory->Success."

time.sleep(3)

for category in Mycategorys:

print "[G]->GetArticleList:" + category['name']

GetArticleList(category['url'])