复制代码 代码如下:

#-*- coding: UTF-8 -*-

'''

Created on 2013-12-5

@author: good-temper

'''

import urllib2

import bs4

import time

def getPage(urlStr):

'''

获取页面内容

'''

content = urllib2.urlopen(urlStr).read()

return content

def getNextPageUrl(currPageNum):

#http://list.jd.com/9987-653-655-0-0-0-0-0-0-0-1-1-页码-1-1-72-4137-33.html

url = u'http://list.jd.com/9987-653-655-0-0-0-0-0-0-0-1-1-'+str(currPageNum+1)+'-1-1-72-4137-33.html'

#是否有下一页

content = getPage(url);

soup = bs4.BeautifulSoup(content)

list = soup.findAll('span',{'class':'next-disabled'});

if(len(list) == 0):

return url

return ''

def analyzeList():

pageNum = 0

list = []

url = getNextPageUrl(pageNum)

while url !='':

soup = bs4.BeautifulSoup(getPage(url))

pagelist = soup.findAll('div',{'class':'p-name'})

for elem in pagelist:

soup1 = bs4.BeautifulSoup(str(elem))

list.append(soup1.find('a')['href'])

pageNum = pageNum+1

print pageNum

url = getNextPageUrl(pageNum)

return list

def analyzeContent(url):

return ''

def writeToFile(list, path):

f = open(path, 'a')

for elem in list:

f.write(elem+'n')

f.close()

if __name__ == '__main__':

list = analyzeList()

print '共抓取'+str(len(list))+'条n'

writeToFile(list, u'E:jd_phone_list.dat');