from scrapy.spiders import Spiderfrom scrapy.selector import Selectorfrom dirbot.items import Websiteclass DmozSpider(Spider):name = "dmoz"allowed_domains = ["dmoz.org"]start_urls = ["http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",]def parse(self, response):sites = response.css('#site-list-content > div.site-item > div.title-and-desc')items = []for site in sites: item = Website() item['name'] = site.css( 'a > div.site-title::text').extract_first().strip() item['url'] = site.xpath( 'a/@href').extract_first().strip() item['description'] = site.css( 'div.site-descr::text').extract_first().strip() items.append(item) return items上述代码中,待爬取的网页不可能包含( )。
Ahttp://www.dmoz.org/index.html[|]http://www.dmoz.org/register.html[|]http://www.dmoz1.org/[|]http://ftp.dmoz.org/