общий паук для проекта scrapy

Question

Я создаю общий паук (scrapy spider) для нескольких сайтов. ниже - структура моего проекта.

myproject <Directory> 
--- __init__.py 
--- common.py 
--- scrapy.cfg 
--- myproject <Directory> 
    ---__init__.py 
    ---items.py 
    ---pipelines.py 
    ---settings.py 
    ---spiders <Directory> 
     ---__init__.py 
     ---spider.py    (generic spider) 
     ---stackoverflow_com.py (spider per website) 
     ---anotherwebsite1_com.py (spider per website) 
     ---anotherwebsite2_com.py (spider per website) 

common.py

#!/usr/bin/env python 
# -*- coding: UTF-8 -*- 
# 
''' 
    common file 
''' 

from scrapy.spider import BaseSpider 
from scrapy.selector import HtmlXPathSelector 
from scrapy.crawler import CrawlerProcess 
from scrapy.stats import stats 
from scrapy.http import Request 
from WSS.items import WssItem 
import MySQLdb, time 
import urllib2, sys 

#Database connection 
def open_database_connection(): 
    connection = MySQLdb.connect(user=db_user, passwd=db_password, db=database, host=db_host, port=db_port, charset="utf8", use_unicode=True) 
    cursor = connection.cursor() 
    return connection, cursor 

def close_database_connection(cursor, connection): 
    cursor.close() 
    connection.close() 
    return 

class Domain_: 
    def __init__(self, spider_name, allowed_domains, start_urls, extract_topics_xpath, extract_viewed_xpath): 
     self.spider_name = spider_name 
     self.extract_topics_xpath = extract_topics_xpath 
     self.extract_viewed_xpath = extract_viewed_xpath 
     self.allowed_domains = allowed_domains 
     self.start_urls = start_urls 

spider.py

#!/usr/bin/env python 
# -*- coding: UTF-8 -*- 
# 

from common import * 

class DomainSpider(BaseSpider): 
    name = "generic_spider" 
    def __init__(self, current_domain): 
     self.allowed_domains = current_domain.allowed_domains 
     self.start_urls = current_domain.start_urls 
     self.current_domain = current_domain 

    def parse(self, response): 
     hxs = HtmlXPathSelector(response) 
     for topics in list(set(hxs.select(self.current_domain.extract_topics_xpath).extract())): 
      yield Request(topics, dont_filter=True, callback=self.extract_all_topics_data) 

    def extract_all_topics_data(self, response): 
     hxs = HtmlXPathSelector(response) 
     item = WssItem() 
     print "Processing "+response.url 
     connection, cursor = open_database_connection() 
     for viewed in hxs.select(self.current_domain.extract_viewed_xpath).extract(): 
        item['TopicURL'] = response.url 
        item['Topic_viewed'] = viewed 
        yield item 
     close_database_connection(cursor, connection) 
     return 

stackoverflow_com.py

#!/usr/bin/env python 
# -*- coding: UTF-8 -*- 
# 

from common import * 

current_domain = Domain_(
    spider_name = 'stackoverflow_com', 
    allowed_domains = ["stackoverflow.com"], 
    start_urls = ["http://stackoverflow.com/"], 
    extract_topics_xpath = '//div[contains(@class,\"bottomOrder\")]/a/@href', 
    extract_viewed_xpath = '//div[contains(@class,\"views\")]/text()' 
    ) 

import WSS.spiders.spider as spider 
StackOverflowSpider = spider.DomainSpider(current_domain) 

из приведенных выше сценариев, я не хочу трогать паука .py (при условии, что все веб-сайты, имеющие одну структуру o я могу использовать spider.py для всех пауков)

Я просто хочу создать новых пауков на веб-сайт, таких как stackoverflow_com.py, и я хочу вызвать spider.py для процесса обхода.

Можете ли вы, пожалуйста, посоветовать мне, что-то не так в моем коде ?. он показывает ниже сообщение об ошибке

output1: если я бегу «Scrapy ползания stackoverflow_com» это показывает ниже сообщение об ошибке

C:\myproject>scrapy crawl stackoverflow_com 
2013-08-05 09:41:45+0400 [scrapy] INFO: Scrapy 0.16.4 started (bot: WSS) 
2013-08-05 09:41:45+0400 [scrapy] DEBUG: Enabled extensions: LogStats, TelnetConsole, CloseSpider, WebService, CoreStats, SpiderState 
2013-08-05 09:41:45+0400 [scrapy] DEBUG: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, RedirectMiddleware, CookiesMiddleware, HttpCompressionMiddleware, ChunkedTransferMiddleware, DownloaderStats 
2013-08-05 09:41:45+0400 [scrapy] DEBUG: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware 
2013-08-05 09:41:45+0400 [scrapy] DEBUG: Enabled item pipelines: WssPipeline 
Traceback (most recent call last): 
    File "C:\Python27\lib\runpy.py", line 162, in _run_module_as_main 
    "__main__", fname, loader, pkg_name) 
    File "C:\Python27\lib\runpy.py", line 72, in _run_code 
    exec code in run_globals 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 156, in <module> 
    execute() 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 131, in execute 
    _run_print_help(parser, _run_command, cmd, args, opts) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 76, in _run_print_help 
    func(*a, **kw) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 138, in _run_command 
    cmd.run(args, opts) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\commands\crawl.py", line 43, in run 
    spider = self.crawler.spiders.create(spname, **opts.spargs) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\spidermanager.py", line 43, in create 
    raise KeyError("Spider not found: %s" % spider_name) 
KeyError: 'Spider not found: stackoverflow_com' 

output2: если я бегу «Scrapy ползать generic_spider» это показывает ниже сообщение об ошибке

C:\myproject>scrapy crawl generic_spider 
2013-08-05 12:25:15+0400 [scrapy] INFO: Scrapy 0.16.4 started (bot: WSS) 
2013-08-05 12:25:15+0400 [scrapy] DEBUG: Enabled extensions: LogStats, TelnetConsole, CloseSpider, WebService, CoreStats, SpiderState 
2013-08-05 12:25:16+0400 [scrapy] DEBUG: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, RedirectMiddleware, CookiesMiddleware, HttpCompressionMiddleware, ChunkedTransferMiddleware, DownloaderStats 
2013-08-05 12:25:16+0400 [scrapy] DEBUG: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware 
2013-08-05 12:25:16+0400 [scrapy] DEBUG: Enabled item pipelines: WssPipeline 
Traceback (most recent call last): 
    File "C:\Python27\lib\runpy.py", line 162, in _run_module_as_main 
    "__main__", fname, loader, pkg_name) 
    File "C:\Python27\lib\runpy.py", line 72, in _run_code 
    exec code in run_globals 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 156, in <module> 
    execute() 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 131, in execute 
    _run_print_help(parser, _run_command, cmd, args, opts) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 76, in _run_print_help 
    func(*a, **kw) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\cmdline.py", line 138, in _run_command 
    cmd.run(args, opts) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\commands\crawl.py", line 43, in run 
    spider = self.crawler.spiders.create(spname, **opts.spargs) 
    File "C:\Python27\lib\site-packages\scrapy-0.16.4-py2.7.egg\scrapy\spidermanager.py", line 44, in create 
    return spcls(**spider_kwargs) 
TypeError: __init__() takes exactly 2 arguments (1 given) 

заранее благодарю :)

Answer 1

Try следуя шаблону из http://doc.scrapy.org/en/latest/topics/spiders.html#spider-arguments

class DomainSpider(BaseSpider): 
    name = "generic_spider" 
    def __init__(self, current_domain, *args, **kwargs): 
     self.allowed_domains = current_domain.allowed_domains 
     self.start_urls = current_domain.start_urls 
     super(DomainSpider, self).__init__(*args, **kwargs) 
     self.current_domain = current_domain 
    ... 

Answer 2

Согласно моему опыту, scrapy throws spider не найдена ошибка, только если имя паук не определено. Убедитесь, что имя вашего паука т.е. stackoverflow_com становится назначать в DomainSpider попробуйте добавить

self.name=current_domain.spider_name