山西省消防总队建设工程备案网站,公司网站开发费用,做网站的技术体系,网站项目策划书模板本文实例讲述了Python实现在线程里运行scrapy的方法。分享给大家供大家参考。具体如下#xff1a;
如果你希望在一个写好的程序里调用scrapy#xff0c;就可以通过下面的代码#xff0c;让scrapy运行在一个线程里。Code to run Scrapy crawler in a thr…本文实例讲述了Python实现在线程里运行scrapy的方法。分享给大家供大家参考。具体如下
如果你希望在一个写好的程序里调用scrapy就可以通过下面的代码让scrapy运行在一个线程里。Code to run Scrapy crawler in a thread - works on Scrapy 0.8import threading, Queue
from twisted.internet import reactor
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.manager import scrapymanager
from scrapy.core.engine import scrapyengine
from scrapy.core import signals
class CrawlerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.running False
def run(self):
self.running True
scrapymanager.configure(control_reactorFalse)
scrapymanager.start()
reactor.run(installSignalHandlersFalse)
def crawl(self, *args):
if not self.running:
raise RuntimeError(CrawlerThread not running)
self._call_and_block_until_signal(signals.spider_closed, \
scrapymanager.crawl, *args)
def stop(self):
reactor.callFromThread(scrapyengine.stop)
def _call_and_block_until_signal(self, signal, f, *a, **kw):
q Queue.Queue()
def unblock():
q.put(None)
dispatcher.connect(unblock, signalsignal)
reactor.callFromThread(f, *a, **kw)
q.get()
# Usage example below:
import os
os.environ.setdefault(SCRAPY_SETTINGS_MODULE, myproject.settings)
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core import signals
from scrapy.conf import settings
from scrapy.crawler import CrawlerThread
settings.overrides[LOG_ENABLED] False # avoid log noise
def item_passed(item):
print Just scraped item:, item
dispatcher.connect(item_passed, signalsignals.item_passed)
crawler CrawlerThread()
print Starting crawler thread...
crawler.start()
print Crawling somedomain.com....
crawler.crawl(somedomain.com) # blocking call
print Crawling anotherdomain.com...
crawler.crawl(anotherdomain.com) # blocking call
print Stopping crawler thread...
crawler.stop()
希望本文所述对大家的Python程序设计有所帮助。