How to scrape a website that requires login first with Python Scrapy-framework (Filling login forms automatically)

We often have to write spiders that need to login to sites, in order to scrape data from them. Our customers provide us with the site, username and password, and we do the rest.

The classic way to approach this problem is:

1.     launch a browser, go to site and search for the login page

2.     inspect the source code of the page to find out:

                      I.        which one is the login form (a page can have many forms, but usually one of them is the login form)

                     II.        which are the field names used for username and password (these could vary a lot)

                    III.        if there are other fields that must be submitted (like an authentication token)

3.     write the Scrapy spider to replicate the form submission using FormRequest

Being fans of automation, we figured we could write some code to automate point 2 (which is actually the most time-consuming) and the result is login form, a library to automatically fill login forms given the login page, username and password.

Here is the code of a simple spider that would use loginform to login to sites automatically.

# ------------------------------------------------------------------------------- # Name: module1 # Purpose: # # Author: Ranvijay Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay Sachan 2014 # Licence: <your licence> # ------------------------------------------------------------------------------- from scrapy.spider import BaseSpider from scrapy.http import FormRequest from scrapy.http.request import Request from loginform import fill_login_form from scrapy import log from scraping.articles import ArticleItem class GitHubLogin(BaseSpider): name = 'GitHubLogin' allowed_domains = ['github.com'] start_urls = ['http://github.com/login'] login_user = 'ranvijay5686' login_pass = '' def parse(self, response): (args, url, method) = fill_login_form(response.url, response.body, self.login_user, self.login_pass) return FormRequest(url, method=method, formdata=args, callback=self.after_login) def after_login(self, response): # for link in response.xpath("//*[@id='site-container']/div[2]/div[4]/p/a/@href").extract(): item = ArticleItem() item['title'] = 'ranvijay' log.msg('*************** : ' + str(response.xpath("//form[@class='subnav-search left']/input/@value" ).extract())) item['url'] = \ response.xpath("//*[@id='site-container']/div[1]/div/div/span/span/text()" ).extract() yield item # ------------------------------------------------------------------------------- # Name: # Purpose: # # Author: Ranvijay Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay Sachan 2014 # Licence: <your licence> # ------------------------------------------------------------------------------- from scrapy.item import Item, Field class ArticleItem(Item): title = Field() url = Field() # ------------------------------------------------------------------------------- # Name: # Purpose: # # Author: Ranvijay Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay Sachan 2014 # Licence: <your licence> # ------------------------------------------------------------------------------- import sys from argparse import ArgumentParser from collections import defaultdict from lxml import html __version__ = '1.0' # also update setup.py def _form_score(form): score = 0 # In case of user/pass or user/pass/remember-me if len(form.inputs.keys()) in (2, 3): score += 10 typecount = defaultdict(int) for x in form.inputs: type_ = (x.type if isinstance(x, html.InputElement) else 'other' ) typecount[type_] += 1 if typecount['text'] > 1: score += 10 if not typecount['text']: score -= 10 if typecount['password'] == 1: score += 10 if not typecount['password']: score -= 10 if typecount['checkbox'] > 1: score -= 10 if typecount['radio']: score -= 10 return score def _pick_form(forms): """Return the form most likely to be a login form""" return sorted(forms, key=_form_score, reverse=True)[0] def _pick_fields(form): """Return the most likely field names for username and password""" userfield = passfield = emailfield = None for x in form.inputs: if not isinstance(x, html.InputElement): continue type_ = x.type if type_ == 'password' and passfield is None: passfield = x.name elif type_ == 'text' and userfield is None: userfield = x.name elif type_ == 'email' and emailfield is None: emailfield = x.name return (userfield or emailfield, passfield) def submit_value(form): """Returns the value for the submit input, if any""" for x in form.inputs: if x.type == 'submit' and x.name: return [(x.name, x.value)] else: return [] def fill_login_form( url, body, username, password, ): doc = html.document_fromstring(body, base_url=url) form = _pick_form(doc.xpath('//form')) (userfield, passfield) = _pick_fields(form) form.fields[userfield] = username form.fields[passfield] = password form_values = form.form_values() + submit_value(form) return (form_values, form.action or form.base_url, form.method) def main(): ap = ArgumentParser() ap.add_argument('-u', '--username', default='username') ap.add_argument('-p', '--password', default='secret') ap.add_argument('url') args = ap.parse_args() try: import requests except ImportError: print 'requests library is required to use loginform as a tool' r = requests.get(args.url) (values, action, method) = fill_login_form(args.url, r.text, args.username, args.password) print '''url: {0} method: {1} payload:'''.format(action, method) for (k, v) in values: print '- {0}: {1}'.format(k, v) if __name__ == '__main__': sys.exit(main())