对输入来自文本框的网页进行爬网

该网站使用 Form 。所以你需要制作剪贴簿来填写字段并提交表单。我已经编译了一些代码来说明如何使用 scrapy :

import scrapy

class TestSpider(scrapy.Spider):

    name = 'casestatus'
    start_urls = ['https://egov.uscis.gov/casestatus/landing.do']

    def parse(self, response):

        request = scrapy.FormRequest.from_response(
            response,
            formname='caseStatusForm',
            formdata={'appReceiptNum': 'EAC1590674053'},
            callback=self.parse_caseStatus
        )
        print request.body
        yield request

    def parse_caseStatus(self,response):
        sel_current_status = response.xpath('//div[contains(@class,"current-status")]')
        if sel_current_status:
            txt_current_status = sel_current_status.xpath('./text()').extract()
            txt_current_status = " ".join(map(unicode.strip,txt_current_status))
            print txt_current_status
        else:
            print 'NO STATUS FOUND'

# YIELDS THE FOLLOWING OUTPUT FOR ME:
# [casestatus] DEBUG: Crawled (200) <POST https://egov.uscis.gov/casestatus/mycasestatus.do;jsessionid=A19A03FC933A208A2DDF89D98BE9F32E> (referer: https://egov.uscis.gov/casestatus/landing.do)
# Case Rejected Because I Sent An Incorrect Fee