scrapy ja3 tls
  QFbc3MtZbTbh 2023年11月02日 28 0

 

 

 

# -*- coding:utf-8 -*-
import random

from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler, ScrapyAgent

ORIGIN_CIPHERS = 'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-128-GCM-SHA256:ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES'


def shuffle_ciphers():
    ciphers = ORIGIN_CIPHERS.split(":")
    random.shuffle(ciphers)

    ciphers = ":".join(ciphers)

    print("________")
    print(ciphers)
    print("________")

    return ciphers + ":!aNULL:!MD5:!DSS"


class MyHTTPDHandler(HTTP11DownloadHandler):

    def download_request(self, request, spider):
        """Return a deferred for the HTTP download"""

        tls_cliphers = shuffle_ciphers()
        _contextFactory = ScrapyClientContextFactory(tls_ciphers=tls_cliphers)

        agent = ScrapyAgent(
            contextFactory=_contextFactory,
            pool=self._pool,
            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
            fail_on_dataloss=self._fail_on_dataloss,
            crawler=self._crawler,
        )
        return agent.download_request(request)


class MyHTTPDownloadHandler(MyHTTPDHandler):
    def download_request(self, request, spider):
        return super().download_request(request, spider)

 

 

 

 

 

 

'DOWNLOAD_HANDLERS': {
             'http': 'middlewares.sc_middlewares.MyHTTPDownloadHandler',
             'https': 'middlewares.sc_middlewares.MyHTTPDownloadHandler',
         },

 



【版权声明】本文内容来自摩杜云社区用户原创、第三方投稿、转载,内容版权归原作者所有。本网站的目的在于传递更多信息,不拥有版权,亦不承担相应法律责任。如果您发现本社区中有涉嫌抄袭的内容,欢迎发送邮件进行举报,并提供相关证据,一经查实,本社区将立刻删除涉嫌侵权内容,举报邮箱: cloudbbs@moduyun.com

  1. 分享:
最后一次编辑于 2023年11月08日 0

暂无评论

QFbc3MtZbTbh