pycurl库使用-1

pycurl 绑定绑定ip

pycurl使用代理服务器

if self.proxy_info is not None:
self.crl.setopt(pycurl.PROXY, self.proxy_info.server)   #127.0.0.1:8088
if self.proxy_info.user_password != '':
self.crl.setopt(pycurl.PROXYUSERPWD, self.proxy_info.user_password)   #user:password

其他NETWORK OPTIONS 来自黄聪：Python网站采集功能（多线程的采集、WDPYSPIDER类、pycurl）

CURLOPT_URL: http://xxxx,ftp://xxxx
CURLOPT_PROXY:HTTP代理,主机名或IP地址
CURLOPT_PROXYPORT:代理端口,也可在PROXY的地址后加":端口",如 :8080
CURLOPT_PROXYTYPE:代理类型,CURLPROXY_HTTP(默认), CURLPROXY_HTTP_1_0,CURLPROXY_SOCKS4,CURLPROXY_SOCKS5,CURLPROXY_SOCKS4A,CURLPROXY_SOCKS5_HOSTNAME,
CURLOPT_NOPROXY:不使用代理的域
CURLOPT_HTTPPROXYTUNNEL:
CURLOPT_BUFFERSIZE: libcurl的缓冲区大小（以字节为单位）
（认证）
CURLOPT_NETRC: 此参数控制你的密码,CURL_NETRC_OPTIONAL使用 ~/.netrc 文件, CURL_NETRC_IGNORED(默认):忽略文件,CURL_NETRC_REQUIRED:告诉该文件的使用所需的库，要忽略的URL信息
CURLOPT_NETRC_FILE: 指定 ~/.netrc 文件
CURLOPT_USERNAME:
CURLOPT_USERPWD:
CURLOPT_PASSWORD:
CURLOPT_PROXYUSERNAME:
CURLOPT_PROXYUSERPWD:
CURLOPT_HTTPAUTH:
CURLOPT_PROXYAUTH:

• CURLAUTH_BASIC: HTTP基本验证
• CURLAUTH_DIGEST: HTTP摘要身份验证
• CURLAUTH_DIGEST_IE:
• CURLAUTH_GSSNEGOTIATE: Kerberos5认证 要建立GSS - API
• CURLAUTH_NTLM: NTLM身份验证
• CURLAUTH_ANY: 设置所有选项,ibcurl自动选择一个它认为合适的,安全的验证<
• CURLAUTH_ANYSAFE: 设置基本选项....
• CURLAUTH_ONLY: 强制所有请求使用验证

def _curl_setup_request(curl, request, buffer, headers):
    curl.setopt(pycurl.URL, request.url)
    curl.setopt(pycurl.HTTPHEADER,
                ["{72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s: {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s" {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea} i for i in request.headers.iteritems()])
    try:
        curl.setopt(pycurl.HEADERFUNCTION,
                    functools.partial(_curl_header_callback, headers))
    except:
        # Old version of curl; response will not include headers
        pass
    if request.streaming_callback:
        curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback)
    else:
        curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
    curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
    curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
    curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
    curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
    if request.user_agent:
        curl.setopt(pycurl.USERAGENT, request.user_agent)
    else:
        curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
    if request.network_interface:
        curl.setopt(pycurl.INTERFACE, request.network_interface)
    if request.use_gzip:
        curl.setopt(pycurl.ENCODING, "gzip,deflate")
    else:
        curl.setopt(pycurl.ENCODING, "none")

    # Set the request method through curl's retarded interface which makes
    # up names for every single method
    curl_options = {
        "GET": pycurl.HTTPGET,
        "POST": pycurl.POST,
        "PUT": pycurl.UPLOAD,
        "HEAD": pycurl.NOBODY,
    }
    for o in curl_options.values():
        curl.setopt(o, False)
    curl.setopt(curl_options[request.method], True)

    # Handle curl's cryptic options for every individual HTTP method
    if request.method in ("POST", "PUT"):
        request_buffer =  cStringIO.StringIO(request.body)
        curl.setopt(pycurl.READFUNCTION, request_buffer.read)
        if request.method == "POST":
            def ioctl(cmd):
                if cmd == curl.IOCMD_RESTARTREAD:
                    request_buffer.seek(0)
            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
            curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
        else:
            curl.setopt(pycurl.INFILESIZE, len(request.body))

    if request.auth_username and request.auth_password:
        userpwd = "{72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s:{72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s" {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea} (request.auth_username, request.auth_password)
        curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
        curl.setopt(pycurl.USERPWD, userpwd)
        logging.info("{72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s (username: {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}r)", request.method, request.url,
                     request.auth_username)
    else:
        curl.unsetopt(pycurl.USERPWD)
        logging.info("{72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s {72a546303a8f98ff9b20f4c6d84395140df1f51ea7d87899c02ff8df0c9c9cea}s", request.method, request.url)