ret = self.doc.update_one({'announcementId':announcementId},{'$setOnInsert':item},upsert=True)

ret.matched_count, ret.modified_count

find . -type f -name "*.py" | xargs grep "redis"

# -*- coding: utf-8 -*-

# @Time : 2021/4/14 1:46

# @File : search_string_in_folder.py

# @Author : Rocky C@www.30daydo.com



'''

搜索代码脚本

'''

import fire

import glob

import re



# TODO 用PYQT重写一个



PATH_LIST = [r'C:\git\\',r'C:\OtherGit\\',r'C:\OneDrive\viewed_code\\']

POST_FIX = 'py' # 后缀文件

# 关键词

WORDS=[]



EXCLUDE_PATH=[r'C:\OtherGit\cpython']



DEBUG = True



class FileSearcher:



    def __init__(self,kw):

        self.root_path_list = PATH_LIST

        self.default_coding ='utf-8'

        self.exception_handle_coding='gbk'

        self.kw=[]

        if not isinstance(kw,tuple):

            kw=(kw,)



        for k in kw:

            k=k.strip()

            self.kw.append(k)



    def search(self,file,encoding):

        match_dict = dict()



        for w in self.kw:

            match_dict.setdefault(w, False)



        line_number = 0

        line_list=list()

        with open(file, 'r', encoding=encoding) as fp:



            while 1:

                try:

                    line = fp.readline()



                except UnicodeDecodeError as e:



                    if DEBUG:

                        print(f'Error coding in file {file}')

                        print(e)



                    return None,None,None



                except Exception as e:

                    if DEBUG:

                        print(f'Error in file {file}')

                        print(e)

                    break



                if not line:

                    break



                line = line.strip()

                if not line:

                    continue



                for w in self.kw:

                    m=re.search(w,line,re.IGNORECASE)

                    if m:

                        match_dict.update({w:True})

                        line_list.append(line_number)



                line_number+=1



        return True,match_dict.copy(),line_list.copy()



    def print_match_result(self,file,line_list,encoding):



        with open(file, 'r', encoding=encoding) as fp:

            line_number = 0

            while 1:

                try:

                    line = fp.readline()

                except Exception as e:

                    if DEBUG:

                        print(f'Error in file {file}')

                        print(e)

                    break



                if not line:

                    break

                line=line.strip()



                if not line:

                    continue



                if line_number in line_list:

                    print(f'{file} :: {line_number} ====>\n {line[:50]}\n')



                line_number += 1



    def run(self):

        for path in self.root_path_list:



            search_path=path+'**/*.'+POST_FIX



            for file in glob.iglob(search_path,recursive=True):



                for ex_path in EXCLUDE_PATH:

                    ex_path=ex_path.replace('\\','')

                    temp_file=file.replace('\\','')

                    if ex_path in temp_file:

                        continue



                    use_encoding=self.default_coding

                    encode_proper,match_dict,line_list=self.search(file,use_encoding)



                    if not encode_proper:

                        use_encoding = self.exception_handle_coding

                        encode_proper,match_dict,line_list=self.search(file, use_encoding)



                    if match_dict is not None and len(match_dict)>0 and all(match_dict.values()):

                        # print(match_dict.values())

                        self.print_match_result(file,line_list,use_encoding)

                        # print(line_list)





def test_error_file():

    path=r'C:\git\CodePool\example-code\19-dyn-attr-prop\oscon\schedule2.py'

    with open(path,'r',encoding='utf8') as fp:

        while 1:

            x=fp.readline()

            if not x:

                break

            print(x)



def main(kw):

    app = FileSearcher(kw)

    app.run()



if __name__ == '__main__':

    fire.Fire(main)

def check_coroutine(fun):

    if iscoroutinefunction(fun):

        print('是协程')

    else:

        print('不是协程')

async def visit_web():

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000},

         'ignoreDefaultArgs':True,

         }

    )

    page = await browser.newPage()



    # 可以在launch下配置

    # await page.setViewport({

    #     "width": 1900,

    #     "height": 1020

    # })





    # 先执行下面的JS 再去goto

    await page.evaluate(

        '''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')

    # await page.screenshot({'path': 'test.png', 'fullPage': True})

    # await page.pdf({'path': 'test.pdf'})

    # await asyncio.sleep(5)



    await page.goto(url=URL)



    # 这里的js是异步的写法

    dimensions = await page.evaluate(

        '''

        ()=>{

        return {

        width:document.documentElement.clientWidth,

        height:document.documentElement.clientHeight,

        deviceScaleFactor_:window.devicePixelRatio,

        }

        }

        '''

    )



    result = await page.evaluate(

        '''

        ()=>{

        var title = document.title;

          return {title:title};

          }

          '''



    )





    await browser.close()

check_coroutine(visit_web)

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000},

         # 'enable-automation':False,

         'ignoreDefaultArgs':['--enable-automation'],

         }

    )

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000}

         }

    )

        self.handleSIGINT = options.get('handleSIGINT', True)

        self.handleSIGTERM = options.get('handleSIGTERM', True)

        self.handleSIGHUP = options.get('handleSIGHUP', True)

        self.ignoreHTTPSErrors = options.get('ignoreHTTPSErrors', False)

        self.defaultViewport = options.get('defaultViewport', {'width': 800, 'height': 600})  # noqa: E501

        self.slowMo = options.get('slowMo', 0)

        self.timeout = options.get('timeout', 30000)

        self.autoClose = options.get('autoClose', True)

urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='storage.googleapis.com', port=443): Max retries exceeded with url: /chromium-browser-snapshots/Win_

x64/588429/chrome-win32.zip (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x00000000037D3880>: Failed to establish a new connect

ion: [Errno 11004] getaddrinfo failed'))

res = requests.get(url, stream=True, headers=headers, verify=False)

        with open(dst, "ab") as f:

            dp = Down_progress(file_size, first_size, dst)

            dp.start()

            chunk_size = 1024

            for chunk in res.iter_content(chunk_size = chunk_size):

                if chunk:

                    f.write(chunk)

                    dp.update(chunk_size)

import xlsxwriter   #导入模块

workbook = xlsxwriter.Workbook('new_people.xlsx')     #新建excel表

worksheet = workbook.add_worksheet('sheet1')       #新建sheet（sheet的名称为"sheet1"）

for index,item in df.iterrows():

    date=item['上市日期']

    count=item['申购人数']

    date=date.replace(' 00:00:00','')

    worksheet.write(0,index,date)

    worksheet.write(1,index,count)



workbook.close()

import mmap

import contextlib

from Evtx.Evtx import FileHeader

from Evtx.Views import evtx_file_xml_view

from xml.dom import minidom

from ip_convertor import IP

import re



class WindowsLogger():



    def __init__(self,path):

        self.path = path

        self.formator = 'IP:{:10}\tlocation:{:20}\tUser:{:15}\tProcess:{}'

    

    def read_file(self):

        with open(self.path,'r') as f:

            with contextlib.closing(mmap.mmap(f.fileno(),0,access=mmap.ACCESS_READ)) as buf:

                fh = FileHeader(buf,0)

                return fh

        

        return None



    def parse_log_detail(self,filteID):

        with open(self.path,'r') as f:

            with contextlib.closing(mmap.mmap(f.fileno(),0,access=mmap.ACCESS_READ)) as buf:

                fh = FileHeader(buf,0)

                for xml, record in evtx_file_xml_view(fh):

                    #只输出事件ID为4624的内容

                    # InterestEvent(xml,4624)

                    for IpAddress,ip,targetUsername,ProcessName in self.filter_event(xml,filteID):

                        print(self.formator.format(IpAddress,ip,targetUsername,ProcessName))



    # 过滤掉不需要的事件，输出感兴趣的事件

    def filter_event(self,xml,EventID,use_filter=True):

        xmldoc = minidom.parseString(xml)

        # 获取EventID节点的事件ID

        collections = xmldoc.documentElement

        events=xmldoc.getElementsByTagName('Event') 

        for evt in events: 

            eventId = evt.getElementsByTagName('EventID')[0].childNodes[0].data

            time_create = evt.getElementsByTagName('TimeCreated')[0].getAttribute('SystemTime')

            eventData = evt.getElementsByTagName('EventData')[0]

            

            for data in eventData.getElementsByTagName('Data'):

                if data.getAttribute('Name')=='IpAddress':

                    IpAddress=data.childNodes[0].data



                if data.getAttribute('Name')=='TargetUserName':

                    targetUsername = data.childNodes[0].data



                if data.getAttribute('Name')=='ProcessName':

                    ProcessName = data.childNodes[0].data

            

            if use_filter is True and eventId==EventID:

                ip=''

                if re.search('^\d+',IpAddress):

                    ip = IP(IpAddress).ip_address



                yield IpAddress,ip,targetUsername,ProcessName



def main():

    path=r'D:\share\1.evtx'

    filter_id = '4624'

    app = WindowsLogger(path)

    app.parse_log_detail(filter_id)



if __name__ == '__main__':

    main()

感觉苏宁的抢购是耍猴的，那个按钮基本处于不可点状态，所以就放弃了，感觉官方就是没放多少量，加上苏宁公司过往的尿性，所以洗洗睡了 

import sys



from maotai.jd_spider_requests import ProdectPurchase





if __name__ == '__main__':

    tip = """                                            

功能列表：                                                                                

 1.预约商品

 2.秒杀抢购商品

    """

    print(tip)



    product = ProdectPurchase()

    choice_function = input('请选择:')

    if choice_function == '1':

        product.reserve()

    elif choice_function == '2':

        product.seckill_by_proc_pool()

    else:

        print('没有此功能')

        sys.exit(1)

import random

import time

import requests

import functools

import json

import os

import pickle



from lxml import etree



from error.exception import SKException

from maotai.jd_logger import logger

from maotai.timer import Timer

from maotai.config import global_config

from concurrent.futures import ProcessPoolExecutor

from helper.jd_helper import (

    parse_json,

    send_wechat,

    wait_some_time,

    response_status,

    save_image,

    open_image

)





class SpiderSession:

    """

    Session相关操作

    """



    def __init__(self):

        self.cookies_dir_path = "./cookies/"

        self.user_agent = global_config.getRaw('config', 'DEFAULT_USER_AGENT')



        self.session = self._init_session()



    def _init_session(self):

        session = requests.session()

        session.headers = self.get_headers()

        return session



    def get_headers(self):

        return {"User-Agent": self.user_agent,

                "Accept": "text/html,application/xhtml+xml,application/xml;"

                          "q=0.9,image/webp,image/apng,*/*;"

                          "q=0.8,application/signed-exchange;"

                          "v=b3",

                "Connection": "keep-alive"}



    def get_user_agent(self):

        return self.user_agent



    def get_session(self):

        """

        获取当前Session

        :return:

        """

        return self.session



    def get_cookies(self):

        """

        获取当前Cookies

        :return:

        """

        return self.get_session().cookies



    def set_cookies(self, cookies):

        self.session.cookies.update(cookies)



    def load_cookies_from_local(self):

        """

        从本地加载Cookie

        :return:

        """

        cookies_file = ''

        if not os.path.exists(self.cookies_dir_path):

            return False

        for name in os.listdir(self.cookies_dir_path):

            if name.endswith(".cookies"):

                cookies_file = '{}{}'.format(self.cookies_dir_path, name)

                break

        if cookies_file == '':

            return False

        with open(cookies_file, 'rb') as f:

            local_cookies = pickle.load(f)

        self.set_cookies(local_cookies)



    def save_cookies_to_local(self, cookie_file_name):

        """

        保存Cookie到本地

        :param cookie_file_name: 存放Cookie的文件名称

        :return:

        """

        cookies_file = '{}{}.cookies'.format(self.cookies_dir_path, cookie_file_name)

        directory = os.path.dirname(cookies_file)

        if not os.path.exists(directory):

            os.makedirs(directory)

        with open(cookies_file, 'wb') as f:

            pickle.dump(self.get_cookies(), f)





class QrLogin:

    """

    扫码登录

    """



    def __init__(self, spider_session: SpiderSession):

        """

        初始化扫码登录

        大致流程：

            1、访问登录二维码页面，获取Token

            2、使用Token获取票据

            3、校验票据

        :param spider_session:

        """

        self.qrcode_img_file = 'qr_code.png'



        self.spider_session = spider_session

        self.session = self.spider_session.get_session()



        self.is_login = False

        self.refresh_login_status()



    def refresh_login_status(self):

        """

        刷新是否登录状态

        :return:

        """

        self.is_login = self._validate_cookies()



    def _validate_cookies(self):

        """

        验证cookies是否有效（是否登陆）

        通过访问用户订单列表页进行判断：若未登录，将会重定向到登陆页面。

        :return: cookies是否有效 True/False

        """

        url = 'https://order.jd.com/center/list.action'

        payload = {

            'rid': str(int(time.time() * 1000)),

        }

        try:

            resp = self.session.get(url=url, params=payload, allow_redirects=False)

            if resp.status_code == requests.codes.OK:

                return True

        except Exception as e:

            logger.error("验证cookies是否有效发生异常", e)

        return False



    def _get_login_page(self):

        """

        获取PC端登录页面

        阻塞，更新cookies

        :return:

        """

        url = "https://passport.jd.com/new/login.aspx"

        page = self.session.get(url, headers=self.spider_session.get_headers())

        return page



    def _get_qrcode(self):

        """

        缓存并展示登录二维码

        :return:

        """

        url = 'https://qr.m.jd.com/show'

        payload = {

            'appid': 133,

            'size': 147,

            't': str(int(time.time() * 1000)),

        }

        headers = {

            'User-Agent': self.spider_session.get_user_agent(),

            'Referer': 'https://passport.jd.com/new/login.aspx',

        }

        resp = self.session.get(url=url, headers=headers, params=payload)



        if not response_status(resp):

            logger.info('获取二维码失败')

            return False



        save_image(resp, self.qrcode_img_file)

        logger.info('二维码获取成功，请打开京东APP扫描')

        open_image(self.qrcode_img_file)

        return True



    def _get_qrcode_ticket(self):

        """

        通过 token 获取票据 ticket

        :return:

        """

        url = 'https://qr.m.jd.com/check'

        payload = {

            'appid': '133',

            'callback': 'jQuery{}'.format(random.randint(1000000, 9999999)),

            'token': self.session.cookies.get('wlfstk_smdl'), # 从cookies获取值

            '_': str(int(time.time() * 1000)),

        }

        headers = {

            'User-Agent': self.spider_session.get_user_agent(),

            'Referer': 'https://passport.jd.com/new/login.aspx',

        }

        resp = self.session.get(url=url, headers=headers, params=payload)



        if not response_status(resp):

            logger.error('获取二维码扫描结果异常')

            return False



        resp_json = parse_json(resp.text)

        if resp_json['code'] != 200:

            logger.info('Code: %s, Message: %s', resp_json['code'], resp_json['msg'])

            return None

        else:

            logger.info('已完成手机客户端确认')

            return resp_json['ticket']



    def _validate_qrcode_ticket(self, ticket):

        """

        通过已获取的票据进行校验

        :param ticket: 已获取的票据

        :return:

        """

        url = 'https://passport.jd.com/uc/qrCodeTicketValidation'

        headers = {

            'User-Agent': self.spider_session.get_user_agent(),

            'Referer': 'https://passport.jd.com/uc/login?ltype=logout',

        }



        resp = self.session.get(url=url, headers=headers, params={'t': ticket})

        if not response_status(resp):

            return False



        resp_json = json.loads(resp.text)

        if resp_json['returnCode'] == 0:

            return True

        else:

            logger.info(resp_json)

            return False



    def login_by_qrcode(self):

        """

        二维码登陆

        :return:

        """

        self._get_login_page() # 更新cookies



        # download QR code

        if not self._get_qrcode():

            raise SKException('二维码下载失败')



        # get QR code ticket

        ticket = None

        retry_times = 85

        for _ in range(retry_times):

            # 重试 拿到ticket

            ticket = self._get_qrcode_ticket()

            if ticket:

                break

            time.sleep(2)

        else:

            raise SKException('二维码过期，请重新获取扫描')



        # validate QR code ticket

        if not self._validate_qrcode_ticket(ticket):

            raise SKException('二维码信息校验失败')



        self.refresh_login_status()



        logger.info('二维码登录成功')





class ProdectPurchase(object):

    def __init__(self):

        self.spider_session = SpiderSession()

        self.spider_session.load_cookies_from_local()

        # 共享一个session



        self.qrlogin = QrLogin(self.spider_session)



        # 初始化信息

        self.sku_id = global_config.getRaw('config', 'sku_id')

        self.seckill_num = global_config.getRaw('config', 'seckill_num')

        self.work_count = global_config.getRaw('config','process_num')

        self.seckill_init_info = dict()

        self.seckill_url = dict()

        self.seckill_order_data = dict()

        self.timers = Timer()



        self.session = self.spider_session.get_session()

        self.user_agent = self.spider_session.user_agent

        self.nick_name = None



    def login_by_qrcode(self):

        """

        二维码登陆

        :return:

        """

        if self.qrlogin.is_login:

            logger.info('登录成功')

            return



        self.qrlogin.login_by_qrcode()



        if self.qrlogin.is_login:

            self.nick_name = self.get_username()

            self.spider_session.save_cookies_to_local(self.nick_name)

        else:

            raise SKException("二维码登录失败！")



    def check_login(func):

        """

        用户登陆态校验装饰器。若用户未登陆，则调用扫码登陆

        """



        @functools.wraps(func)

        def new_func(self, *args, **kwargs):

            if not self.qrlogin.is_login:

                logger.info("{0} 需登陆后调用，开始扫码登陆".format(func.__name__))

                self.login_by_qrcode()

            return func(self, *args, **kwargs)



        return new_func



    @check_login

    def reserve(self):

        """

        预约

        """

        self._reserve()



    @check_login

    def seckill(self):

        """

        抢购

        """

        self._seckill()



    @check_login

    def seckill_by_proc_pool(self):

        """

        多进程进行抢购

        work_count：进程数量

        """

        with ProcessPoolExecutor() as pool:

            for i in range(self.work_count):

                pool.submit(self.seckill)



    def _reserve(self):

        """

        预约

        """

        while True:

            try:

                self.make_reserve()

                break

            except Exception as e:

                logger.info('预约发生异常!', e)

            wait_some_time()



    def _seckill(self):

        """

        抢购

        """

        while True:

            try:

                self.request_seckill_url()

                while True:

                    self.request_seckill_checkout_page()

                    self.submit_seckill_order()

            except Exception as e:

                logger.info('抢购发生异常，稍后继续执行！', e)

            wait_some_time()



    def make_reserve(self):

        """商品预约"""

        logger.info('商品名称:{}'.format(self.get_sku_title()))

        url = 'https://yushou.jd.com/youshouinfo.action?'

        payload = {

            'callback': 'fetchJSON',

            'sku': self.sku_id,

            '_': str(int(time.time() * 1000)),

        }

        headers = {

            'User-Agent': self.user_agent,

            'Referer': 'https://item.jd.com/{}.html'.format(self.sku_id),

        }

        resp = self.session.get(url=url, params=payload, headers=headers)

        resp_json = parse_json(resp.text)

        reserve_url = resp_json.get('url')

        # self.timers.start()

        while True:

            try:

                self.session.get(url='https:' + reserve_url)

                logger.info('预约成功，已获得抢购资格 / 您已成功预约过了，无需重复预约')

                if global_config.getRaw('messenger', 'enable') == 'true':

                    success_message = "预约成功，已获得抢购资格 / 您已成功预约过了，无需重复预约"

                    send_wechat(success_message)

                break

            except Exception as e:

                logger.error('预约失败正在重试...')



    def get_username(self):

        """获取用户信息"""

        url = 'https://passport.jd.com/user/petName/getUserInfoForMiniJd.action'

        payload = {

            'callback': 'jQuery{}'.format(random.randint(1000000, 9999999)),

            '_': str(int(time.time() * 1000)),

        }

        headers = {

            'User-Agent': self.user_agent,

            'Referer': 'https://order.jd.com/center/list.action',

        }



        resp = self.session.get(url=url, params=payload, headers=headers)



        try_count = 5

        while not resp.text.startswith("jQuery"):

            try_count = try_count - 1

            if try_count > 0:

                resp = self.session.get(url=url, params=payload, headers=headers)

            else:

                break

            wait_some_time()

        # 响应中包含了许多用户信息，现在在其中返回昵称

        # jQuery2381773({"imgUrl":"//storage.360buyimg.com/i.imageUpload/xxx.jpg","lastLoginTime":"","nickName":"xxx","plusStatus":"0","realName":"xxx","userLevel":x,"userScoreVO":{"accountScore":xx,"activityScore":xx,"consumptionScore":xxxxx,"default":false,"financeScore":xxx,"pin":"xxx","riskScore":x,"totalScore":xxxxx}})

        return parse_json(resp.text).get('nickName')



    def get_sku_title(self):

        """获取商品名称"""

        url = 'https://item.jd.com/{}.html'.format(global_config.getRaw('config', 'sku_id'))

        resp = self.session.get(url).content

        x_data = etree.HTML(resp)

        sku_title = x_data.xpath('/html/head/title/text()')

        return sku_title[0]



    def get_seckill_url(self):

        """获取商品的抢购链接

        点击"抢购"按钮后，会有两次302跳转，最后到达订单结算页面

        这里返回第一次跳转后的页面url，作为商品的抢购链接

        :return: 商品的抢购链接

        """

        url = 'https://itemko.jd.com/itemShowBtn'

        payload = {

            'callback': 'jQuery{}'.format(random.randint(1000000, 9999999)),

            'skuId': self.sku_id,

            'from': 'pc',

            '_': str(int(time.time() * 1000)),

        }

        headers = {

            'User-Agent': self.user_agent,

            'Host': 'itemko.jd.com',

            'Referer': 'https://item.jd.com/{}.html'.format(self.sku_id),

        }

        while True:

            resp = self.session.get(url=url, headers=headers, params=payload)

            resp_json = parse_json(resp.text)

            if resp_json.get('url'):

                # https://divide.jd.com/user_rou ... %3Dpc

                router_url = 'https:' + resp_json.get('url')

                # https://marathon.jd.com/captch ... %3Dpc

                seckill_url = router_url.replace(

                    'divide', 'marathon').replace(

                    'user_routing', 'captcha.html')

                logger.info("抢购链接获取成功: %s", seckill_url)

                return seckill_url

            else:

                logger.info("抢购链接获取失败，稍后自动重试")

                wait_some_time()



    def request_seckill_url(self):

        """访问商品的抢购链接（用于设置cookie等"""

        logger.info('用户:{}'.format(self.get_username()))

        logger.info('商品名称:{}'.format(self.get_sku_title()))

        self.timers.start() # 阻塞

        

        self.seckill_url[self.sku_id] = self.get_seckill_url()

        logger.info('访问商品的抢购连接...')

        headers = {

            'User-Agent': self.user_agent,

            'Host': 'marathon.jd.com',

            'Referer': 'https://item.jd.com/{}.html'.format(self.sku_id),

        }

        self.session.get(

            url=self.seckill_url.get(

                self.sku_id),

            headers=headers,

            allow_redirects=False)



    def request_seckill_checkout_page(self):

        """访问抢购订单结算页面"""

        logger.info('访问抢购订单结算页面...')

        url = 'https://marathon.jd.com/seckill/seckill.action'

        payload = {

            'skuId': self.sku_id,

            'num': self.seckill_num,

            'rid': int(time.time())

        }

        headers = {

            'User-Agent': self.user_agent,

            'Host': 'marathon.jd.com',

            'Referer': 'https://item.jd.com/{}.html'.format(self.sku_id),

        }

        self.session.get(url=url, params=payload, headers=headers, allow_redirects=False)



    def _get_seckill_init_info(self):

        """获取秒杀初始化信息（包括：地址，发票，token）

        :return: 初始化信息组成的dict

        """

        logger.info('获取秒杀初始化信息...')

        url = 'https://marathon.jd.com/seckillnew/orderService/pc/init.action'

        data = {

            'sku': self.sku_id,

            'num': self.seckill_num,

            'isModifyAddress': 'false',

        }

        headers = {

            'User-Agent': self.user_agent,

            'Host': 'marathon.jd.com',

        }

        resp = self.session.post(url=url, data=data, headers=headers)



        resp_json = None

        try:

            resp_json = parse_json(resp.text)

        except Exception:

            raise SKException('抢购失败，返回信息:{}'.format(resp.text[0: 128]))



        return resp_json



    def _get_seckill_order_data(self):

        """生成提交抢购订单所需的请求体参数

        :return: 请求体参数组成的dict

        """

        logger.info('生成提交抢购订单所需参数...')

        # 获取用户秒杀初始化信息

        self.seckill_init_info[self.sku_id] = self._get_seckill_init_info()

        init_info = self.seckill_init_info.get(self.sku_id)

        default_address = init_info['addressList'][0]  # 默认地址dict

        invoice_info = init_info.get('invoiceInfo', {})  # 默认发票信息dict, 有可能不返回

        token = init_info['token']

        data = {

            'skuId': self.sku_id,

            'num': self.seckill_num,

            'addressId': default_address['id'],

            'yuShou': 'true',

            'isModifyAddress': 'false',

            'name': default_address['name'],

            'provinceId': default_address['provinceId'],

            'cityId': default_address['cityId'],

            'countyId': default_address['countyId'],

            'townId': default_address['townId'],

            'addressDetail': default_address['addressDetail'],

            'mobile': default_address['mobile'],

            'mobileKey': default_address['mobileKey'],

            'email': default_address.get('email', ''),

            'postCode': '',

            'invoiceTitle': invoice_info.get('invoiceTitle', -1),

            'invoiceCompanyName': '',

            'invoiceContent': invoice_info.get('invoiceContentType', 1),

            'invoiceTaxpayerNO': '',

            'invoiceEmail': '',

            'invoicePhone': invoice_info.get('invoicePhone', ''),

            'invoicePhoneKey': invoice_info.get('invoicePhoneKey', ''),

            'invoice': 'true' if invoice_info else 'false',

            'password': global_config.get('account', 'payment_pwd'),

            'codTimeType': 3,

            'paymentType': 4,

            'areaCode': '',

            'overseas': 0,

            'phone': '',

            'eid': global_config.getRaw('config', 'eid'),

            'fp': global_config.getRaw('config', 'fp'),

            'token': token,

            'pru': ''

        }



        return data



    def submit_seckill_order(self):

        """提交抢购（秒杀）订单

        :return: 抢购结果 True/False

        """

        url = 'https://marathon.jd.com/seckillnew/orderService/pc/submitOrder.action'

        payload = {

            'skuId': self.sku_id,

        }

        try:

            self.seckill_order_data[self.sku_id] = self._get_seckill_order_data()

        except Exception as e:

            logger.info('抢购失败，无法获取生成订单的基本信息，接口返回:【{}】'.format(str(e)))

            return False



        logger.info('提交抢购订单...')

        headers = {

            'User-Agent': self.user_agent,

            'Host': 'marathon.jd.com',

            'Referer': 'https://marathon.jd.com/seckill/seckill.action?skuId={0}&num={1}&rid={2}'.format(

                self.sku_id, self.seckill_num, int(time.time())),

        }

        resp = self.session.post(

            url=url,

            params=payload,

            data=self.seckill_order_data.get(

                self.sku_id),

            headers=headers)

        resp_json = None

        try:

            resp_json = parse_json(resp.text)

        except Exception as e:

            logger.info('抢购失败，返回信息:{}'.format(resp.text[0: 128]))

            return False

        # 返回信息

        # 抢购失败：

        # {'errorMessage': '很遗憾没有抢到，再接再厉哦。', 'orderId': 0, 'resultCode': 60074, 'skuId': 0, 'success': False}

        # {'errorMessage': '抱歉，您提交过快，请稍后再提交订单！', 'orderId': 0, 'resultCode': 60017, 'skuId': 0, 'success': False}

        # {'errorMessage': '系统正在开小差，请重试~~', 'orderId': 0, 'resultCode': 90013, 'skuId': 0, 'success': False}

        # 抢购成功：

        # {"appUrl":"xxxxx","orderId":820227xxxxx,"pcUrl":"xxxxx","resultCode":0,"skuId":0,"success":true,"totalMoney":"xxxxx"}

        if resp_json.get('success'):

            order_id = resp_json.get('orderId')

            total_money = resp_json.get('totalMoney')

            pay_url = 'https:' + resp_json.get('pcUrl')

            logger.info('抢购成功，订单号:{}, 总价:{}, 电脑端付款链接:{}'.format(order_id, total_money, pay_url))

            if global_config.getRaw('messenger', 'enable') == 'true':

                success_message = "抢购成功，订单号:{}, 总价:{}, 电脑端付款链接:{}".format(order_id, total_money, pay_url)

                send_wechat(success_message)

            return True

        else:

            logger.info('抢购失败，返回信息:{}'.format(resp_json))

            if global_config.getRaw('messenger', 'enable') == 'true':

                error_message = '抢购失败，返回信息:{}'.format(resp_json)

                send_wechat(error_message)

            return False

print ('hello')

hello

def sayhi():

  ...:     print('Done')

  ...:     

sayhi () # 这里有一个空格

Done

import keyword

keyword.kwlist

Out[3]: 

['False',

 'None',

 'True',

 'and',

 'as',

 'assert',

 'async',

 'await',

 'break',

 'class',

 'continue',

 'def',

 'del',

 'elif',

 'else',

 'except',

 'finally',

 'for',

 'from',

 'global',

 'if',

 'import',

 'in',

 'is',

 'lambda',

 'nonlocal',

 'not',

 'or',

 'pass',

 'raise',

 'return',

 'try',

 'while',

 'with',

 'yield']

len(keyword.kwlist)

Out[4]: 35

token = '123456789'

def Services(request):

    print(request.method)

    if request.method=='GET':



        signature = request.GET.get('signature')

        echostr = request.GET.get('echostr')

        timestamp = request.GET.get('timestamp')

        nonce = request.GET.get('nonce')

        list_ = [token, timestamp, nonce]

        list_.sort()

        list_str = ''.join(list_)



        sha1 = hashlib.sha1(list_str.encode('utf8'))

        hashcode = sha1.hexdigest()

        if hashcode==signature:

            return HttpResponse(echostr)

        else:

            return HttpResponse('')

http:\u002F\u002F1256122120.vod2.myqcloud.com\u002F53ad1740vodtranscq1256122120\u002F17ebe5145285890810945319765\u002Fv.f20.mp4

#!/usr/bin/env python

# http://30daydo.com

import os

import re

import js2py

import requests

from lxml import etree

from clint.textui import progress

import fire

from loguru import logger



file='crawler'

logger.add(

    "logs/%s.log" % file,

    format="{time:MM-DD HH:mm:ss} {level} {message}",

)



headers = {

    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",

}

proxies = {}



# 如果代理不稳定，不推荐使用

# local proxy service

# proxies example:

# proxies = {

#     "http": "socks5://127.0.0.1:1080",

#     "https": "socks5://127.0.0.1:1080",

# }



def list_page(url):

    logger.info("crawling : %s" % url)

    resp = requests.get(url, headers=headers, proxies=proxies, verify=False)

    html = etree.HTML(resp.text)



    buff = '//*[@class="phimage"]/a/'

    names = html.xpath(f"{buff}@href")

    urls = html.xpath(f"{buff}img/@data-mediabook")

    for i in range(len(urls)):

        try:

            url = urls

[i]            name = re.findall("=ph(\w+)", names[i])[-1]

            logger.info(f"{url} {name}")

            download(url, name, "webm")

        except Exception as err:

            logger.error(err)





def detail_page(url):

    s = requests.Session()

    resp = s.get(url, headers=headers, proxies=proxies, verify=False)

    html = etree.HTML(resp.content)



    title = "".join(html.xpath("//h1//text()")).strip()

    logger.info(title)



    js_temp = html.xpath("//script/text()")

    for j in js_temp:

        if "flashvars" in j:

            videoUrl = exeJs(j)



            download(videoUrl, title, "mp4")

            continue





def exeJs(js):

    flashvars = re.findall("flashvars_\d+", js)[0]

    js = "\n\t".join(js.split("\n\t")[:-5]).strip()



    js = js.replace("// var nextVideoObject = flashvars_['nextVideo'];",'')

    js+=flashvars

    res = js2py.eval_js(js)



    if res.quality_720p:

        return res.quality_720p

    elif res.quality_480p:

        return res.quality_480p

    elif res.quality_240p:

        return res.quality_240p

    else:

        logger.error("parse url error")





def download(url, name, filetype):

    logger.info(f"{url} {name} {filetype}")

    filepath = "%s/%s.%s" % (filetype, name, filetype)

    if os.path.exists(filepath):

        logger.info("this file had been downloaded :: %s" % filepath)

        return

    else:

        response = requests.get(url, headers=headers, proxies=proxies, stream=True)

        with open(filepath, "wb") as file:

            total_length = int(response.headers.get("content-length"))

            for ch in progress.bar(

                response.iter_content(chunk_size=2391975),

                expected_size=(total_length / 1024) + 1,

            ):

                if ch:

                    file.write(ch)



        logger.info("download success :: %s" % filepath)





def run(_arg=None):

    paths = ["webm", "mp4"]

    for path in paths:

        if not os.path.exists(path):

            os.mkdir(path)

    if _arg == "webm":

        # https://www.pornhub.com/categories

        urls = [

            # "https://www.pornhub.com/video?o=tr",

            # "https://www.pornhub.com/video?o=ht",

            # "https://www.pornhub.com/video?o=mv",

            "https://www.pornhub.com/video",

        ]

        for url in urls:

            list_page(url)

    elif _arg == "mp4":

        with open("download.txt", "r") as file:

            keys = list(set(file.readlines()))

        logger.info(keys)

        keys += [d.strip(".webm") for d in os.listdir("webm/")]

        for key in keys:

            if not key.strip():

                continue

            url = "https://www.pornhub.com/view_v ... ot%3B % key.strip()

            logger.info("url: {}", url)

            detail_page(url)

    else:

        _str = """

tips:

    python crawler.py webm

        - 下载热门页面的缩略图，路径为webm文件夹下



    python crawler.py mp4

        - 该命令会下载webm文件下对应的mp4文件

        - 也可以将目标地址写入download.txt中

        """

        logger.info(_str)

        return

    logger.info("finish !")





if __name__ == "__main__":

    fire.Fire(run)

[/i][/i]

[i]

def main():

    for i in range(100):

        print(i*i)





if __name__ == '__main__':

    main()

var radra27radra27 = "D";

var ra72419ra91ra72419ra91 = "7.241.9" + ".";

var raurst500ra63raurst500ra63 = "urst=500";

var ravalidtora49ravalidtora49 = "validto=";

var raevphncdra57raevphncdra57 = "ev.ph" + "ncd";

var ra16067161ra17ra16067161ra17 = "16067161";

var radeos202ra16radeos202ra16 = "deos/20" + "2";

var ra1080p4ra73ra1080p4ra73 = "/1080P_4";

var ra209hashra72ra209hashra72 = "209&hash";

var ra2bmkdz7nra36ra2bmkdz7nra36 = "2BMKd" + "z7N";

var ra6708909ra29ra6708909ra29 = "6708909" + "&";

var ra00kip4ra41ra00kip4ra41 = "00k&i" + "p=4";

var ra006163ra73ra006163ra73 = "006/16/" + "3";

var raro7upu3ra66raro7upu3ra66 = "Ro7UPU%3";

var raroiu6qra26raroiu6qra26 = "=rOiU6q%";

var ra075351mra26ra075351mra26 = "075351." + "m";

var ramgdmctbvra11ramgdmctbvra11 = "MgdmCtbV";

var rap4validra25rap4validra25 = "p4?valid";

var ra09ratera79ra09ratera79 = "09&rate=";

var rancomvira35rancomvira35 = "n.com/vi";

var ra24075351ra94ra24075351ra94 = "24075" + "351";

var ra000k324ra70ra000k324ra70 = "000K_324";

var ra50000kbra49ra50000kbra49 = "50000k&" + "b";

var rahttpsra83rahttpsra83 = "https://";

var rafrom160ra56rafrom160ra56 = "from=16" + "0";

var quality_1080p =/* + radra27radra27 + */rahttpsra83rahttpsra83 + /* + rancomvira35rancomvira35 + */raevphncdra57raevphncdra57 + /* + radra27radra27 + */rancomvira35rancomvira35 + /* + ra006163ra73ra006163ra73 + */radeos202ra16radeos202ra16 + /* + ra09ratera79ra09ratera79 + */ra006163ra73ra006163ra73 + /* + ra1080p4ra73ra1080p4ra73 + */ra24075351ra94ra24075351ra94 + /* + raroiu6qra26raroiu6qra26 + */ra1080p4ra73ra1080p4ra73 + /* + ra000k324ra70ra000k324ra70 + */ra000k324ra70ra000k324ra70 + /* + rancomvira35rancomvira35 + */ra075351mra26ra075351mra26 + /* + ravalidtora49ravalidtora49 + */rap4validra25rap4validra25 + /* + ra209hashra72ra209hashra72 + */rafrom160ra56rafrom160ra56 + /* + ra1080p4ra73ra1080p4ra73 + */ra6708909ra29ra6708909ra29 + /* + ra209hashra72ra209hashra72 + */ravalidtora49ravalidtora49 + /* + ramgdmctbvra11ramgdmctbvra11 + */ra16067161ra17ra16067161ra17 + /* + ra24075351ra94ra24075351ra94 + */ra09ratera79ra09ratera79 + /* + ra50000kbra49ra50000kbra49 + */ra50000kbra49ra50000kbra49 + /* + ramgdmctbvra11ramgdmctbvra11 + */raurst500ra63raurst500ra63 + /* + ra209hashra72ra209hashra72 + */ra00kip4ra41ra00kip4ra41 + /* + raroiu6qra26raroiu6qra26 + */ra72419ra91ra72419ra91 + /* + ra09ratera79ra09ratera79 + */ra209hashra72ra209hashra72 + /* + raro7upu3ra66raro7upu3ra66 + */raroiu6qra26raroiu6qra26 + /* + ra075351mra26ra075351mra26 + */ra2bmkdz7nra36ra2bmkdz7nra36 + /* + ra50000kbra49ra50000kbra49 + */ramgdmctbvra11ramgdmctbvra11 + /* + radeos202ra16radeos202ra16 + */raro7upu3ra66raro7upu3ra66 + /* + ra075351mra26ra075351mra26 + */radra27radra27;

flashvars_324075351["quality_1080p"] = quality_1080p;

var radra27radra27 = "D";

var ra72419ra91ra72419ra91 = "7.241.9" + ".";

var raurst500ra63raurst500ra63 = "urst=500";

var ravalidtora49ravalidtora49 = "validto=";

var raevphncdra57raevphncdra57 = "ev.ph" + "ncd";

var ra16067161ra17ra16067161ra17 = "16067161";

var radeos202ra16radeos202ra16 = "deos/20" + "2";

var ra1080p4ra73ra1080p4ra73 = "/1080P_4";

var ra209hashra72ra209hashra72 = "209&hash";

var ra2bmkdz7nra36ra2bmkdz7nra36 = "2BMKd" + "z7N";

var ra6708909ra29ra6708909ra29 = "6708909" + "&";

var ra00kip4ra41ra00kip4ra41 = "00k&i" + "p=4";

var ra006163ra73ra006163ra73 = "006/16/" + "3";

var raro7upu3ra66raro7upu3ra66 = "Ro7UPU%3";

var raroiu6qra26raroiu6qra26 = "=rOiU6q%";

var ra075351mra26ra075351mra26 = "075351." + "m";

var ramgdmctbvra11ramgdmctbvra11 = "MgdmCtbV";

var rap4validra25rap4validra25 = "p4?valid";

var ra09ratera79ra09ratera79 = "09&rate=";

var rancomvira35rancomvira35 = "n.com/vi";

var ra24075351ra94ra24075351ra94 = "24075" + "351";

var ra000k324ra70ra000k324ra70 = "000K_324";

var ra50000kbra49ra50000kbra49 = "50000k&" + "b";

var rahttpsra83rahttpsra83 = "https://";

var rafrom160ra56rafrom160ra56 = "from=16" + "0";

var quality_1080p =/* + radra27radra27 + */rahttpsra83rahttpsra83 + /* + rancomvira35rancomvira35 + */raevphncdra57raevphncdra57 + /* + radra27radra27 + */rancomvira35rancomvira35 + /* + ra006163ra73ra006163ra73 + */radeos202ra16radeos202ra16 + /* + ra09ratera79ra09ratera79 + */ra006163ra73ra006163ra73 + /* + ra1080p4ra73ra1080p4ra73 + */ra24075351ra94ra24075351ra94 + /* + raroiu6qra26raroiu6qra26 + */ra1080p4ra73ra1080p4ra73 + /* + ra000k324ra70ra000k324ra70 + */ra000k324ra70ra000k324ra70 + /* + rancomvira35rancomvira35 + */ra075351mra26ra075351mra26 + /* + ravalidtora49ravalidtora49 + */rap4validra25rap4validra25 + /* + ra209hashra72ra209hashra72 + */rafrom160ra56rafrom160ra56 + /* + ra1080p4ra73ra1080p4ra73 + */ra6708909ra29ra6708909ra29 + /* + ra209hashra72ra209hashra72 + */ravalidtora49ravalidtora49 + /* + ramgdmctbvra11ramgdmctbvra11 + */ra16067161ra17ra16067161ra17 + /* + ra24075351ra94ra24075351ra94 + */ra09ratera79ra09ratera79 + /* + ra50000kbra49ra50000kbra49 + */ra50000kbra49ra50000kbra49 + /* + ramgdmctbvra11ramgdmctbvra11 + */raurst500ra63raurst500ra63 + /* + ra209hashra72ra209hashra72 + */ra00kip4ra41ra00kip4ra41 + /* + raroiu6qra26raroiu6qra26 + */ra72419ra91ra72419ra91 + /* + ra09ratera79ra09ratera79 + */ra209hashra72ra209hashra72 + /* + raro7upu3ra66raro7upu3ra66 + */raroiu6qra26raroiu6qra26 + /* + ra075351mra26ra075351mra26 + */ra2bmkdz7nra36ra2bmkdz7nra36 + /* + ra50000kbra49ra50000kbra49 + */ramgdmctbvra11ramgdmctbvra11 + /* + radeos202ra16radeos202ra16 + */raro7upu3ra66raro7upu3ra66 + /* + ra075351mra26ra075351mra26 + */radra27radra27;

flashvars_324075351["quality_1080p"] = quality_1080p;

....

ra50000kbra49ra50000kbra49 + */ramgdmctbvra11ramgdmctbvra11 + /* + radeos202ra16radeos202ra16 + */raro7upu3ra66raro7upu3ra66 + /* + ra075351mra26ra075351mra26 + */radra27radra27;

flashvars_324075351["quality_1080p"] = quality_1080p;

flashvars_324075351;