import sys

sys.path.append('..')

import asyncio

import datetime

import aiohttp

import re

import time

from parsel import Selector

from configure.settings import DBSelector

from common.BaseService import BaseService



SLEEP = 2



headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}



URL_MAP = {'home_page': 'https://holdle.com/stocks/industry', 'base': 'https://holdle.com'}





class AsyncMongo():

    def __init__(self):

        self.DB = DBSelector()

        self.client = self.DB.mongo(location_type='qq', async_type=True)

        self.db = self.client['db_stock']



    async def update(self, table,data):

        self.doc= self.db[table]

        await self.doc.insert_many(data)





class Holdle(BaseService):



    def __init__(self):

        super(Holdle, self).__init__()

        self.data_processor = AsyncMongo()

        self.tables_list =['ROE','Cash_Ratio','Gross_Margin','Operation_Margin','Net_Profit_Ratio','Dividend_ratio']



    async def home_page(self):

        start = time.time()

        async with aiohttp.ClientSession() as session:

            async with session.get(url=URL_MAP['home_page'], headers=headers) as response:

                html = await response.text()  # 这个阻塞

                resp = Selector(text=html)

                industries = resp.xpath('//ul[@class="list-unstyled"]/a')

                task_list = []

                for industry in industries:

                    json_data = {}

                    industry_url = industry.xpath('.//@href').extract_first()

                    industry_name = industry.xpath('.//li/text()').extract_first()

                    industry_name = industry_name.replace('-', '').strip()

                    json_data['industry_url'] = industry_url

                    json_data['industry_name'] = industry_name



                    task = asyncio.ensure_future(self.detail_list(session, industry_url, json_data))

                    task_list.append(task)



                await asyncio.gather(*task_list)

                end = time.time()



                print(f'time used {end - start}')



    async def detail_list(self, session, url, json_data):



        async with session.get(URL_MAP['base'] + url, headers=headers) as response:

            response = await response.text()

            await self.parse_detail(response, json_data)



    async def parse_detail(self, html, json_data=None):

            resp = Selector(text=html)

            industry=json_data['industry_name']

            tables = resp.xpath('//table[@class="table table-bordered"]')

            if len(tables)!=6:

                raise ValueError



            for index,table in enumerate(self.tables_list):

                rows = tables[index].xpath('.//tr')

                result = []

                for row in rows[1:]:

                    stock_name = row.xpath('.//td[1]/text()').extract_first()

                    value = row.xpath('.//td[2]/text()').extract_first()

                    value = float(value)

                    d={'industry':industry,'name':stock_name,'value':value,'crawltime':datetime.datetime.now()}

                    result.append(d)

                await self.data_processor.update(table,result)





app = Holdle()

loop = asyncio.get_event_loop()

loop.run_until_complete(app.home_page())

# -*- coding: utf-8 -*-

# @Time : 2020/11/24 21:42

# @File : sync_spider.py

# @Author : Rocky C@www.30daydo.com

import requests

import sys

sys.path.append('..')

import asyncio

import datetime

import aiohttp

import re

import time

from parsel import Selector

from configure.settings import DBSelector

from common.BaseService import BaseService



SLEEP = 2



headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}



URL_MAP = {'home_page': 'https://holdle.com/stocks/industry', 'base': 'https://holdle.com'}





class Holdle(BaseService):



    def __init__(self):

        super(Holdle, self).__init__()



        self.DB = DBSelector()

        self.client = self.DB.mongo(location_type='qq', async_type=True)

        self.session = requests.Session()



    def run(self):

        start = time.time()



        response = self.session.get(url=URL_MAP['home_page'], headers=headers)

        html =  response.text  # 这个阻塞

        resp = Selector(text=html)

        industries = resp.xpath('//ul[@class="list-unstyled"]/a')

        for industry in industries:

            json_data = {}

            industry_url = industry.xpath('.//@href').extract_first()

            industry_name = industry.xpath('.//li/text()').extract_first()

            json_data['industry_url'] = industry_url

            json_data['industry_name'] = industry_name

            self.detail_list(industry_url, json_data)



        end = time.time()

        print(f'time used {end-start}')



    def detail_list(self, url, json_data):



        response = self.session.get(URL_MAP['base']+url, headers=headers)

        response =response.text

        self.parse_detail(response, json_data)



    def parse_detail(self, html, json_data=None):

        resp = Selector(text=html)

        title =resp.xpath('//title/text()').extract_first()

        print(title)





app = Holdle()

app.run()

import aiohttp

import asyncio

import execjs

import threading

global pages

global count



headers = {

    "Accept": "*/*",

    "Accept-Encoding": "gzip, deflate",

    "Accept-Language": "en-US,en;q=0.9",

    "Cache-Control": "no-cache",

    "Connection": "keep-alive",

    "Host": "dcfm.eastmoney.com",

    "Pragma": "no-cache",

    "Referer": "http://data.eastmoney.com/xg/xg/default.html",

    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/69.0.3497.81 Chrome/69.0.3497.81 Safari/537.36",

}



home_url = 'http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?type=XGSG_LB&token=70f12f2f4f091e459a279469fe49eca5&st=purchasedate,securitycode&sr=-1&p={}&ps=50&js=var%20hsEnHLwG={{pages:(tp),data:(x)}}&rt=53512217'



loop = asyncio.get_event_loop()

# lock = threading.Lock()

lock = asyncio.Lock()

def parse_json(content):

    content += ';function getV(){return hsEnHLwG;}'

    ctx = execjs.compile(content)

    result = ctx.call('getV')

    return result





async def fetch(session,page):

    global pages

    global count

    async with session.get(home_url.format(page),headers=headers) as resp:

        # print(f'here：： {page}')

        content = await resp.text()



        try:

            js_content = parse_json(content)

            for stock_info in js_content['data']:

                securityshortname = stock_info['securityshortname']

                # print(securityshortname)

        except Exception as e:

            print(e)



        async with lock:

            count=count+1



        print(f'count:{count}')

        if count == pages:

            print('End of loop')

            loop.stop()







async def main():

    global pages

    global count

    count=0

    async with aiohttp.ClientSession() as session:

        async with session.get(home_url.format(1), headers=headers) as resp:



            content = await resp.text()

            js_data = parse_json(content)

            pages = js_data['pages']

            print(f'pages: {pages}')

            for page in range(1,pages+1):

                task = asyncio.ensure_future(fetch(session,page))



            await asyncio.sleep(1)





asyncio.ensure_future(main())

loop.run_forever()

import os

from pyecharts.render import make_snapshot

from snapshot_selenium import snapshot

import pandas as pd

from pyecharts import options as opts

from pyecharts.charts import Bar

import sys

from selenium import webdriver

from pyecharts.commons.utils import JsCode



if sys.platform == 'win32':

    SELENIUM_PATH = r'C:\OneDrive\Tool\phantomjs-2.1.1-windows\phantomjs-2.1.1-windows\bin\phantomjs.exe'

    driver = None

else:

    SELENIUM_PATH = './phantomjs'

    driver = webdriver.PhantomJS(executable_path=SELENIUM_PATH)

    

    

    bar = (

        Bar()

            .add_xaxis(list(result_dict .keys()))

            .add_yaxis(f"{today}-可转债价格分布", y_list, category_gap=3)

            .add_yaxis(f"{today}-正股价格分布", y_zg_list, category_gap=3)

            .set_series_opts(

            label_opts=opts.LabelOpts(is_show=True),

            axispointer_opts=opts.AxisPointerOpts(is_show=True))

            .set_global_opts(

            title_opts=opts.TitleOpts(title="可转债价格分布"),

            xaxis_opts=opts.AxisOpts(

                name="涨跌幅",

                is_show=True,

                name_rotate=30,

            ),

            graphic_opts=[

                opts.GraphicGroup(

                    graphic_item=opts.GraphicItem(

                        left="70%",

                        top="20%",

                    ),

                    children=[

                        opts.GraphicText(

                            graphic_item=opts.GraphicItem(

                                left="center",

                                top="middle",

                                z=100,

                            ),

                            graphic_textstyle_opts=opts.GraphicTextStyleOpts(

                                text=JsCode(

                                    f"['涨幅>=0：{bigger}',"

                                    f"'涨幅<0：{smaller}',"

                                    f"'平均涨幅：{avg}%',"

                                    f"'波动方差：{std}',"

                                    f"'',"

                                    f"'最大：{max_name}  {max_pct}%',"

                                    f"'最小：{min_name}  {min_pct}%',"

                                    "''].join('\\n')"

                                ),

                                font="14px Microsoft YaHei",

                                graphic_basicstyle_opts=opts.GraphicBasicStyleOpts(

                                    fill="#333"

                                )

                            )

                        )

                    ]

                )

            ],

        )



    )



    bar.render(os.path.join('data', f"{today}_cb.html"))

    make_snapshot(snapshot, bar.render(), f"data/{today}_cb.png", driver=driver)

server = SSHTunnelForwarder(

    ssh_address_or_host=host,

    ssh_port=port,

    ssh_username=user,

    ssh_password=password,

    local_bind_address=('127.0.0.1', local_port),

    remote_bind_address=(host, mysql_port)

)



server.start()

conn = pymysql.connect(

    host='127.0.0.1',

    port=local_port,

    user=user,

    password=password,

    db='db_stock'

)



cursor = conn.cursor()

cursor.execute('select count(*) from tb_cb_index')

ret = cursor.fetchall()

print(ret)

server.stop()

print('stop')

print(ret)

conn.close()

server.stop()

print('stop')

Traceback (most recent call last):

  File "c:\anaconda\envs\py37\lib\runpy.py", line 193, in _run_module_as_main

    "__main__", mod_spec)

  File "c:\anaconda\envs\py37\lib\runpy.py", line 85, in _run_code

    exec(code, run_globals)

  File "C:\anaconda\envs\py37\Scripts\pipreqs.exe\__main__.py", line 7, in <module>

  File "c:\anaconda\envs\py37\lib\site-packages\pipreqs\pipreqs.py", line 470, in main

    init(args)

  File "c:\anaconda\envs\py37\lib\site-packages\pipreqs\pipreqs.py", line 409, in init

    follow_links=follow_links)

  File "c:\anaconda\envs\py37\lib\site-packages\pipreqs\pipreqs.py", line 138, in get_all_imports

    raise exc

  File "c:\anaconda\envs\py37\lib\site-packages\pipreqs\pipreqs.py", line 124, in get_all_imports

    tree = ast.parse(contents)

  File "c:\anaconda\envs\py37\lib\ast.py", line 35, in parse

    return compile(source, filename, mode, PyCF_ONLY_AST)

  File "<unknown>", line 162

    except Exception ,e:

                     ^

SyntaxError: invalid syntax

  1 # -*- coding: utf-8 -*-

  2              

  3 import asyncio

  4               

  5 async def main():

  6     await asyncio.sleep(10)

  7     print('Done')

  8     myloop = asyncio.get_running_loop()

  9     print('current loop ')

 10     print(id(myloop))

 11                                                                             

 12 # loop = asyncio.get_running_loop()

 13 loop = asyncio.get_event_loop()

 14 print('current loop id')

 15 print(id(loop))

 16              

 17 # print(id(myloop))

 18 try:         

 19     loop.run_until_complete(main())

 20 except KeyboardInterrupt:

 21     print('key board inpterrupt!')

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2020/9/22 10:07

# @File : bilibili_hot_anchor.py



# 异步爬取首页与列表



import asyncio

import datetime

import aiohttp

import re

import time

from motor.motor_asyncio import AsyncIOMotorClient

from parsel import Selector

from settings import _json_data



SLEEP = 60 * 10

INFO = _json_data['mongo']['arm']

host = INFO['host']

port = INFO['port']

user = INFO['user']

password = INFO['password']

connect_uri = f'mongodb://{user}:{password}@{host}:{port}'



client = AsyncIOMotorClient(connect_uri)



db = client['db_parker']



home_url = 'https://www.bilibili.com/ranking'

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}





def convertor(number_str):

    '''

    将小数点的万变为整数

    :param number_str:

    :return:

    '''

    number = re.search('(\d+\.+\d+)', number_str)

    if number:

        number = float(number.group(1))

        if re.search('万', number_str):

            number = int(number * 10000)

    else:

        number = 0



    return number





async def home_page():

    async with aiohttp.ClientSession() as session:

        while True:

            start = time.time()

            async with session.get(url=home_url, headers=headers) as response:

                html = await response.text()

                resp = Selector(text=html)

                items = resp.xpath('//ul[@class="rank-list"]/li')

                for item in items:

                    json_data = {}

                    number = item.xpath('.//div[@class="num"]/text()').extract_first()

                    info = item.xpath('.//div[@class="info"][1]')

                    title = info.xpath('.//a/text()').extract_first()



                    detail_url = info.xpath('.//a/@href').extract_first()

                    play_number = info.xpath('.//div[@class="detail"]/span[1]/text()').extract_first()

                    viewing_number = info.xpath('.//div[@class="detail"]/span[2]/text()').extract_first()



                    json_data['number'] = int(number)

                    json_data['title'] = title

                    json_data['play_number'] = convertor(play_number)

                    json_data['viewing_number'] = convertor(viewing_number)



                    json_data['url'] = detail_url



                    task = asyncio.create_task(detail_list(session, detail_url, json_data))

                    # await detail_url()

                end = time.time()

                print(f'time used {end-start}')

                await asyncio.sleep(SLEEP)  # 暂停10分钟

                print(f'sleep for {SLEEP}')





async def detail_list(session, url, json_data):

    async with session.get(url, headers=headers) as response:

        response = await response.text()

        await parse_detail(response, json_data)





async def parse_detail(html, json_data=None):

    resp = Selector(text=html)

    info = resp.xpath('//div[@id="v_desc"]/div[@class="info open"]/text()').extract_first()



    if not info:

        info = '这个家伙很懒'



    json_data['info'] = info.strip()

    current = datetime.datetime.now()

    json_data['crawltime'] = current

    await db['bilibili'].update_one({'url': json_data['url']}, {'$set': json_data}, True, True)





loop = asyncio.get_event_loop()

loop.run_until_complete(home_page())

m = re.match(p1, line)

if m:

return m.group(1)

else:

m = re.match(p2, line)

if m:

return m.group(2)

else:

m = re.match(p3, line)

    ...

if m := re.match(p1, line):

return m.group(1)

elif m := re.match(p2, line):

return m.group(2)

elif m := re.match(p3, line):

  File "C:\ProgramData\Anaconda3\lib\threading.py", line 926, in _bootstrap_inner

    self.run()

  File "C:\ProgramData\Anaconda3\lib\threading.py", line 870, in run

    self._target(*self._args, **self._kwargs)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 1238, in _readerthread

    buffer.append(fh.read())

UnicodeDecodeError: 'gbk' codec can't decode byte 0xbd in position 52: illegal multibyte sequence

  File "C:\ProgramData\Anaconda3\lib\site-packages\execjs\_external_runtime.py", line 103, in _exec_with_pipe

    stdoutdata, stderrdata = p.communicate(input=input)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 939, in communicate

    stdout, stderr = self._communicate(input, endtime, timeout)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 1288, in _communicate

    stdout = stdout[0]

IndexError: list index out of range

    def __init__(self, args, bufsize=-1, executable=None,

                 stdin=None, stdout=None, stderr=None,

                 preexec_fn=None, close_fds=True,

                 shell=False, cwd=None, env=None, universal_newlines=None,

                 startupinfo=None, creationflags=0,

                 restore_signals=True, start_new_session=False,

                 pass_fds=(), *, encoding="None", errors=None, text=None):

# -*- coding: utf-8 -*-

 

"""

PyQt5 tutorial

 

In this example, we create a custom widget.

"""

import sys

from PyQt5.QtWidgets import (QWidget, QSlider, QApplication,

                             QHBoxLayout, QVBoxLayout)

from PyQt5.QtCore import QObject, Qt, pyqtSignal

from PyQt5.QtGui import QPainter, QFont, QColor, QPen

 

 

class Communicate(QObject):

    updateBW = pyqtSignal(int)

 

 

class BurningWidget(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

 

        self.setMinimumSize(1, 30)

        self.value = 75

        self.num = [75, 150, 225, 300, 375, 450, 525, 600, 675]

 

    def setValue(self, value):

 

        self.value = value

 

    def paintEvent(self, e):

 

        qp = QPainter()

        qp.begin(self)

        self.drawWidget(qp)

        qp.end()

 

    def drawWidget(self, qp):

 

        font = QFont('Serif', 7, QFont.Light)

        qp.setFont(font)

 

        size = self.size()

        w = size.width()

        h = size.height()

 

        step = int(round(w / 10.0))

 

        till = int(((w / 750.0) * self.value))

        full = int(((w / 750.0) * 700))

 

        if self.value >= 700:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, full, h)

            qp.setPen(QColor(255, 175, 175))

            qp.setBrush(QColor(255, 175, 175))

            qp.drawRect(full, 0, till - full, h)

 

        else:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, till, h)

 

        pen = QPen(QColor(20, 20, 20), 1,

                   Qt.SolidLine)

 

        qp.setPen(pen)

        qp.setBrush(Qt.NoBrush)

        qp.drawRect(0, 0, w - 1, h - 1)

 

        j = 0

 

        for i in range(step, 10 * step, step):

            qp.drawLine(i, 0, i, 5)

            metrics = qp.fontMetrics()

            fw = metrics.width(str(self.num[j]))

            qp.drawText(i - fw / 2, h / 2, str(self.num[j]))

            j = j + 1

 

 

class Example(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

        sld = QSlider(Qt.Horizontal, self)

        sld.setFocusPolicy(Qt.NoFocus)

        sld.setRange(1, 750)

        sld.setValue(75)

        sld.setGeometry(30, 40, 150, 30)

 

        self.c = Communicate()

        self.wid = BurningWidget()

        self.c.updateBW[int].connect(self.wid.setValue)

 

        sld.valueChanged[int].connect(self.changeValue)

        hbox = QHBoxLayout()

        hbox.addWidget(self.wid)

        vbox = QVBoxLayout()

        vbox.addStretch(1)

        vbox.addLayout(hbox)

        self.setLayout(vbox)

 

        self.setGeometry(300, 300, 390, 210)

        self.setWindowTitle('Burning widget')

        self.show()

 

    def changeValue(self, value):

        self.c.updateBW.emit(value)

        self.wid.repaint()

 

 

if __name__ == '__main__':

    app = QApplication(sys.argv)

    ex = Example()

    sys.exit(app.exec_())

class BurningWidget(QWidget):

  

    def __init__(self):      

        super().__init__() 

self.setMinimumSize(1, 30)

font = QFont('Serif', 7, QFont.Light)

qp.setFont(font)

size = self.size()

w = size.width()

h = size.height()

 

step = int(round(w / 10.0))

 

 

till = int(((w / 750.0) * self.value))

full = int(((w / 750.0) * 700))

metrics = qp.fontMetrics()

fw = metrics.width(str(self.num[j]))

qp.drawText(i-fw/2, h/2, str(self.num[j]))

def changeValue(self, value):

          

    self.c.updateBW.emit(value)        

    self.wid.repaint()

from PyQt5.QtCore import Qt,QRect

import jieba.posseg as pseg

seg_list = pseg.cut("我爱北京天安门")

for word,flag in seg_list:

    print(word)

    print(flag)

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-5-f105f6980f88> in <module>()

      1 import jieba.posseg as pseg

      2 seg_list = pseg.cut("我爱北京天安门")

----> 3 for word,flag in seg_list:

      4     print(word)

      5     print(flag)



TypeError: cannot unpack non-iterable pair object

原因是新版本中seg_list是一个生成器，所以只能 for win seg_list

然后从word中解包出来

print(w.word)

print(w.flag)

这样问题就解决了。