B站视频下载

自动批量下载B站一个系列的视频



下载某个UP主的所有视频



使用：



下载you-get库，git clone https://github.com/soimort/you-get.git 复制其本地路径，比如/root/you-get/you-get



初次运行，删除history.db 文件， 修改配置文件config.py



START=1 # 下载系列视频的 第一个

END=1 # 下载系列视频的最后一个 ， 比如一个系列教程有30个视频， start=5 ,end = 20 下载从第5个到第20个

ID='BV1oK411L7au' # 视频的ID

YOU_GET_PATH='/home/xda/othergit/you-get/you-get' # 你的you-get路径

MINS=1 # 每次循环等待1分钟

user_id = '518973111' # UP主的ID

total_page = 3 # up主的视频的页数

执行 python downloader.py ，进行下载循环



python people.py ，把某个up主的视频链接加入到待下载队列



python add_data.py --id=BV1oK411L7au --start=4 --end=8 下载视频id为BV1oK411L7au的系列教程，从第4开始，到第8个结束，如果只有一个的话，start和end设为1即可。



可以不断地往队列里面添加下载链接。

# @Time : 2019/1/28 14:19

# @File : youtube_downloader.py



import logging

import os

import subprocess

import datetime

import sqlite3

import time

from config import YOU_GET_PATH,MINS

CMD = 'python {} {}'

filename = 'url.txt'





class SQLite():

    def __init__(self):

        self.conn = sqlite3.connect('history.db')

        self.cursor = self.conn.cursor()

        self.create_table()



    def create_table(self):

        create_sql = 'create table if not exists tb_download (url varchar(100),status tinyint,crawltime datetime)'

        create_record_tb = 'create table if not exists tb_record (idx varchar(100) PRIMARY KEY,start tinyint,end tinyint,status tinyint)'

        self.cursor.execute(create_record_tb)

        self.conn.commit()

        self.cursor.execute(create_sql)

        self.conn.commit()



    def exists(self,url):

        querySet = 'select * from tb_download where url = ? and status = 1'

        self.cursor.execute(querySet,(url,))

        ret = self.cursor.fetchone()

        return True if ret else False



    def insert_history(self,url,status):

        query = 'select * from tb_download where url=?'

        self.cursor.execute(query,(url,))

        ret = self.cursor.fetchone()

        current = datetime.datetime.now()



        if ret:

            insert_sql='update tb_download set status=?,crawltime=? where url = ?'

            args=(status,status,current,url)

        else:

            insert_sql = 'insert into tb_download values(?,?,?)'

            args=(url,status,current)



        try:

            self.cursor.execute(insert_sql,args)

        except:

            self.conn.rollback()

            return False

        else:

            self.conn.commit()

            return True



    def get(self):

        sql = 'select idx,start,end from tb_record where status=0'

        self.cursor.execute(sql)

        ret= self.cursor.fetchone()

        return ret



    def set(self,idx):

        print('set status =1')

        sql='update tb_record set status=1 where idx=?'

        self.cursor.execute(sql,(idx,))

        self.conn.commit()



def llogger(filename):

    logger = logging.getLogger(filename)  # 不加名称设置root logger



    logger.setLevel(logging.DEBUG)  # 设置输出级别



    formatter = logging.Formatter(

        '[%(asctime)s][%(filename)s][line: %(lineno)d]\[%(levelname)s] ## %(message)s)',

        datefmt='%Y-%m-%d %H:%M:%S')



    # 使用FileHandler输出到文件

    prefix = os.path.splitext(filename)[0]

    fh = logging.FileHandler(prefix + '.log')

    fh.setLevel(logging.DEBUG)

    fh.setFormatter(formatter)



    # 使用StreamHandler输出到屏幕

    ch = logging.StreamHandler()

    ch.setLevel(logging.DEBUG)

    ch.setFormatter(formatter)



    # 添加两个Handler

    logger.addHandler(ch)

    logger.addHandler(fh)

    return logger





logger = llogger('download.log')

sql_obj = SQLite()



def run():

    while 1:

        result = sql_obj.get()

        print(result)

        if result:

            idx=result[0]

            start=result[1]

            end=result[2]

            try:

                download_bilibili(idx,start,end)

            except:

                pass

            else:

                sql_obj.set(idx)

        else:

            time.sleep(MINS*60)



def download_bilibili(id,start_page,total_page):

    global doc



    bilibili_url = 'https://www.bilibili.com/video/{}?p={}'

    for i in range(start_page, total_page+1):



        next_url = bilibili_url.format(id, i)

        if sql_obj.exists(next_url):

            print('have download')

            continue



        try:

            command = CMD.format(YOU_GET_PATH, next_url)

            p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,

                                 shell=True)



            output, error = p.communicate()



        except Exception as e:

            print('has execption')

            sql_obj.insert_history(next_url,status=0)

            logger.error(e)

            continue

        else:

            output_str = output.decode()

            if len(output_str) == 0:

                sql_obj.insert_history(next_url,status=0)

                logger.info('下载失败')

                continue



            logger.info('{} has been downloaded !'.format(next_url))

            sql_obj.insert_history(next_url,status=1)



run()

def initialize(context):

    # 初始化此策略

    # 设置我们要操作的股票池, 这里我们只操作一支股票

    g.security = '600570.SS'

    set_universe(g.security)

    #每天9:23分运行集合竞价处理函数

    run_daily(context, aggregate_auction_func, time='9:23')  



def aggregate_auction_func(context):

    stock = g.security

    #最新价

    snapshot = get_snapshot(stock)

    price = snapshot[stock]['last_px']

    #涨停价

    up_limit = snapshot[stock]['up_px']

    #如果最新价不小于涨停价，买入

    if float(price) >= float(up_limit):

        order(g.security, 100, limit_price=up_limit)

    

def handle_data(context, data):

    pass

def initialize(context):

    # 初始化此策略

    # 设置我们要操作的股票池, 这里我们只操作一支股票

    g.security = '600570.SS'

    set_universe(g.security)

    pass

    

#当五日均线高于十日均线时买入，当五日均线低于十日均线时卖出

def handle_data(context, data):

    security = g.security



    #得到十日历史价格

    df = get_history(10, '1d', 'close', security, fq=None, include=False)

	

    # 得到五日均线价格

    ma5 = round(df['close'][-5:].mean(), 3)

    

    # 得到十日均线价格

    ma10 = round(df['close'][-10:].mean(), 3)



    # 取得昨天收盘价

    price = data[security]['close']



    # 得到当前资金余额

    cash = context.portfolio.cash

    

    # 如果当前有余额，并且五日均线大于十日均线

    if ma5 > ma10:

        # 用所有 cash 买入股票

        order_value(security, cash)

        # 记录这次买入

        log.info("Buying %s" % (security))

        

    # 如果五日均线小于十日均线，并且目前有头寸

    elif ma5 < ma10 and get_position(security).amount > 0:

        # 全部卖出

        order_target(security, 0)

        # 记录这次卖出

        log.info("Selling %s" % (security))

def initialize(context):

    # 初始化此策略

    # 设置我们要操作的股票池, 这里我们只操作一支股票

    g.security = '600570.SS'

    set_universe(g.security)

    #每3秒运行一次主函数

    run_interval(context, func, seconds=3)

      

#盘前准备历史数据

def before_trading_start(context, data):

    history = get_history(10, '1d', 'close', g.security, fq='pre', include=False)

    g.close_array = history['close'].values

    

#当五日均线高于十日均线时买入，当五日均线低于十日均线时卖出

def func(context):

    

    stock = g.security

    

    #获取最新价

    snapshot = get_snapshot(stock)

    price = snapshot[stock]['last_px']

    

    # 得到五日均线价格

	days = 5

    ma5 = get_MA_day(stock, days, g.close_array[-4:], price)   

    # 得到十日均线价格

	days = 10

    ma10 = get_MA_day(stock, days, g.close_array[-9:], price)



    # 得到当前资金余额

    cash = context.portfolio.cash

    

    # 如果当前有余额，并且五日均线大于十日均线

    if ma5 > ma10:

        # 用所有 cash 买入股票

        order_value(stock, cash)

        # 记录这次买入

        log.info("Buying %s" % (stock))

        

    # 如果五日均线小于十日均线，并且目前有头寸

    elif ma5 < ma10 and get_position(stock).amount > 0:

        # 全部卖出

        order_target(stock, 0)

        # 记录这次卖出

        log.info("Selling %s" % (stock))    



#计算实时均线函数

def get_MA_day(stock,days,close_array,current_price):

    close_sum = close_array[-(days-1):].sum()

    MA = (current_price + close_sum)/days

    return MA



def handle_data(context, data):

    pass

def f_expma(N,m,EXPMA1,price):

    a = m/(N+1)

    EXPMA2 = a * price + (1 - a)*EXPMA1

    return EXPMA2 #2为后一天值



#定义macd函数，输入平滑系数参数、前一日值，输出当日值

def macd(N1,N2,N3,m,EXPMA12_1,EXPMA26_1,DEA1,price):

    EXPMA12_2 = f_expma(N1,m,EXPMA12_1,price)

    EXPMA26_2 = f_expma(N2,m,EXPMA26_1,price)

    DIF2 = EXPMA12_2 - EXPMA26_2

    a = m/(N3+1)

    DEA2 = a * DIF2 + (1 - a)*DEA1

    BAR2=2*(DIF2-DEA2)

    return EXPMA12_2,EXPMA26_2,DIF2,DEA2,BAR2



def initialize(context):

    global init_price

    init_price = None

    # 获取沪深300股票

    g.security = get_index_stocks('000300.SS')

    #g.security = ['600570.SS']

    # 设置我们要操作的股票池, 这里我们只操作一支股票

    set_universe(g.security)



def handle_data(context, data):

    # 获取历史数据，这里只获取了2天的数据，如果希望最终MACD指标结果更准确最好是获取

    # 从股票上市至今的所有历史数据，即增加获取的天数

    close_price = get_history(2, '1d', field='close', security_list=g.security)

    #如果是停牌不进行计算

    for security in g.security:

        if data[security].is_open >0:

            global init_price,EXPMA12_1,EXPMA26_1,EXPMA12_2,EXPMA26_2,DIF1,DIF2,DEA1,DEA2

            if init_price is None:

                init_price = close_price[security].mean()#nan和N-1个数，mean为N-1个数的均值

                EXPMA12_1 = init_price

                EXPMA26_1 = init_price

                DIF1 = init_price

                DEA1 = init_price

            # m用于计算平滑系数a=m/(N+1)

            m = 2.0

            #设定指数平滑基期数

            N1 = 12

            N2 = 26

            N3 = 9

            EXPMA12_2,EXPMA26_2,DIF2,DEA2,BAR2 = macd(N1,N2,N3,m,EXPMA12_1,EXPMA26_1,DEA1,close_price[security][-1])

            # 取得当前价格

            current_price = data[security].price

            # 取得当前的现金

            cash = context.portfolio.cash

            # DIF、DEA均为正，DIF向上突破DEA，买入信号参考

            if DIF2 > 0 and DEA2 > 0 and DIF1 < DEA1 and DIF2 > DEA2:

                # 计算可以买多少只股票

                number_of_shares = int(cash/current_price)

                # 购买量大于0时，下单

                if number_of_shares > 0:

                    # 以市单价买入股票，日回测时即是开盘价

                    order(security, +number_of_shares)

                    # 记录这次买入

                    log.info("Buying %s" % (security))

            # DIF、DEA均为负，DIF向下突破DEA，卖出信号参考

            elif DIF2 < 0 and DEA2 < 0 and DIF1 > DEA1 and DIF2 < DEA2 and get_position(security).amount > 0:

                # 卖出所有股票,使这只股票的最终持有量为0

                order_target(security, 0)

                # 记录这次卖出

                log.info("Selling %s" % (security))

            # 将今日的值赋给全局变量作为下一次前一日的值

            DEA1 = DEA2

            DIF1 = DIF2

            EXPMA12_1 = EXPMA12_2

            EXPMA26_1 = EXPMA26_2

    def initialize(context):

 2    # 初始化此策略

 3    # 设置我们要操作的股票池, 这里我们只操作一支股票

 4    g.security = '600570.SS'

 5    set_universe(g.security)

 6    pass

 7

 8 #当五日均线高于十日均线时买入，当五日均线低于十日均线时卖出

 9 def handle_data(context, data):

10    security = g.security

11

12    #得到十日历史价格

13    df = get_history(10, '1d', 'close', security, fq=None, include=False)

14

15    # 得到五日均线价格

16    ma5 = round(df['close'][-5:].mean(), 3)

17

18    # 得到十日均线价格

19    ma10 = round(df['close'][-10:].mean(), 3)

20

21    # 取得昨天收盘价

22    price = data[security]['close']

23

24    # 得到当前资金余额

25    cash = context.portfolio.cash

26

27    # 如果当前有余额，并且五日均线大于十日均线

28    if ma5 > ma10:

29        # 用所有 cash 买入股票

30        order_value(security, cash)

31        # 记录这次买入

32        log.info("Buying %s" % (security))

33

34    # 如果五日均线小于十日均线，并且目前有头寸

35    elif ma5 < ma10 and get_position(security).amount > 0:

36        # 全部卖出

37        order_target(security, 0)

38        # 记录这次卖出

39        log.info("Selling %s" % (security))

    def f_expma(N,m,EXPMA1,price):

 2    a = m/(N+1)

 3    EXPMA2 = a * price + (1 - a)*EXPMA1

 4    return EXPMA2 #2为后一天值

 5

 6 #定义macd函数，输入平滑系数参数、前一日值，输出当日值

 7  def macd(N1,N2,N3,m,EXPMA12_1,EXPMA26_1,DEA1,price):

 8    EXPMA12_2 = f_expma(N1,m,EXPMA12_1,price)

 9    EXPMA26_2 = f_expma(N2,m,EXPMA26_1,price)

10    DIF2 = EXPMA12_2 - EXPMA26_2

11    a = m/(N3+1)

12    DEA2 = a * DIF2 + (1 - a)*DEA1

13    BAR2=2*(DIF2-DEA2)

14    return EXPMA12_2,EXPMA26_2,DIF2,DEA2,BAR2

15

16  def initialize(context):

17    global init_price

18    init_price = None

19    # 获取沪深300股票

20    g.security = get_index_stocks('000300.SS')

21    #g.security = ['600570.SS']

22    # 设置我们要操作的股票池, 这里我们只操作一支股票

23    set_universe(g.security)

24

25  def handle_data(context, data):

26    # 获取历史数据，这里只获取了2天的数据，如果希望最终MACD指标结果更准确最好是获取

27    # 从股票上市至今的所有历史数据，即增加获取的天数

28    close_price = get_history(2, '1d', field='close', security_list=g.security)

29    #如果是停牌不进行计算

30    for security in g.security:

31        if data[security].is_open >0:

32            global init_price,EXPMA12_1,EXPMA26_1,EXPMA12_2,EXPMA26_2,DIF1,DIF2,DEA1,DEA2

33            if init_price is None:

34                init_price = close_price[security].mean()#nan和N-1个数，mean为N-1个数的均值

35                EXPMA12_1 = init_price

36                EXPMA26_1 = init_price

37                DIF1 = init_price

38                DEA1 = init_price

39            # m用于计算平滑系数a=m/(N+1)

40            m = 2.0

41            #设定指数平滑基期数

42            N1 = 12

43            N2 = 26

44            N3 = 9

45            EXPMA12_2,EXPMA26_2,DIF2,DEA2,BAR2 = macd(N1,N2,N3,m,EXPMA12_1,EXPMA26_1,DEA1,close_price[security][-1])

46            # 取得当前价格

47            current_price = data[security].price

48            # 取得当前的现金

49            cash = context.portfolio.cash

50            # DIF、DEA均为正，DIF向上突破DEA，买入信号参考

51            if DIF2 > 0 and DEA2 > 0 and DIF1 < DEA1 and DIF2 > DEA2:

52                # 计算可以买多少只股票

53                number_of_shares = int(cash/current_price)

54                # 购买量大于0时，下单

55                if number_of_shares > 0:

56                    # 以市单价买入股票，日回测时即是开盘价

57                    order(security, +number_of_shares)

58                    # 记录这次买入

59                    log.info("Buying %s" % (security))

60            # DIF、DEA均为负，DIF向下突破DEA，卖出信号参考

61            elif DIF2 < 0 and DEA2 < 0 and DIF1 > DEA1 and DIF2 < DEA2 and get_position(security).amount > 0:

62                # 卖出所有股票,使这只股票的最终持有量为0

63                order_target(security, 0)

64                # 记录这次卖出

65                log.info("Selling %s" % (security))

66            # 将今日的值赋给全局变量作为下一次前一日的值

67            DEA1 = DEA2

68            DIF1 = DIF2

69            EXPMA12_1 = EXPMA12_2

70            EXPMA26_1 = EXPMA26_2

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/2/23 19:34

# @File : youdao.py

# 解密有道词典的JS





import hashlib

import random

import requests

import time





def md5_(word):

    s = bytes(word, encoding='utf8')

    m = hashlib.md5()

    m.update(s)

    ret = m.hexdigest()

    return ret



def get_sign(word, salt):

    ret = md5_('fanyideskweb' + word + salt + 'p09@Bn{h02_BIEe]$P^nG')

    return ret





def youdao(word):

    url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'

    headers = {

        'Host': 'fanyi.youdao.com',

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0',

        'Accept': 'application/json, text/javascript, */*; q=0.01',

        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

        'Accept-Encoding': 'gzip, deflate',

        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',

        'X-Requested-With': 'XMLHttpRequest',

        'Referer': 'http://fanyi.youdao.com/',

        'Content-Length': '252',

        'Cookie': 'YOUDAO_MOBILE_ACCESS_TYPE=1; OUTFOX_SEARCH_USER_ID=1672542763@10.169.0.83; JSESSIONID=aaaWzxpjeDu1gbhopLzKw; ___rl__test__cookies=1550913722828; OUTFOX_SEARCH_USER_ID_NCOO=372126049.6326876',

        'Connection': 'keep-alive',

        'Pragma': 'no-cache',

        'Cache-Control': 'no-cache',

    }



    ts = str(int(time.time()*1000))

    salt=ts+str(random.randint(0,10))

    bv = md5_("5.0 (Windows)")

    sign= get_sign(word,salt)



    post_data = {

        'i': word,

        'from': 'AUTO', 'to': 'AUTO', 'smartresult': 'dict', 'client': 'fanyideskweb', 'salt': salt,

        'sign': sign, 'ts': ts, 'bv': bv, 'doctype': 'json', 'version': '2.1',

        'keyfrom': 'fanyi.web', 'action': 'FY_BY_REALTIME', 'typoResult': 'false'

    }



    r = requests.post(

        url=url,

        headers=headers,

        data=post_data

    )



    for item in r.json().get('smartResult',{}).get('entries'):

        print(item)



word='student'

youdao(word)

from PIL import Image

im = Image.open('chinese.jpg')

plt.figure(figsize=(20,20))

plt.imshow(im)



pytesseract.image_to_string(im,lang='chi_sim')

'可L又使用以下的语句i上图片显示大 此'

""" 读取图片 """

def get_file_content(filePath):

    with open(filePath, 'rb') as fp:

        return fp.read()



image = get_file_content('example.jpg')



""" 调用通用文字识别, 图片参数为本地图片 """

client.basicGeneral(image);



""" 如果有可选参数 """

options = {}

options["language_type"] = "CHN_ENG"

options["detect_direction"] = "true"

options["detect_language"] = "true"

options["probability"] = "true"



from aip import AipOcr



""" 你的 APPID AK SK """

APP_ID = '你的 App ID'

API_KEY = '你的 Api Key'

SECRET_KEY = '你的 Secret Key'



client = AipOcr(APP_ID, API_KEY, SECRET_KEY)





""" 带参数调用通用文字识别, 图片参数为本地图片 """

client.basicGeneral(image, options)



url = "https//www.x.com/sample.jpg"



""" 调用通用文字识别, 图片参数为远程url图片 """

client.basicGeneralUrl(url);



""" 如果有可选参数 """

options = {}

options["language_type"] = "CHN_ENG"

options["detect_direction"] = "true"

options["detect_language"] = "true"

options["probability"] = "true"



""" 带参数调用通用文字识别, 图片参数为远程url图片 """

client.basicGeneralUrl(url, options)

可以使用以下的语句让图片显示大些

from  setting import get_mysql_conn,get_engine

import pandas as pd

import pymongo

from pyecharts import Geo,Style,Map

engine = get_engine('db_stock',local='local')

# 堆叠图

from pyecharts import Bar

df = pd.read_sql('tb_bond_jisilu',con=engine)



result ={}

for name,grades in df.groupby('评级'):

#     print(name,grades[['可转债名称','可转债价格']])

    for each in grades['可转债价格']:

        result.setdefault(name,)

        result[name].append(each)

        

        

# 确定价格的范围



value = [str(i) for i in range(85,140)]

ret = [0]*len(value)

ret1 = dict(zip(value,ret))



ret_A_add = ret1.copy()

for item in result['A+']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            ret_A_add[k]+=1

            

retAA_ = ret1.copy()

for item in result['']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA_[k]+=1



retAA = ret1.copy()

for item in result['AA']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA[k]+=1

            

retAA_add = ret1.copy()

for item in result['AA+']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA_add[k]+=1

            

retAAA = ret1.copy()

for item in result['AAA']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAAA[k]+=1



bar = Bar('可转债价格分布')

bar.add('A+',value,list(ret_A_add.values()),is_stack=True,yaxis_max=11)

bar.add('',value,list(retAA_.values()),is_stack=True,yaxis_max=11)

bar.add('AA',value,list(retAA.values()),is_stack=True,yaxis_max=11)

bar.add('AA+',value,list(retAA_add.values()),is_stack=True,yaxis_max=11)

bar.add('AAA',value,list(retAAA.values()),is_stack=True,yaxis_max=11)

#-*-coding=utf-8

'''

可转债监控

'''

import tushare as ts

from setting import get_engine

engine = get_engine('db_bond')

import pandas as pd

import datetime

class ConvertBond():



    def __init__(self):

        self.conn=ts.get_apis()

        self.allBonds=ts.new_cbonds(pause=2)

        self.onSellBond=self.allBonds.dropna(subset=['marketprice'])

        self.today=datetime.datetime.now().strftime('%Y-%m-%d %H:%M')



    def stockPrice(self,code):

        stock_df = ts.get_realtime_quotes(code)

        price = float(stock_df['price'].values[0])

        return price



    def dataframe(self):

        price_list=

        for code in self.onSellBond['scode']:

            price_list.append(self.stockPrice(code))

        self.onSellBond['stock_price']=price_list

        self.onSellBond['ratio'] = (

self.onSellBond['marketprice']

/(self.onSellBond['stock_price'] / self.onSellBond['convprice'])-1)*100

        self.onSellBond['Updated']=self.today

        self.onSellBond.to_sql('tb_bond',engine,if_exists='replace')



    def closed(self):

        ts.close_apis(self.conn)



def main():

    bond=ConvertBond()

    bond.dataframe()

    bond.closed()

if __name__=='__main__':

    main()

import os

import MySQLdb

MYSQL_USER = *********

MYSQL_PASSWORD = ********

MYSQL_HOST = *********

MYSQL_PORT = *****



def get_engine(db):

    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, db))

    return engine

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print('df', df)

    print('df2', df2)



    df_x = [df, df2]

    result = pd.concat(df_x)

    print('first result ', result)

result2= result.reset_index()

result2 = result.reset_index(drop=True)

result3 = result.reindex(columns=['A','C'])

__author__ = 'rocchen'

# working v1.0

from bs4 import BeautifulSoup

import urllib2, datetime, time, codecs, cookielib, random, threading

import os,sys





def getInfo(max_index_user=5):

    stock_news_site =

"http://ggjd.cnstock.com/gglist/search/ggkx/"

 

    my_userAgent = [

        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)']

    index = 0

    max_index = max_index_user

    num = 1

    temp_time = time.strftime("[%Y-%m-%d]-[%H-%M]", time.localtime())



    store_filename = "StockNews-%s.log" % temp_time

    fOpen = codecs.open(store_filename, 'w', 'utf-8')



    while index < max_index:

        user_agent = random.choice(my_userAgent)

        # print user_agent

        company_news_site = stock_news_site + str(index)

        # content = urllib2.urlopen(company_news_site)

        headers = {'User-Agent': user_agent, 'Host': "ggjd.cnstock.com", 'DNT': '1',

                   'Accept': 'text/html, application/xhtml+xml, */*', }

        req = urllib2.Request(url=company_news_site, headers=headers)

        resp = None

        raw_content = ""

        try:

            resp = urllib2.urlopen(req, timeout=30)



        except urllib2.HTTPError as e:

            e.fp.read()

        except urllib2.URLError as e:

            if hasattr(e, 'code'):

                print "error code %d" % e.code

            elif hasattr(e, 'reason'):

                print "error reason %s " % e.reason



        finally:

            if resp:

                raw_content = resp.read()

                time.sleep(2)

                resp.close()



        soup = BeautifulSoup(raw_content, "html.parser")

        all_content = soup.find_all("span", "time")



        for i in all_content:

            news_time = i.string

            node = i.next_sibling

            str_temp = "No.%s \n%s\t%s\n---> %s \n\n" % (str(num), news_time, node['title'], node['href'])

            #print "inside %d" %num

            #print str_temp

            fOpen.write(str_temp)

            num = num + 1



        #print "index %d" %index

        index = index + 1



    fOpen.close()





def execute_task(n=60):

    period = int(n)

    while True:

        print datetime.datetime.now()

        getInfo(3)

        

        time.sleep(60 * period)

        





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "stock")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    start_time = time.time()  # user can change the max index number getInfo(10), by default is getInfo(5)

    if len(sys.argv) <2:

        n = raw_input("Input Period : ? mins to download every cycle")

    else:

        n=int(sys.argv[1])

    execute_task(n)

    end_time = time.time()

    print "Total time: %s s." % str(round((end_time - start_time), 4))

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

# -*- coding=utf-8 -*-

__author__ = 'rocky @ www.30daydo.com'

import urllib2, re, os, codecs,sys,datetime

from bs4 import BeautifulSoup

# example https://zhhrb.sinaapp.com/index.php?date=20160610

from mail_template import MailAtt

reload(sys)

sys.setdefaultencoding('utf-8')



def save2file(filename, content):

    filename = filename + ".txt"

    f = codecs.open(filename, 'a', encoding='utf-8')

    f.write(content)

    f.close()





def getPost(date_time, filter_p):

    url = 'https://zhhrb.sinaapp.com/index.php?date=' + date_time

    user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

    header = {"User-Agent": user_agent}

    req = urllib2.Request(url, headers=header)

    resp = urllib2.urlopen(req)

    content = resp.read()

    p = re.compile('<h2 class="question-title">(.*)</h2></br></a>')

    result = re.findall(p, content)

    count = -1

    row = -1

    for i in result:

        #print i

        return_content = re.findall(filter_p, i)



        if return_content:

            row = count

            break

            #print return_content[0]

        count = count + 1

    #print row

    if row == -1:

        return 0

    link_p = re.compile('<a href="(.*)" target="_blank" rel="nofollow">')

    link_result = re.findall(link_p, content)[row + 1]

    print link_result

    result_req = urllib2.Request(link_result, headers=header)

    result_resp = urllib2.urlopen(result_req)

    #result_content= result_resp.read()

    #print result_content



    bs = BeautifulSoup(result_resp, "html.parser")

    title = bs.title.string.strip()

    #print title

    filename = re.sub('[\/:*?"<>|]', '-', title)

    print filename

    print date_time

    save2file(filename, title)

    save2file(filename, "\n\n\n\n--------------------%s Detail----------------------\n\n" %date_time)



    detail_content = bs.find_all('div', class_='content')



    for i in detail_content:

        #print i

        save2file(filename,"\n\n-------------------------answer  -------------------------\n\n")

        for j in i.strings:



            save2file(filename, j)



    smtp_server = 'smtp.126.com'

    from_mail = sys.argv[1]

    password = sys.argv[2]

    to_mail = 'xxxxx@kindle.cn'

    send_kindle = MailAtt(smtp_server, from_mail, password, to_mail)

    send_kindle.send_txt(filename)





def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)





    date_time = '20160611'

    filter_p = re.compile('大误.*')

    ori_day=datetime.date(datetime.date.today().year,01,01)

    t=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    delta=(t-ori_day).days

    print delta

    for i in range(delta):

        day=datetime.date(datetime.date.today().year,01,01)+datetime.timedelta(i)

        getPost(day.strftime("%Y%m%d"),filter_p)

    #getPost(date_time, filter_p)



if __name__ == "__main__":

    main()

# -*-coding=utf-8-*-

#抓取雪球的收藏文章

__author__ = 'Rocky'

import requests,cookielib,re,json,time

from toolkit import Toolkit

from lxml import etree

url='https://xueqiu.com/snowman/login'

session = requests.session()



session.cookies = cookielib.LWPCookieJar(filename="cookies")

try:

    session.cookies.load(ignore_discard=True)

except:

    print "Cookie can't load"



agent = 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'

headers = {'Host': 'xueqiu.com',

           'Referer': 'https://xueqiu.com/',

           'Origin':'https://xueqiu.com',

           'User-Agent': agent}

account=Toolkit.getUserData('data.cfg')

print account['snowball_user']

print account['snowball_password']



data={'username':account['snowball_user'],'password':account['snowball_password']}

s=session.post(url,data=data,headers=headers)

print s.status_code

#print s.text

session.cookies.save()

fav_temp='https://xueqiu.com/favs?page=1'

collection=session.get(fav_temp,headers=headers)

fav_content= collection.text

p=re.compile('"maxPage":(\d+)')

maxPage=p.findall(fav_content)[0]

print maxPage

print type(maxPage)

maxPage=int(maxPage)

print type(maxPage)

for i in range(1,maxPage+1):

    fav='https://xueqiu.com/favs?page=%d' %i

    collection=session.get(fav,headers=headers)

    fav_content= collection.text

    #print fav_content

    p=re.compile('var favs = {(.*?)};',re.S|re.M)

    result=p.findall(fav_content)[0].strip()



    new_result='{'+result+'}'

    #print type(new_result)

    #print new_result

    data=json.loads(new_result)

    use_data= data['list']

    host='https://xueqiu.com'

    for i in use_data:

        url=host+ i['target']

        print url

        txt_content=session.get(url,headers=headers).text

        #print txt_content.text



        tree=etree.HTML(txt_content)

        title=tree.xpath('//title/text()')[0]



        filename = re.sub('[\/:*?"<>|]', '-', title)

        print filename



        content=tree.xpath('//div[@class="detail"]')

        for i in content:

            Toolkit.save2filecn(filename, i.xpath('string(.)'))

        #print content

        #Toolkit.save2file(filename,)

        time.sleep(10)

import socket,sys

import time

from thread_test import MyThread



socket.setdefaulttimeout(1)

#设置每个线程socket的timeou时间，超过1秒没有反应就认为端口不开放

thread_num=4

#线程数目 

ip_end=256

ip_start=0

scope=ip_end/thread_num



def scan(ip_head,ip_low, port):

    try:

        # Alert !!! below statement should be inside scan function. Else each it is one s

        ip=ip_head+str(ip_low)

	print ip

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

	s.connect((ip, port))

	#通过这一句判断 是否连通

        s.close()

	print "ip %s port %d open\n" %(ip,port)

        return True

    except:

        return False





def scan_range(ip_head,ip_range,port):

	start,end=ip_range

	for i in range(start,end):

		scan(ip_head,i,port)



if len(sys.argv)<3:

	print "input ip and port"

	exit()



ip_head=sys.argv[1]

port=int(sys.argv[2])





ip_range=

for i in range(thread_num):

	x_range=[i*scope,(i+1)*scope-1]

	ip_range.append(x_range)



threads=

for i in range(thread_num):

	t=MyThread(scan_range,(ip_head,ip_range,port))

	threads.append(t)

for i in range(thread_num):

	threads.start()

for i in range(thread_num):

	threads.join()

	#设置进程阻塞，防止主线程退出了，其他的多线程还在运行



print "*****end*****"