|
| 1 | +import random |
| 2 | +import json |
| 3 | +from datetime import timedelta |
| 4 | +from os import path |
| 5 | + |
| 6 | +from py12306.cluster.cluster import Cluster |
| 7 | +from py12306.config import Config |
| 8 | +from py12306.app import app_available_check |
| 9 | +from py12306.helpers.api import API_CHECK_CDN_AVAILABLE, HOST_URL_OF_12306 |
| 10 | +from py12306.helpers.func import * |
| 11 | +from py12306.helpers.request import Request |
| 12 | +from py12306.log.common_log import CommonLog |
| 13 | + |
| 14 | + |
| 15 | +@singleton |
| 16 | +class Cdn: |
| 17 | + """ |
| 18 | + CDN 管理 |
| 19 | + """ |
| 20 | + items = [] |
| 21 | + available_items = [] |
| 22 | + unavailable_items = [] |
| 23 | + recheck_available_items = [] |
| 24 | + recheck_unavailable_items = [] |
| 25 | + retry_time = 3 |
| 26 | + is_ready = False |
| 27 | + is_finished = False |
| 28 | + is_ready_num = 10 # 当可用超过 10，已准备好 |
| 29 | + is_alive = True |
| 30 | + is_recheck = False |
| 31 | + |
| 32 | + safe_stay_time = 0.2 |
| 33 | + retry_num = 1 |
| 34 | + thread_num = 8 |
| 35 | + check_time_out = 3 |
| 36 | + |
| 37 | + last_check_at = 0 |
| 38 | + save_second = 5 |
| 39 | + check_keep_second = 60 * 60 * 24 |
| 40 | + |
| 41 | + def __init__(self): |
| 42 | + self.cluster = Cluster() |
| 43 | + create_thread_and_run(self, 'watch_cdn', False) |
| 44 | + |
| 45 | + def init_data(self): |
| 46 | + self.items = [] |
| 47 | + self.available_items = [] |
| 48 | + self.unavailable_items = [] |
| 49 | + self.is_finished = False |
| 50 | + self.is_ready = False |
| 51 | + self.is_recheck = False |
| 52 | + |
| 53 | + def update_cdn_status(self, auto=False): |
| 54 | + if auto: |
| 55 | + if Config().is_cdn_enabled(): |
| 56 | + self.run() |
| 57 | + else: |
| 58 | + self.destroy() |
| 59 | + |
| 60 | + @classmethod |
| 61 | + def run(cls): |
| 62 | + self = cls() |
| 63 | + app_available_check() |
| 64 | + self.is_alive = True |
| 65 | + self.start() |
| 66 | + pass |
| 67 | + |
| 68 | + def start(self): |
| 69 | + if not Config.is_cdn_enabled() or Config().is_slave(): return |
| 70 | + self.load_items() |
| 71 | + CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_START_TO_CHECK.format(len(self.items))).flush() |
| 72 | + self.restore_items() |
| 73 | + for i in range(self.thread_num): # 多线程 |
| 74 | + create_thread_and_run(jobs=self, callback_name='check_available', wait=Const.IS_TEST) |
| 75 | + |
| 76 | + def load_items(self): |
| 77 | + with open(Config().CDN_ITEM_FILE, encoding='utf-8') as f: |
| 78 | + for line, val in enumerate(f): |
| 79 | + self.items.append(val.rstrip('\n')) |
| 80 | + |
| 81 | + def restore_items(self): |
| 82 | + """ |
| 83 | + 恢复已有数据 |
| 84 | + :return: bool |
| 85 | + """ |
| 86 | + result = False |
| 87 | + if path.exists(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE): |
| 88 | + with open(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE, encoding='utf-8') as f: |
| 89 | + result = f.read() |
| 90 | + try: |
| 91 | + result = json.loads(result) |
| 92 | + except json.JSONDecodeError as e: |
| 93 | + result = {} |
| 94 | + |
| 95 | + # if Config.is_cluster_enabled(): # 集群不用同步 cdn |
| 96 | + # result = self.get_data_from_cluster() |
| 97 | + |
| 98 | + if result: |
| 99 | + self.last_check_at = result.get('last_check_at', '') |
| 100 | + if self.last_check_at: self.last_check_at = str_to_time(self.last_check_at) |
| 101 | + self.available_items = result.get('items', []) |
| 102 | + self.unavailable_items = result.get('fail_items', []) |
| 103 | + CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_RESTORE_SUCCESS.format(self.last_check_at, |
| 104 | + self.last_check_at + timedelta( |
| 105 | + seconds=self.check_keep_second))).flush() |
| 106 | + return True |
| 107 | + return False |
| 108 | + |
| 109 | + # def get_data_from_cluster(self): |
| 110 | + # available_items = self.cluster.session.smembers(Cluster.KEY_CDN_AVAILABLE_ITEMS) |
| 111 | + # last_time = self.cluster.session.get(Cluster.KEY_CDN_LAST_CHECK_AT, '') |
| 112 | + # if available_items and last_time: |
| 113 | + # return {'items': available_items, 'last_check_at': last_time} |
| 114 | + # return False |
| 115 | + |
| 116 | + def is_need_to_recheck(self): |
| 117 | + """ |
| 118 | + 是否需要重新检查 cdn |
| 119 | + :return: |
| 120 | + """ |
| 121 | + if self.last_check_at and ( |
| 122 | + time_now() - self.last_check_at).seconds > self.check_keep_second: |
| 123 | + return True |
| 124 | + return False |
| 125 | + |
| 126 | + def get_unchecked_item(self): |
| 127 | + if not self.is_recheck: |
| 128 | + items = list(set(self.items) - set(self.available_items) - set(self.unavailable_items)) |
| 129 | + else: |
| 130 | + items = list(set(self.items) - set(self.recheck_available_items) - set(self.recheck_unavailable_items)) |
| 131 | + if items: return random.choice(items) |
| 132 | + return None |
| 133 | + |
| 134 | + def check_available(self): |
| 135 | + while True and self.is_alive: |
| 136 | + item = self.get_unchecked_item() |
| 137 | + if not item: return self.check_did_finished() |
| 138 | + self.check_item_available(item) |
| 139 | + |
| 140 | + def watch_cdn(self): |
| 141 | + """ |
| 142 | + 监控 cdn 状态，自动重新检测 |
| 143 | + :return: |
| 144 | + """ |
| 145 | + while True: |
| 146 | + if self.is_alive and not self.is_recheck and self.is_need_to_recheck(): # 重新检测 |
| 147 | + self.is_recheck = True |
| 148 | + self.is_finished = False |
| 149 | + CommonLog.add_quick_log( |
| 150 | + CommonLog.MESSAGE_CDN_START_TO_RECHECK.format(len(self.items), time_now())).flush() |
| 151 | + for i in range(self.thread_num): # 多线程 |
| 152 | + create_thread_and_run(jobs=self, callback_name='check_available', wait=Const.IS_TEST) |
| 153 | + stay_second(self.retry_num) |
| 154 | + |
| 155 | + def destroy(self): |
| 156 | + """ |
| 157 | + 关闭 CDN |
| 158 | + :return: |
| 159 | + """ |
| 160 | + CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_CLOSED).flush() |
| 161 | + self.is_alive = False |
| 162 | + self.init_data() |
| 163 | + |
| 164 | + def check_item_available(self, item, try_num=0): |
| 165 | + session = Request() |
| 166 | + response = session.get(API_CHECK_CDN_AVAILABLE.format(item), headers={'Host': HOST_URL_OF_12306}, |
| 167 | + timeout=self.check_time_out, |
| 168 | + verify=False) |
| 169 | + |
| 170 | + if response.status_code == 200: |
| 171 | + if not self.is_recheck: |
| 172 | + self.available_items.append(item) |
| 173 | + else: |
| 174 | + self.recheck_available_items.append(item) |
| 175 | + if not self.is_ready: self.check_is_ready() |
| 176 | + elif try_num < self.retry_num: # 重试 |
| 177 | + stay_second(self.safe_stay_time) |
| 178 | + return self.check_item_available(item, try_num + 1) |
| 179 | + else: |
| 180 | + if not self.is_recheck: |
| 181 | + self.unavailable_items.append(item) |
| 182 | + else: |
| 183 | + self.recheck_unavailable_items.append(item) |
| 184 | + if not self.is_recheck and ( |
| 185 | + not self.last_check_at or (time_now() - self.last_check_at).seconds > self.save_second): |
| 186 | + self.save_available_items() |
| 187 | + stay_second(self.safe_stay_time) |
| 188 | + |
| 189 | + def check_did_finished(self): |
| 190 | + self.is_ready = True |
| 191 | + if not self.is_finished: |
| 192 | + self.is_finished = True |
| 193 | + if self.is_recheck: |
| 194 | + self.is_recheck = False |
| 195 | + self.available_items = self.recheck_available_items |
| 196 | + self.unavailable_items = self.recheck_unavailable_items |
| 197 | + self.recheck_available_items = [] |
| 198 | + self.recheck_unavailable_items = [] |
| 199 | + CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_CHECKED_SUCCESS.format(len(self.available_items))).flush() |
| 200 | + self.save_available_items() |
| 201 | + |
| 202 | + def save_available_items(self): |
| 203 | + self.last_check_at = time_now() |
| 204 | + data = {'items': self.available_items, 'fail_items': self.unavailable_items, |
| 205 | + 'last_check_at': str(self.last_check_at)} |
| 206 | + with open(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE, 'w') as f: |
| 207 | + f.write(json.dumps(data)) |
| 208 | + |
| 209 | + # if Config.is_master(): |
| 210 | + # self.cluster.session.sadd(Cluster.KEY_CDN_AVAILABLE_ITEMS, self.available_items) |
| 211 | + # self.cluster.session.set(Cluster.KEY_CDN_LAST_CHECK_AT, time_now()) |
| 212 | + |
| 213 | + def check_is_ready(self): |
| 214 | + if len(self.available_items) > self.is_ready_num: |
| 215 | + self.is_ready = True |
| 216 | + else: |
| 217 | + self.is_ready = False |
| 218 | + |
| 219 | + @classmethod |
| 220 | + def get_cdn(cls): |
| 221 | + self = cls() |
| 222 | + if self.is_ready: |
| 223 | + return random.choice(self.available_items) |
| 224 | + return None |
| 225 | + |
| 226 | + |
| 227 | +if __name__ == '__main__': |
| 228 | + # Const.IS_TEST = True |
| 229 | + Cdn.run() |
| 230 | + while not Cdn().is_finished: |
| 231 | + stay_second(1) |
0 commit comments