# encoding: utf-8 # 维护人员：qiaoxinjiu import re import time import requests import json import urllib3 from base_framework.public_tools.read_config import get_current_env, ReadConfig urllib3.disable_warnings() class HuoHuaDBS: def __init__(self, user_name=None, pwd=None): self.conf = ReadConfig() # if not user_name: # user_name = self.conf.get_value(sections="lrq", options="show_username") # pwd = self.conf.get_value(sections="lrq", options="password") self.req = requests.session() self.user_name = user_name self.user_pwd = pwd self.headers = {} self.session_id = None self.authenticate_csrf_token = None self.query_url = "https://dbs.qc.huohua.cn/query/" self.check_url = "https://dbs.qc.huohua.cn/simplecheck/" self.commit_url = "https://dbs.qc.huohua.cn/autoreview/" self.execute_url = "https://dbs.qc.huohua.cn/execute/" self.get_status_url = "https://dbs.qc.huohua.cn/sqlworkflow/detail_content/?workflow_id=" self.env = get_current_env().lower() if self.env == "sim": # sim环境读取一次db配置, 方便后续场景中直接使用，避免重复查询 self.sim_db = {} # 最终格式： {"teach_teacher": "sim126", "teach_classes": "sim128"} instance_list = self.conf.get_options(section="DB_LIST") for instance in instance_list: db_list = self.conf.get_value(sections="DB_LIST", options=instance) dbs = db_list.split(',') for db in dbs: self.sim_db[db] = instance def __get_instance_name(self, sql_content): if self.env.lower() == "qa" or "-" in self.env: # 如果是qa或独立环境，则默认查询qadb-slave if sql_content.split(' ')[0].lower() == "select": instance_name = "qadb-slave" # sql查询时，使用此名 else: instance_name = "qadb-master" # sql执行时，使用此名 else: # db_name = sql_content.split(".")[0].split(" ")[-1].replace("`", "").replace(" ", "") db_names = self.__get_db_names_from_sql(sql_statements=sql_content) instance_list = [] no_config_list = [] for db_name in db_names: if db_name not in self.sim_db: no_config_list.append(db_name) continue if self.sim_db[db_name] not in instance_list: instance_list.append(self.sim_db[db_name]) if len(no_config_list) > 0: raise Exception("你本次查询的数据库：{} 未做sim配置，请添加....".format(no_config_list)) if len(set(instance_list)) > 1: raise Exception("你本次查询的数据库：{}\n位于不同的实例:{}\n请修改sql....".format(db_names,instance_list)) instance_name = instance_list[0] return instance_name @staticmethod def __check_sql_content(sql_content): """检查sql语句是否符合规范""" sql_list = sql_content.split(" ") if sql_list[0].lower() in ["insert", "update", "delete", "select"]: if "." not in sql_content: raise Exception("sql语句必现是db_name.table_name格式，请修正:{}".format(sql_content)) @staticmethod def __get_db_names_from_sql(sql_statements): """ 提取sql语句中的数据库名称 :param sql_statements: sql语句 :return: 数据库名称列表,去重 """ sql_statements = sql_statements.replace("`", "") pattern = re.compile(r'\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-zA-Z0-9_]+)\.', re.IGNORECASE) databases = set() # 移除注释和多余空格 sql_clean = re.sub(r'--.*', '', sql_statements) # 移除行注释 sql_clean = ' '.join(sql_clean.split()) # 合并多余空格 # 查找所有匹配项 matches = pattern.findall(sql_clean) if matches: databases.update(matches) return list(databases) def dbs_login(self): login_index = 'http://dbs.qc.huohua.cn/login/' res_login = self.req.get(login_index) self.authenticate_csrf_token = res_login.cookies.get('csrftoken') authenticate_url = 'https://dbs.qc.huohua.cn/authenticate/' params_json_str = {"username": self.user_name, "password": self.user_pwd} self.headers.update({"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", "Cookie": "experimentation_subject_id=IjdmMmE5MmYxLTk4N2QtNDU1YS1hMGRjLTZhZWIwZWY3YWEyMiI%3D--fc2a7fdd42c874af494e99f3f5fdf0c372d3b4d2; __UUID=93b21b91-675f-4019-fe4f-4442ec9f6e65-x; __DEVICE_ID=93b21b91-675f-4019-fe4f-4442ec9f6e65-x; csrftoken={}".format( self.authenticate_csrf_token)}) self.headers.update({"X-CSRFToken": "{}".format(self.authenticate_csrf_token), "X-Requested-With": "XMLHttpRequest"}) resp = self.req.post(authenticate_url, params_json_str, headers=self.headers) # print(json.loads(resp.text).get("msg")) self.session_id = resp.cookies.get('sessionid') self.headers.update({ "Cookie": "experimentation_subject_id=IjdmMmE5MmYxLTk4N2QtNDU1YS1hMGRjLTZhZWIwZWY3YWEyMiI%3D--fc2a7fdd42c874af494e99f3f5fdf0c372d3b4d2; __UUID=93b21b91-675f-4019-fe4f-4442ec9f6e65-x; __DEVICE_ID=93b21b91-675f-4019-fe4f-4442ec9f6e65-x; csrftoken={};sessionid={}".format( self.authenticate_csrf_token, self.session_id)}) # print(self.session_id) def dbs_query(self, instance_name, sql_content, limit_num="100"): """ dbs查询 | 请求参数名 | 说明 | 类型 | 是否必填 | | instance_name | 数据库实例: 例如sim-my-allschool | string | True | | db_name | 数据库名称: hulk_class_help | string | True | | schema_name | 数据库模式: 默认空不填 | string | True | | tb_name | 表名 | string | True | | sql_content | 查询sql | string | True | | limit_num | 查询行数 | string | True | """ if not self.session_id: self.dbs_login() db_name = sql_content.split(".")[0].split(" ")[-1].replace("`", "").replace(" ", "") data = {"instance_name": instance_name, "db_name": db_name, "schema_name": "", "tb_name": "", "sql_content": sql_content, "limit_num": limit_num} self.headers.update({"Referer": "https://dbs.qc.huohua.cn/sqlquery/"}) resp = self.req.post(self.query_url, data, headers=self.headers) result = json.loads(resp.text) return result.get('data').get('rows') def dbs_query_as_json(self, instance_name, sql_content, limit_num="1000"): """ 通过DBS查询数据，并返回json格式 Args: instance_name: 数据库实例: 例如sim-my-allschool sql_content: 查询语句 limit_num: 查询行数 Returns: [{key:value}] """ if not self.session_id: self.dbs_login() self.headers.update({"Referer": "https://dbs.qc.huohua.cn/sqlquery/"}) # db_name = sql_content.lower().split("from ")[1].split(".")[0].replace("`", "").replace(" ", "") db_name = self.__get_db_names_from_sql(sql_statements=sql_content)[0] data = {"instance_name": instance_name, "db_name": db_name, "schema_name": "", "tb_name": "", "sql_content": sql_content, "limit_num": limit_num, "workflow_name": "ODS线上数据监控"} resp = self.req.post(self.query_url, data, headers=self.headers) result = json.loads(resp.text) if result.get('status') != 0: raise Exception(result) else: r_data = self.__make_format_data(key_list=result.get('data').get('column_list'), column_list=result.get('data').get('rows')) return r_data def dbs_query_as_json_auto_identify_env(self, sql_content, instance_name=None, limit_num="1000"): """ 通过DBS查询数据，并返回json格式【建议别链表查询，因为sim环境不支持，除非你的函数不在sim使用】 Args: sql_content: 查询语句 instance_name: 数据库实例: env_choose中是qa或独立环境时，此参数失效，否则已输入为准，不传则默认为sim126 limit_num: 查询行数 Returns: Note：可以根据你的查询表，直接指定sim环境的instance_name，因为qa环境时不使用此参数 """ self.__check_sql_content(sql_content) # sql格式检查 if not instance_name: instance_name = self.__get_instance_name(sql_content=sql_content) # 自动识别环境 if not self.session_id: self.dbs_login() self.headers.update({"Referer": "https://dbs.qc.huohua.cn/sqlquery/"}) # db_name = sql_content.split(".")[0].split(" ")[-1].replace("`", "").replace(" ", "") db_name = sql_content.lower().split("from ")[1].split(".")[0].replace("`", "").replace(" ", "") data = {"instance_name": instance_name, "db_name": db_name, "schema_name": "", "tb_name": "", "sql_content": sql_content, "limit_num": limit_num, "workflow_name": "Test_online_data_check"} resp = self.req.post(self.query_url, data, headers=self.headers) result = json.loads(resp.text) if result.get('status') != 0: raise Exception(result) else: r_data = self.__make_format_data(key_list=result.get('data').get('column_list'), column_list=result.get('data').get('rows')) return r_data def dbs_query_all_data_as_json(self, sql_content, instance_name=None, sort_key='id', sort_type='asc'): """ 通过DBS查询db中的所有数据(超过1000条也行)，并返回json格式，别带limit关键字，带了就走普通查询 Args: sql_content: 查询语句 instance_name: 数据库实例: env_choose中是qa或独立环境时，此参数失效，否则已输入为准，不传则默认为sim126 sort_key: 排序字段，默认为id sort_type: 排序类型，默认为asc-顺排，desc-倒排 """ # 如果查询语句带有limit限制，则直接走普通查询 if " limit " in sql_content.lower(): return self.dbs_query_as_json_auto_identify_env(sql_content=sql_content, instance_name=instance_name) # 否则走全量查询 if " join " in sql_content.lower() or " union " in sql_content.lower() or sql_content.lower().count("from") > 1: raise Exception("链表查询不支持全量查询，请修改查询语句") if "where" not in sql_content.lower(): raise Exception("全量查询必须带有where条件，防止数量过多，请修改查询语句") # 全量查询 all_data = [] off_set = 0 str_list = sql_content.lower().split("where") while True: if sort_type.lower() == 'asc': sql_str = ("{} where {}>{} and {}" .format(str_list[0], sort_key, off_set, str_list[1])) elif sort_type.lower() == 'desc': sql_str = ("{} where {}<{} and {}" .format(str_list[0], sort_key, off_set, str_list[1])) db_data = self.dbs_query_as_json_auto_identify_env(instance_name='prod-my-teach_classes-slave01', sql_content=sql_str) all_data += db_data if len(db_data) < 1000: break else: off_set = db_data[-1][sort_key] return all_data def dbs_query_as_list(self, instance_name, sql_content, limit_num="100"): """ 通过DBS查询数据，并返回list格式 Args: instance_name: 数据库实例: 例如sim-my-allschool sql_content: 查询语句 limit_num: 查询行数 Returns: [{key:value}] """ res = self.dbs_query_as_json(instance_name=instance_name, sql_content=sql_content, limit_num=limit_num) out_data = [] if len(res) > 0: for item in res: tmp_data = [] for key in item: if len(item) > 1: # 查询结课是多列，则返回二维数组 tmp_data.append(item[key]) else: # 查询结课是单列，则返回一维数组 tmp_data = item[key] out_data.append(tmp_data) return out_data @staticmethod def __make_format_data(key_list, column_list): """ 组装json格式的数据类型 Args: key_list: 列名，如['a','b'] column_list: 数据，如[[1,2],[3,4]] Returns: [{key:value}],如[['a':1,'b':2],['a':3,'b':4]] """ obj_data = [] for column in column_list: col_data = {} for index in range(len(key_list)): col_data[key_list[index]] = column[index] obj_data.append(col_data) return obj_data def dbs_execute(self, instance_name, sql_content, time_out=60): """ SIM环境执行insert、update、del数据库操作 Args: instance_name: 选择实例，例如sim126 sql_content: sql语句 time_out: 超时时间，默认60秒 """ self.__check_sql_content(sql_content) # sql格式检查 sql_info = sql_content.split(".")[0].split(" ") db_name = sql_info[-1].replace("`", "").replace(" ", "") if not self.session_id: self.dbs_login() try: # 检查SQL req_params = {"sql_content": sql_content, "instance_name": instance_name, "db_name": db_name, "sql_type": "online"} resp_check = self.req.post(self.check_url, req_params, headers=self.headers) if resp_check.status_code == 200: result_check = json.loads(resp_check.text) if result_check.get("data").get("CheckErrorCount") != 0: raise Exception("sql检查未通过，请检查调整后，再提交，msg： {}".format(result_check)) else: raise Exception("dbs执行检查sql失败，错误状态码： {}， res：{}".format(resp_check.status_code, resp_check.text)) # 提交SQL req_params = {"csrfmiddlewaretoken": self.authenticate_csrf_token, "sql_content": sql_content, "instance_name": instance_name, "db_name": db_name, "sql_type": "online", "sql-upload": "", "workflow_id": "", "workflow_name": "test", "demand_url": "", "group_name": "测试组", "run_date_start": "", "run_date_end": "", "workflow_auditors": 1} resp_commit = self.req.post(self.commit_url, req_params, headers=self.headers) if resp_commit.status_code == 200: workflow_id = resp_commit.history[0].headers.get("Location").split("/")[2] else: raise Exception("dbs执行提交sql失败，错误状态码： {}， res：{}".format(resp_commit.status_code, resp_commit.text)) # 执行sql req_params = {"csrfmiddlewaretoken": self.authenticate_csrf_token, "workflow_id": workflow_id, "mode": "auto"} resp_execute = self.req.post(self.execute_url, req_params, headers=self.headers) if resp_execute.status_code == 200: if not resp_execute.ok: raise Exception("sql执行未通过，请检查调整后，再提交，msg： {}".format(resp_execute.reason)) else: raise Exception("dbs执行sql失败，错误状态码： {}， res：{}".format(resp_execute.status_code, resp_execute.text)) # 查询执行状态 time_out = int(time_out) while time_out: time_out -= 1 resp_get_status = self.req.get(self.get_status_url + str(workflow_id), headers=self.headers) if resp_get_status.status_code == 200: result_status = json.loads(resp_get_status.text) if "Execute Successfully" in result_status.get("rows")[1].get("stagestatus"): return True else: time.sleep(1) else: time.sleep(1) raise Exception("超时未查询到结果，请确认sql是不是需要执行很久，若有必要请调整超时参数") except Exception as e: raise Exception(e) def dbs_query_by_db_name(self, instance_name, db_name, sql_content, limit_num="100"): """ dbs查询 | 请求参数名 | 说明 | 类型 | 是否必填 | | instance_name | 数据库实例: 例如sim-my-allschool | string | True | | db_name | 数据库名称: hulk_class_help | string | True | | schema_name | 数据库模式: 默认空不填 | string | True | | tb_name | 表名 | string | True | | sql_content | 查询sql | string | True | | limit_num | 查询行数 | string | True | """ if not self.session_id: self.dbs_login() data = {"instance_name": instance_name, "db_name": db_name, "schema_name": "", "tb_name": "", "sql_content": sql_content, "limit_num": limit_num} self.headers.update({"Referer": "https://dbs.qc.huohua.cn/sqlquery/"}) resp = self.req.post(self.query_url, data, headers=self.headers) result = json.loads(resp.text) return result.get('data').get('rows') def dbs_select(self, sql_content, r_type='json', instance_name=None): """ 功能：通过dbs查询并返回全部数据 Arg： sql: 查询语句 r_type: 返回类型，json或list instance_name: 数据库实例名，不传则根据env_choose文件中的环境配置来获取 Note: 1. 若想要返回一条数据，自行在sql语句中添加limit 1限制 2. 用于线上环境查询时，指定instance_name参数即可 """ self.__check_sql_content(sql_content) # sql格式检查 if not instance_name: instance_name = self.__get_instance_name(sql_content=sql_content) # 获取数据库实例名 if r_type == 'json': db_data = self.dbs_query_as_json(sql_content=sql_content, instance_name=instance_name) elif r_type == 'list': db_data = self.dbs_query_as_list(instance_name=instance_name, sql_content=sql_content) else: raise Exception("返回类型只能是json或list") # if 'limit 1;' in sql_content: # if len(db_data) == 0: # return [] # return db_data[0] # else: # return db_data return db_data def dbs_execute_sql(self, sql_content, instance_name=None, time_out=60): """ 通过DBS执行insert、update、del操作，支持concat拼接sql Args: sql_content: 待执行语句 instance_name: 数据库实例: env_choose中是qa或独立环境时，此参数失效，否则已输入为准，不传则默认为sim126 time_out: 超时时间，默认60秒 Returns: Note：可以根据你的查询表，直接指定sim环境的instance_name，因为qa环境时不使用此参数 """ if not instance_name: instance_name = self.__get_instance_name(sql_content=sql_content) # 自动识别环境 if "concat" in sql_content.lower(): # 如果是拼接sql，则拆分后执行 sql_list = self.dbs_query_as_json_auto_identify_env(sql_content=sql_content) for sql in sql_list: for key in sql: self.dbs_execute_sql(sql_content=sql[key]) else: self.dbs_execute(instance_name=instance_name, sql_content=sql_content, time_out=time_out) if __name__ == '__main__': hh_dbs = HuoHuaDBS() # hh_dbs.dbs_query_as_json("") sql = "SELECT ui.user_id,sp.id AS student_id,ui.user_name,ui.logo,ui.phone,ui.phone_code,ui.nick_name,ui.english_nickname,ui.share_code,ui.invite_code,ui.customer_id,ui.channel_id,ui.from_user_id,ui.email,ss.business_line FROM (SELECT up.id as user_id,up.user_name,up.logo,up.phone,up.phone_code,up.nick_name,up.english_nickname,up.share_code,up.invite_code,up.customer_id,up.channel_id,up.from_user_id,uc.contact_info as email FROM ucenter.user_profile up LEFT JOIN ucenter.user_contact uc ON up.id=uc.user_id WHERE up.phone_code='653666709542760459') ui LEFT JOIN peppa_channel.statistics_setting ss ON ui.channel_id = ss.channel_id LEFT JOIN ucenter.student_profile sp ON sp.user_id=ui.user_id;" print(hh_dbs.get_instance_name(sql_content=sql))