优化查询sql

src/core/manager/profile_manager.py

@@ -18,6 +18,13 @@ class ProfileManager(DatabaseDataManager):
 
     def __init__(self, client=None) -> None:
         super().__init__(client, create_logger(__name__, 'profile_manager.log'))
+        self.select_items_str = ', '.join([
+            'uid', 'country_code', 'channel_id_type', 'ffrom_login', 'user_preference_cate', 'consult_pay_money',
+            'listen_pay_money', 'test_items_pay_money', 'course_pay_money', 'consult_order_num',
+            'listen_order_num', 'test_items_order_num', 'course_order_num', 'aidi_cst_bias_city',
+            'aidi_cst_bias_sex', 'aidi_cst_bias_price', 'aidi_cst_bias_server_type', 'user_login_city',
+            'd30_inquire_order_num', 'd30_session_num',
+        ])
 
 
     def _make_query_sql(self, conditions=None):
@@ -25,7 +32,7 @@ class ProfileManager(DatabaseDataManager):
         if conditions:
             condition_sql = ' WHERE ' + ' AND '.join(conditions)
 
-        sql = 'SELECT * FROM ads.ads_register_user_profiles'
+        sql = 'SELECT {} FROM ads.ads_register_user_profiles'.format(self.select_items_str)
         sql += ' WHERE uid IN (SELECT DISTINCT uid FROM ods.ods_ydl_standard_order{})'.format(condition_sql)
         return sql
 

src/core/recommender.py

@@ -50,6 +50,13 @@ class UserCFRecommender(Recommender):
             c2c: [咨询师->咨询师] 索引方法，None 表示不使用咨询师拓展
         """
         super().__init__()
+        self.select_items_str = ', '.join([
+            'uid', 'country_code', 'channel_id_type', 'ffrom_login', 'user_preference_cate', 'consult_pay_money',
+            'listen_pay_money', 'test_items_pay_money', 'course_pay_money', 'consult_order_num',
+            'listen_order_num', 'test_items_order_num', 'course_order_num', 'aidi_cst_bias_city',
+            'aidi_cst_bias_sex', 'aidi_cst_bias_price', 'aidi_cst_bias_server_type', 'user_login_city',
+            'd30_inquire_order_num', 'd30_session_num',
+        ])
         # 召回 top_n 个相似用户
         self.top_n = top_n
         # 每个召回的用户取 k 个相关咨询师
@@ -100,7 +107,7 @@ class UserCFRecommender(Recommender):
         if user_id == '0':
             return []
             
-        sql = 'SELECT * FROM ads.ads_register_user_profiles'
+        sql = 'SELECT {} FROM ads.ads_register_user_profiles'.format(self.select_items_str)
         sql += ' WHERE uid={}'.format(user_id)
         try:
             _, all_data = self.client.query(sql)

src/data/mysql_client.py

 # -*- coding: utf-8 -*-
 
+import time
 import configparser
 
 import pymysql
@@ -30,10 +31,12 @@ class MySQLClient():
 
     def query(self, sql):
         # sql += ' limit 1000'
+        st = time.time()
         self.logger.debug('begin execute sql: %s', sql)
         row_count = self.cursor.execute(sql)
         data = self.cursor.fetchall()
-        self.logger.debug('fetch row count: %s', row_count)
+        cost_time = time.time() - st
+        self.logger.debug('fetch row count: %s, cost time %s', row_count, cost_time)
         return row_count, data
 
 
@@ -96,12 +99,14 @@ class MySQLClientPool():
 
     def query(self, sql, args=None):
         # sql += ' limit 1000'
+        st = time.time()
         self.logger.debug('begin execute sql: %s', sql)
         conn, cur = self.open()
         row_count = cur.execute(sql, args)
         data = cur.fetchall()
         self.close(conn, cur)
-        self.logger.debug('fetch row count: %s', row_count)
+        cost_time = time.time() - st
+        self.logger.debug('fetch row count: %s, cost time %s', row_count, cost_time)
         return row_count, data
 
     @classmethod