上传文件至 'modules'

modules/dataPuller.dos

@@ -98,9 +98,16 @@ def get_portfolio_weekly_rets(portfolio_ids, start_date, end_date, isFromMySQL)
 }
 
 /*
- *  取月收益
+ *  通用取月收益
+ *  
+ *  @param entity_type <STRING>:
+ *  @param entity_ids <VECTOR|STRING>:
+ *  @param start_date <DATE>:
+ *  @param end_date <DATE>:
+ *  @param isFromMySQL <BOOL>:
+ * 
  * 
- *  Example: get_monthly_ret('FD', ['HF000004KN','HF000103EU','HF00018WXG'], 2000.01.01, 2024.03.01, true);
+ *  Example: get_monthly_ret('HF', ['HF000004KN','HF000103EU','HF00018WXG'], 2000.01.01, 2024.03.01, true);
  */
 def get_monthly_ret(entity_type, entity_ids, start_date, end_date, isFromMySQL) {
 
@@ -115,7 +122,7 @@ def get_monthly_ret(entity_type, entity_ids, start_date, end_date, isFromMySQL)
 
     if(isFromMySQL == true) {
 
-        s_query = "SELECT " + tmp.sec_id_col[0] + ", end_date, price_date, ret_1m AS ret, " + tmp.cumulative_nav_col[0] + " AS nav, ret_ytd_a, ret_incep_a
+        s_query = "SELECT " + tmp.sec_id_col[0] + " AS entity_id, end_date, price_date, ret_1m AS ret, " + tmp.cumulative_nav_col[0] + " AS nav
                    FROM " + tmp.table_name[0] + "
                    WHERE " + tmp.sec_id_col[0] + " IN (" + s_entity_ids + ")
                       AND isvalid = 1
@@ -133,7 +140,7 @@ def get_monthly_ret(entity_type, entity_ids, start_date, end_date, isFromMySQL)
         tb_local = load_table_from_local("fundit", tmp.table_name[0])
 
         s_col = (sqlCol(tmp.sec_id_col[0]), sqlCol("end_date"), sqlColAlias(<ret_1m>, "ret"), sqlColAlias(<cumulative_nav>, "nav"), sqlCol("ret_ytd_a"), sqlCol("ret_incep_a"))
-        // TODO: how to make the "fund_id" dynamicly decided by tmp.sec_id_col[0]?
+        // TODO: how to make the "fund_id" dynamicly decided by tmp.sec_id_col[0], then rename to "entity_id"?
         s_where = expr(<fund_id>, in, s_entity_ids.strReplace("'", "").split(","))
         
         t = sql(s_col, tb_local, s_where).eval()
@@ -143,18 +150,6 @@ def get_monthly_ret(entity_type, entity_ids, start_date, end_date, isFromMySQL)
     return t
 }
 
-
-/*
- *  取公私募基金月收益
- *
- *  get_fund_monthly_ret("'MF00003PW1','MF00003PW1'", 1990.01.01, today(), true)
- */
-
-def get_fund_monthly_ret(fund_ids, start_date, end_date, isFromMySQL) {
-
-    return get_monthly_ret('FD', fund_ids, start_date, end_date, isFromMySQL);
-}
-
 /*
  * 取无风险月度利率
  *
@@ -162,7 +157,7 @@ def get_fund_monthly_ret(fund_ids, start_date, end_date, isFromMySQL) {
  */
 def get_risk_free_rate(start_date, end_date) {
     
-    return get_monthly_ret('IX', "'IN0000000M'", start_date, end_date, true);
+    return get_monthly_ret('MI', "'IN0000000M'", start_date, end_date, true);
 }
 
 
@@ -204,7 +199,7 @@ def get_fund_latest_nav_performance(fund_ids, isFromMySQL) {
 }
 
 /*
- * 取基金净值
+ * 通用取净值
  * 
  * 
  * Create: 202408                                                    Joey
@@ -212,7 +207,7 @@ def get_fund_latest_nav_performance(fund_ids, isFromMySQL) {
  * 
  *
  * Example: get_nav_by_price_date('HF', "'HF000004KN','HF00018WXG'", 2024.05.01, true);
- *          get_nav_by_price_date('IN', "'IN00000008','IN0000000M'", 2024.05.01, true);
+ *          get_nav_by_price_date('MI', "'IN00000008','IN0000000M'", 2024.05.01, true);
  */
 def get_nav_by_price_date(entity_type, entity_ids, price_date, isFromMySQL) {
 
@@ -226,7 +221,7 @@ def get_nav_by_price_date(entity_type, entity_ids, price_date, isFromMySQL) {
 
         nav_table_name = tmp.table_name[0];
     
-        s_query = "SELECT " + tmp.sec_id_col[0] + ", price_date, " + tmp.cumulative_nav_col[0] + ", " + tmp.nav_col[0] + "
+        s_query = "SELECT " + tmp.sec_id_col[0] + " AS entity_id, price_date, " + tmp.cumulative_nav_col[0] + " AS cumulative_nav, " + tmp.nav_col[0] + " AS nav
                    FROM " + tmp.table_name[0] + "
                    WHERE " + tmp.sec_id_col[0] + " IN (" + s_entity_ids + ")
                      AND isvalid = 1
@@ -360,7 +355,7 @@ def get_portfolio_info(portfolio_ids) {
 /*
  *   取基金组合基础有效信息
  * 
- *   Example: get_entity_info('FD', ['HF000004KN','HF000103EU','HF00018WXG']);
+ *   Example: get_entity_info('HF', ['HF000004KN','HF000103EU','HF00018WXG']);
  *            get_entity_info('PF', '166002,166114');
  */
 def get_entity_info(entity_type, entity_ids) {
@@ -371,7 +366,7 @@ def get_entity_info(entity_type, entity_ids) {
 
     if(s_entity_ids == null || s_entity_ids == '') return null;
 
-    if(entity_type == 'FD') {
+    if(entity_type == 'MF' || entity_type == 'HF') {
 
     	t = get_fund_info(s_entity_ids);
 
@@ -389,52 +384,59 @@ def get_entity_info(entity_type, entity_ids) {
 
 
 /*
- * 取基金净值更新信息, 返回基金及其净值更新的最早净值日期
+ *  取某时间后更新的各基金组合最早净值日期
  *
- * @param fund_ids: fund_id STRING VECTOR
- * @param update_time: all updates after this time
+ *  @param entity_type <STRING>: MF, HF, EQ, CF, MI, TI, CI, FA, PF
+ *  @param entity_ids <VECTOR|STRING>: NULL时取全量
+ *  @param update_time <DATETIME>: all updates after this time
+ *  @param isFromMySQL <BOOL>:
  *
- * Example: get_fund_list_by_nav_updatetime('MF', null, 2024.09.26);
- *          get_fund_list_by_nav_updatetime('HF', null, 2024.07.19T10:00:00)
+ *  Example: get_entity_list_by_nav_updatetime('MF', ['MF00003PW1', 'MF00003PW2'], 2024.09.26, true);
+ *           get_entity_list_by_nav_updatetime('HF', null, 2024.07.19T10:00:00, true);
+ *           get_entity_list_by_nav_updatetime('PF', '166002,166114', 2024.06.20, true);
  * 
  */
-def get_fund_list_by_nav_updatetime(entity_type, fund_ids, updatetime) {
+def get_entity_list_by_nav_updatetime(entity_type, entity_ids, updatetime, isFromMySQL) {
 
-    s_fund_sql = '';
-    // 这里要用 isVoid， 因为 isNull对向量返回的是布尔向量
-    if (! isVoid(fund_ids)){
-        s_fund_ids = fund_ids.concat("','");
-        s_fund_sql = " AND fi.fund_id IN ('" + s_fund_ids + "')";
-    }
+    tmp = get_nav_table_description(entity_type);
 
-    if(entity_type == 'HF') {
-    	nav_table = 'mfdb.nav';
-    } else {
-    	nav_table = 'mfdb.public_nav';
+    s_entity_ids = ids_to_string(entity_ids);
+
+    sql_entity_id = '';
+
+    if(s_entity_ids != NULL) {
+    	sql_entity_id = " AND " + tmp.sec_id_col[0] + " IN (" + s_entity_ids + ")";
     }
-    
-    s_query = "SELECT fi.fund_id, MIN(nav.price_date) AS price_date,
-                      fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value
-               FROM mfdb.fund_information fi
-               INNER JOIN " + nav_table + " nav ON fi.fund_id = nav.fund_id
-               WHERE fi.isvalid = 1" +
-                 s_fund_sql + "
-                 AND nav.cumulative_nav > 0
-                 AND nav.updatetime >= '" + updatetime + "'
-               GROUP BY fi.fund_id
-               ORDER BY fi.fund_id"
 
-    conn = connect_mysql()
+    if(isFromMySQL == true) {
 
-    t = odbc::query(conn, s_query)
+        nav_table_name = tmp.table_name[0];
+    
+        s_query = "SELECT " + tmp.sec_id_col[0] + " AS entity_id, MIN(price_date) AS price_date 
+                   FROM " + tmp.table_name[0] + "
+                   WHERE isvalid = 1 " +
+                     sql_entity_id + "
+                     AND " + tmp.cumulative_nav_col[0] + " > 0
+                     AND updatetime >= '" + updatetime$STRING + "'
+                   GROUP BY " + tmp.sec_id_col[0] + "
+                   ORDER BY " + tmp.sec_id_col[0] + ", price_date";
+    
+        conn = connect_mysql();
+    
+        t = odbc::query(conn, s_query);
+    
+        conn.close();
 
-    conn.close()
+    } else {
+    	//TODO
+    }
 
     return t
 
 }
 
 
+
 /*
  * 取私募基金用于月末 fund_performance 表更新的净值
  * 
@@ -518,7 +520,7 @@ def get_portfolio_primary_benchmark(portfolio_ids, month_start, month_end) {
  *  取某时间段的基金组合主基准
  *
  * 
- *  Example: get_entity_primary_benchmark('FD', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ *  Example: get_entity_primary_benchmark('MF', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
  *           get_entity_primary_benchmark('PF', [166002,166114], '1990-01', '2024-08');
  */
 def get_entity_primary_benchmark(entity_type, entity_ids, month_start, month_end) {
@@ -529,7 +531,7 @@ def get_entity_primary_benchmark(entity_type, entity_ids, month_start, month_end
 
     if(s_entity_ids == null || s_entity_ids == '') return null;
 
-    if(entity_type == 'FD') {
+    if(entity_type == 'MF' || entity_type == 'HF') {
 
     	t = get_fund_primary_benchmark(s_entity_ids, month_start, month_end);
 
@@ -602,7 +604,7 @@ def get_portfolio_bfi_factors(portfolio_ids, month_start, month_end) {
  *  取某时间段的基金组合BFI基准
  *
  * 
- *  Example: get_entity_bfi_factors('FD', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ *  Example: get_entity_bfi_factors('MF', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
  *           get_entity_bfi_factors('PF', [166002,166114], '1990-01', '2024-08');
  */
 def get_entity_bfi_factors(entity_type, entity_ids, month_start, month_end) {
@@ -613,7 +615,7 @@ def get_entity_bfi_factors(entity_type, entity_ids, month_start, month_end) {
 
     if(s_entity_ids == null || s_entity_ids == '') return null;
 
-    if(entity_type == 'FD') {
+    if(entity_type == 'MF' || entity_type == 'HF') {
 
     	t = get_fund_bfi_factors(s_entity_ids, month_start, month_end);
 
@@ -701,10 +703,10 @@ def get_benchmark_return(benchmarks, end_day) {
 
 
     // 目前指数的月度业绩存在 fund_performance 表
-    t_bmk = SELECT fund_id AS benchmark_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('IX', s_index_ids, 1990.01.01, end_day, true);
+    t_bmk = SELECT entity_id AS benchmark_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('MI', s_index_ids, 1990.01.01, end_day, true);
 
     // 而因子的月度业绩存在 cm_factor_performance 表
-    INSERT INTO t_bmk SELECT factor_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('FA', s_factor_ids, 1990.01.01, end_day, true);
+    INSERT INTO t_bmk SELECT entity_id AS factor_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('FA', s_factor_ids, 1990.01.01, end_day, true);
 
 	return t_bmk;
 }