支持BFI的每日净值计算

modules/navCalculator.dos

@@ -52,6 +52,66 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
 }
 
 /*
+ *   根据持仓收益计算组合净值
+ * 
+ *   @param entity_cal_dates <TABLE>: 组合净值计算时间区间表，记录 [COLUMNS] entity_id, first_cal_date, latest_cal_date
+ *   @parm holdings <TABLE>：带有各证券净值前值的截面持仓表 [COLUMNS] entity_id, price_date, sec_id, ret, weight
+ *   
+ *   @return <TABLE>: [COLUMNS] entity_id, price_date, ret
+ */
+def cal_nav_by_return(entity_type, entity_cal_dates, holdings) {
+// entity_type = 'FA'
+// entity_cal_dates=t_factor
+// holdings = t
+
+    // 组合收益计算: RET = ∑( weight_i * ret_i )
+    tb_portfolio_ret = SELECT entity_id, price_date, (weight * ret).sum() AS ret
+                       FROM holdings
+                       GROUP BY entity_id, price_date;
+
+    // 取组合净值前值
+    s_json = (SELECT entity_id, price_date.max() AS price_date
+              FROM ej(tb_portfolio_ret, entity_cal_dates, 'entity_id')
+              WHERE tb_portfolio_ret.price_date < entity_cal_dates.first_cal_date
+              GROUP BY entity_id).toStdJson();
+
+    tb_pre_nav = get_entity_nav_by_date(entity_type, s_json, true);
+
+    INSERT INTO tb_pre_nav
+    	SELECT entity_id, first_cal_date, NULL
+    	FROM entity_cal_dates
+    	WHERE NOT exists( SELECT * FROM tb_pre_nav WHERE tb_pre_nav.entity_id = entity_cal_dates.entity_id);
+   
+    tb_portfolio_ret.addColumn('nav', DOUBLE);
+
+    // start_cal_date 是最早净值日期
+    UPDATE tb_portfolio_ret
+        SET nav = 1, ret = 0
+    FROM ej(tb_portfolio_ret, ej(entity_cal_dates, tb_pre_nav, 'entity_id'), ['entity_id', 'price_date'], ['entity_id', 'first_cal_date'])
+    WHERE tb_pre_nav.cumulative_nav IS NULL;
+
+    // start_cal_date 是最早净值日期，用它作为初始净值日期
+    UPDATE tb_pre_nav
+        SET price_date = first_cal_date, cumulative_nav = 1
+    FROM ej(tb_pre_nav, entity_cal_dates, 'entity_id')
+    WHERE cumulative_nav IS NULL;
+
+    tb_portfolio_ret.sortBy!(['entity_id', 'price_date'], [1, 1]);
+
+    // 通过收益反算净值: nav_i = nav_0 * ∏(1 + ret_i)
+    UPDATE tb_portfolio_ret 
+        SET nav = (tb_pre_nav.cumulative_nav * (1+ret).cumprod()).round(6) 
+    FROM ej(tb_portfolio_ret, tb_pre_nav, 'entity_id')
+    CONTEXT BY entity_id;
+
+    // 返回有用的数据 
+    return (SELECT DISTINCT tb_portfolio_ret.* 
+            FROM ej(tb_portfolio_ret, entity_cal_dates, 'entity_id')
+            WHERE price_date >= first_cal_date AND price_date <= latest_cal_date
+            ORDER BY entity_id, price_date);
+}
+
+/*
  *  计算FOF类组合净值
  *  NOTE: 与MySQL逻辑一致，用户界面输入的交易净值会被暂时忽略，因为我们无法确保同一基金同一时间被输入的净值是相同的；
  *        忽略手工净值会导致收益不精确或无法计算的问题，但可能错误的净值将导致错误的结果，两害取其轻。
@@ -143,7 +203,7 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
     tb_holdings.addColumn(['ret', 'shares', 'market_value', 'total_mkt_value', 'weight'], [DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
 
     // 计算各持仓证券收益
-    UPDATE tb_holdings SET ret = (cumulative_nav.ratios()-1).round(7)
+    UPDATE tb_holdings SET ret = cumulative_nav.ratios()-1
     CONTEXT BY portfolio_id, sec_id;
 
     // 把交易日截面的份额数用于组合收益表
@@ -175,49 +235,15 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
         SET total_mkt_value = market_value.sum()
     CONTEXT BY portfolio_id, price_date;
 
-
     // 计算各持仓的权重
     UPDATE tb_holdings
         SET weight = (market_value \ total_mkt_value).round(6)
     WHERE total_mkt_value <> 0;
 
-    // 组合收益计算: RET = ∑( weight_i * ret_i )
-    tb_portfolio_ret = SELECT portfolio_id, price_date, (weight * ret).sum().round(7) AS ret
-                       FROM tb_holdings
-                       GROUP BY portfolio_id, price_date;
-
-    // 取组合净值前值
-    s_json = (SELECT portfolio_id, price_date.max() AS price_date
-              FROM ej(tb_portfolio_ret, tb_port_first_cal_date, 'portfolio_id')
-              WHERE tb_portfolio_ret.price_date < tb_port_first_cal_date.first_cal_date
-              GROUP BY portfolio_id).toStdJson();
-    tb_pre_nav = get_portfolio_nav_by_date(s_json, true);
-    
-    tb_portfolio_ret.addColumn('nav', DOUBLE);
-
-    // start_cal_date 是最早净值日期
-    UPDATE tb_portfolio_ret
-        SET nav = 1, ret = 0
-    FROM ej(tb_portfolio_ret, ej(tb_port_first_cal_date, tb_pre_nav, 'portfolio_id'), ['portfolio_id', 'price_date'], ['portfolio_id', 'first_cal_date'])
-    WHERE tb_pre_nav.cumulative_nav IS NULL;
-
-    // start_cal_date 是最早净值日期，用它作为初始净值日期
-    UPDATE tb_pre_nav
-        SET price_date = first_cal_date, cumulative_nav = 1
-    FROM ej(tb_pre_nav, tb_port_first_cal_date, 'portfolio_id')
-    WHERE cumulative_nav IS NULL;
+	// 通过持仓收益反算组合收益，再计算组合净值
+	tb_port_first_cal_date.rename!('portfolio_id', 'entity_id');
+	tb_holdings.rename!('portfolio_id', 'entity_id');
 
-    tb_portfolio_ret.sortBy!(['portfolio_id', 'price_date'], [1, 1]);
+	return cal_nav_by_return('PF', tb_port_first_cal_date, tb_holdings);
 
-    // 通过收益反算净值: nav_i = nav_0 * ∏(1 + ret_i)
-    UPDATE tb_portfolio_ret 
-        SET nav = (tb_pre_nav.cumulative_nav * (1+ret).cumprod()).round(6) 
-    FROM ej(tb_portfolio_ret, tb_pre_nav, 'portfolio_id')
-    CONTEXT BY portfolio_id;
-
-    // 返回有用的数据 
-    return (SELECT DISTINCT tb_portfolio_ret.* 
-            FROM ej(tb_portfolio_ret, tb_port_first_cal_date, 'portfolio_id')
-            WHERE price_date >= first_cal_date AND price_date <= latest_cal_date
-            ORDER BY portfolio_id, price_date);
  }

modules/operationDataPuller.dos

@@ -114,7 +114,8 @@ def get_portfolio_info(portfolio_ids) {
     
     s_entity_sql = iif(s_entity_ids == NULL || s_entity_ids == '', '', " AND cpm.id IN (" + s_entity_ids + ")");
 
-    s_query = "SELECT cpm.id AS portfolio_id, cpm.userid, cpm.customer_id, cpm.inception_date, 1 AS ini_value, cpm.portfolio_source, cpm.portfolio_type, 102 AS strategy, sub_type AS substrategy
+    s_query = "SELECT cpm.id AS portfolio_id, cpm.userid, cpm.customer_id, cpm.inception_date, 1 AS ini_value, 
+                      cpm.portfolio_source, cpm.portfolio_type, 102 AS strategy, sub_type AS substrategy, u.org_id
                FROM pfdb.`pf_customer_portfolio_map` cpm
                INNER JOIN pfdb.cm_user u ON cpm.userid = u.userid
                WHERE cpm.isvalid = 1

modules/performanceDataPuller.dos

@@ -584,34 +584,121 @@ def get_portfolio_list_by_fund_nav_updatetime(portfolio_ids, updatetime, isFromM
     return t
 }
 
+/*
+ *   根据指数净值更新日期，取受影响的BFI因子列表
+ * 
+ *   Example: get_bfi_factor_list_by_index_nav_updatetime(['FA00000VMH','FA00000VMK'], 2024.11.08, true);
+ *            get_bfi_factor_list_by_index_nav_updatetime(NULL, NULL, true);
+ */
+def get_bfi_factor_list_by_index_nav_updatetime(factor_ids, updatetime,  isFromMySQL) {
+
+    t = null;
+
+    s_entity_ids = ids_to_string(factor_ids);
+    sql_entity_id = iif(s_entity_ids == NULL || s_entity_ids == '', '', " AND fi.factor_id IN (" + s_entity_ids + ")");
+
+    sql_updatetime = iif(updatetime == NULL, '', " AND mi.updatetime > '" + updatetime + "'");
+
+    if(isFromMySQL == true) {
+
+        s_query = "SELECT fi.factor_id, GREATEST(MIN(mi.price_date), fi.inception_date) AS first_cal_date, LEAST(MAX(mi.price_date), CURRENT_DATE) AS latest_cal_date 
+				   FROM pfdb.cm_factor_information fi
+				   INNER JOIN pfdb.cm_factor_index_map map ON fi.factor_id = map.factor_id
+				   INNER JOIN mfdb.market_indexes mi ON map.index_id = mi.index_id
+				   WHERE fi.isvalid = 1
+				     AND fi.factor_type = 5" +  // 5: bfi factor
+				     sql_entity_id + "
+				     AND map.isvalid = 1
+				     AND mi.isvalid = 1
+				     AND mi.close > 0 " +
+				     sql_updatetime + "
+				   GROUP BY fi.factor_id";
+
+	    conn = connect_mysql();
+	
+	    t = odbc::query(conn, s_query);
+	
+	    conn.close();
+
+    }
+
+    return t
+}
 
 /*
- *  取Json中指定的组合当日净值
+ *   取固定权重的组合持仓，包括BFI因子，日再平衡组合等
  * 
- *  @param s_json <JSON>
- *  
- *  Example: get_portfolio_nav_by_date([{"portfolio_id": 166002,"price_date": "2024.10.25"},{"portfolio_id": 166114,"price_date": "2024.03.13"}], true);
+ *   Example: get_fixed_weight_portfolio_holding('PF', [145589, 170904]);
+ *            get_fixed_weight_portfolio_holding('FA', ['FA00000VMH', 'FA00000VMI']);
  */
-def get_portfolio_nav_by_date(s_json, isFromMySQL) {
+def get_fixed_weight_portfolio_holding(entity_type, entity_ids) {
 
     t = null;
 
+    s_query = '';
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+	if(entity_type == 'FA') {
+
+	    sql_entity_id = iif(s_entity_ids == NULL || s_entity_ids == '', '', " AND factor_id IN (" + s_entity_ids + ")");
+	
+	    s_query = "SELECT factor_id AS entity_id, holding_date, index_id AS sec_id, weight, 2 AS entity_type
+				   FROM pfdb.cm_factor_index_map
+				   WHERE isvalid = 1" +
+				     sql_entity_id;
+
+	} else if(entity_type == 'PF') {
+
+	    sql_entity_id = iif(s_entity_ids == NULL || s_entity_ids == '', '', " AND portfolio_id IN (" + s_entity_ids + ")");
+	
+	    s_query = "SELECT portfolio_id AS entity_id, effective_date AS holding_date, entity_id AS sec_id, weight, entity_type
+				   FROM pfdb.pf_portfolio_rebalance_weights
+				   WHERE isvalid = 1" +
+				     sql_entity_id;
+		
+	}
+
+	if( s_query == '') return t;
+
+    conn = connect_mysql();
+
+    t = odbc::query(conn, s_query);
+
+    conn.close();
+
+    return t;
+}
+
+/*
+ *  通用取Json中指定的证券当日净值
+ * 
+ *
+ *  Example: get_entity_nav_by_date('PF', '[{"entity_id": 166002,"price_date": "2024.10.25"},{"entity_id": 166114,"price_date": "2024.03.13"}]', true);
+ */
+def get_entity_nav_by_date(entity_type, s_json, isFromMySQL=true) {
+
+    t = null;
+
+	desc = get_nav_table_description(entity_type)[0];
+
     if(isFromMySQL == true) {
 
-        s_query = "SELECT t.portfolio_id, t.price_date, nav.cumulative_nav
+        s_query = "SELECT t.entity_id, t.price_date, nav." + desc.nav_col + " AS cumulative_nav
                    FROM JSON_TABLE ( '" + s_json + "', '$[*]'
-                           COLUMNS ( portfolio_id INT PATH '$.portfolio_id',
+                           COLUMNS ( entity_id " + iif(entity_type=='PF', 'INT', 'VARCHAR(10)') + " PATH '$.entity_id',
                                      price_date DATE PATH '$.price_date' ) ) t
-                   LEFT JOIN pfdb.pf_portfolio_nav nav ON t.portfolio_id = nav.portfolio_id AND t.price_date = nav.price_date;";
+                   LEFT JOIN " + desc.table_name + " nav ON t.entity_id = nav." + desc.sec_id_col + " AND t.price_date = nav.price_date;";
 
         conn = connect_mysql();
-     
+
         t = odbc::query(conn, s_query);
      
         conn.close();
     }
 
     return t;
+	
 }
 
 /*

modules/task_portfolioPerformance.dos

@@ -1,5 +1,6 @@
 module fundit::task_portfolioPerformance
 
+use fundit::sqlUtilities;
 use fundit::operationDataPuller;
 use fundit::performanceDataPuller;
 use fundit::portfolioDataPuller;
@@ -66,7 +67,7 @@ def calPortfolioPerformance(navs) {
  * 
  *   TODO: release 时改变同步目标表为正式表
  */
-def cal_and_save_portfolio_nav(cal_portfolio_info) {
+def cal_and_save_portfolio_nav(cal_portfolio_info, is_save_local) {
 
     rt = '';
 
@@ -104,6 +105,11 @@ def cal_and_save_portfolio_nav(cal_portfolio_info) {
             tb_portfolio_nav.rename!('entity_id', 'portfolio_id');
             save_and_sync(tb_portfolio_nav, 'raw_db.pf_portfolio_nav', 'raw_db.pf_portfolio_nav');
 
+            // 数据初始化时将指标存入本地
+            if(is_save_local == true) {
+            	save_table(tb_portfolio_nav, 'pfdb.pf_portfolio_nav', false);
+            }
+
         } catch(ex) {
 
             //TODO: Log errors
@@ -119,7 +125,7 @@ def cal_and_save_portfolio_nav(cal_portfolio_info) {
  * 
  *  TODO: release 时改变同步目标表为正式表
  */
-def cal_and_save_portfolio_indicators(cal_portfolio_info) {
+def cal_and_save_portfolio_indicators(cal_portfolio_info, is_save_local) {
 
     rt = '';
 
@@ -219,6 +225,17 @@ def cal_and_save_portfolio_indicators(cal_portfolio_info) {
 
             save_and_sync(tb_portfolio_latest_performance, 'raw_db.pf_portfolio_latest_performance', 'raw_db.pf_portfolio_latest_performance');
 
+            // 数据初始化时将指标存入本地
+            if(is_save_local == true) {
+            	save_table(tb_portfolio_performance, 'pfdb.pf_portfolio_performance', false);
+            	save_table(tb_portfolio_indicator, 'pfdb.pf_portfolio_indicator', false);
+            	save_table(tb_portfolio_risk_stats, 'pfdb.pf_portfolio_risk_stats', false);
+            	save_table(tb_portfolio_riskadjret_stats, 'pfdb.pf_portfolio_riskadjret_stats', false);
+            	save_table(tb_portfolio_style_stats, 'pfdb.pf_portfolio_style_stats', false);
+	           	save_table(tb_portfolio_performance_weekly, 'pfdb.pf_portfolio_performance_weekly', false);
+            	save_table(tb_portfolio_latest_performance, 'pfdb.pf_portfolio_latest_performance', false);	
+            }
+
         } catch(ex) {
 
             //TODO: Log errors
@@ -234,22 +251,85 @@ def cal_and_save_portfolio_indicators(cal_portfolio_info) {
 /*
  *   [定时任务]批量计算组合净值、收益及指标
  *   
- *   @param updatetime <DATETIME>: 持仓证券净值更新时间，忽略时跑全历史
+ *   @param updatetime <DATETIME>: 持仓证券净值更新时间，忽略或传入1989.01.01及更早的日期被认为在做数据初始化
  * 
  * 
  *   Example: CalPortfolioPerformanceTask(2024.10.28);
+ *            CalPortfolioPerformanceTask(1989.01.01);  -- 【初始化专用】 (45min)
  */
-def CalPortfolioPerformanceTask(updatetime=1900.01.01) {
+def CalPortfolioPerformanceTask(updatetime) {
 
     rt = '';
     // 3 min
     tb_cal_ports = get_portfolio_list_by_fund_nav_updatetime(NULL, updatetime, true);
 
     if(tb_cal_ports.isVoid() || tb_cal_ports.size() == 0) return;
+
+    is_save_local = iif(updatetime <= get_ini_data_const()['date'], true, false);
+
     // 26 min
-    rt = cal_and_save_portfolio_nav(tb_cal_ports);
+    rt = cal_and_save_portfolio_nav(tb_cal_ports, is_save_local);
     // 9 min
-    rt = rt + '; ' + cal_and_save_portfolio_indicators(tb_cal_ports);
+    rt = rt + '; ' + cal_and_save_portfolio_indicators(tb_cal_ports, is_save_local);
 
     return rt;
 }
+
+/*
+ *   批量计算BFI因子净值
+ * 
+ *   Example: cal_and_save_factor_nav(2024.11.15, false);
+ *            cal_and_save_factor_nav(1989.01.01, true);
+ */
+def cal_and_save_factor_nav(updatetime, is_save_local) {
+
+	ret = ''
+
+	// 根据成分指数净值更新日期，取有影响的因子
+	tb_cal_factors = get_bfi_factor_list_by_index_nav_updatetime(NULL, updatetime, true);
+
+    if(tb_cal_factors.isVoid() || tb_cal_factors.size() == 0) return;
+
+	t_factor_value = table(100:0, ['factor_id', 'price_date', 'factor_value'], [SYMBOL, DATE, DOUBLE]);
+
+	// 因子个数有限，用循环更简便
+	for(factor in tb_cal_factors) {
+
+        v_factor_id = array(STRING, 0).append!(factor.factor_id);
+		// 取因子成分指数
+		tb_holdings = get_fixed_weight_portfolio_holding('FA', v_factor_id);
+
+		UPDATE tb_holdings SET first_cal_date = first_cal_date, latest_cal_date = latest_cal_date
+		FROM ej(tb_holdings, tb_cal_factors, 'entity_id', 'factor_id');
+
+		s_json = (SELECT sec_id, first_cal_date.min() AS price_date FROM tb_holdings GROUP BY sec_id).toStdJson();
+
+		// 取含前值的成分指数点位
+		tb_nav = get_nav_for_return_calculation('MI', 'd', s_json).sortBy!(['sec_id', 'price_date'], [1, 1]);
+
+		// 计算每期收益
+		UPDATE tb_nav SET ret = cumulative_nav.ratios() - 1 CONTEXT BY sec_id;
+
+		t = SELECT h.entity_id, n.price_date, h.sec_id, n.ret, h.weight/100 AS weight
+		    FROM tb_holdings AS h
+		    INNER JOIN tb_nav AS n ON h.sec_id = n.sec_id
+	     	ORDER BY h.entity_id, h.sec_id, n.price_date;
+
+		t_factor = SELECT factor_id AS entity_id, first_cal_date, latest_cal_date FROM tb_cal_factors WHERE factor_id = factor.factor_id;
+
+		t_tmp = cal_nav_by_return('FA', t_factor, t);
+
+		if(!t_tmp.isVoid() && t_tmp.size() > 0) {
+	    	INSERT INTO t_factor_value 
+	        	SELECT entity_id AS factor_id, price_date, nav AS factor_value FROM cal_nav_by_return('FA', t_factor, t);
+		}
+	}
+
+	if(! t_factor_value.isVoid() && t_factor_value.size() > 0) {
+		save_and_sync(t_factor_value, 'raw_db.cm_factor_value', 'raw_db.cm_factor_value');
+
+		if(is_save_local == true) {
+			save_table(t_factor_value, 'pfdb.cm_factor_value', false);
+		}
+	}
+}