支持基金BFI计算

modules/dataSaver.dos

@@ -223,6 +223,53 @@ def create_entity_style_stats(is_id_integer=false) {
 }
 
 /*
+ *   建表 XXX_bfi_bm_indicator
+ */
+def create_entity_bfi_indicator(is_id_integer=false) {
+
+	return table(1000:0,
+			   ['entity_id', 'end_date', 'factor_id',
+                'upsidecapture_ret_6m', 'downsidecapture_ret_6m', 'upsidecapture_ratio_6m', 'downsidecapture_ratio_6m',
+                'alpha_6m', 'winrate_6m', 'beta_6m', 'info_ratio_6m', 'tracking_error_6m', 'jensen_6m',
+                'upsidecapture_ret_1y', 'downsidecapture_ret_1y', 'upsidecapture_ratio_1y', 'downsidecapture_ratio_1y',
+                'alpha_1y', 'winrate_1y', 'beta_1y', 'info_ratio_1y', 'tracking_error_1y', 'jensen_1y',
+                'upsidecapture_ret_2y', 'downsidecapture_ret_2y', 'upsidecapture_ratio_2y', 'downsidecapture_ratio_2y',
+                'alpha_2y', 'winrate_2y', 'beta_2y', 'info_ratio_2y', 'tracking_error_2y', 'jensen_2y',
+                'upsidecapture_ret_3y', 'downsidecapture_ret_3y', 'upsidecapture_ratio_3y', 'downsidecapture_ratio_3y',
+                'alpha_3y', 'winrate_3y', 'beta_3y', 'info_ratio_3y', 'tracking_error_3y', 'jensen_3y',
+                'upsidecapture_ret_4y', 'downsidecapture_ret_4y', 'upsidecapture_ratio_4y', 'downsidecapture_ratio_4y',
+                'alpha_4y', 'winrate_4y', 'beta_4y', 'info_ratio_4y', 'tracking_error_4y', 'jensen_4y',
+                'upsidecapture_ret_5y', 'downsidecapture_ret_5y', 'upsidecapture_ratio_5y', 'downsidecapture_ratio_5y',
+                'alpha_5y', 'winrate_5y', 'beta_5y', 'info_ratio_5y', 'tracking_error_5y', 'jensen_5y',
+                'upsidecapture_ret_10y', 'downsidecapture_ret_10y', 'upsidecapture_ratio_10y', 'downsidecapture_ratio_10y',
+                'alpha_10y', 'winrate_10y', 'beta_10y', 'info_ratio_10y', 'tracking_error_10y', 'jensen_10y',
+                'upsidecapture_ret_ytd', 'downsidecapture_ret_ytd', 'upsidecapture_ratio_ytd', 'downsidecapture_ratio_ytd',
+                'alpha_ytd', 'winrate_ytd', 'beta_ytd', 'info_ratio_ytd', 'tracking_error_ytd', 'jensen_ytd',
+                'upsidecapture_ret_incep', 'downsidecapture_ret_incep', 'upsidecapture_ratio_incep', 'downsidecapture_ratio_incep',
+                'alpha_incep', 'winrate_incep', 'beta_incep', 'info_ratio_incep', 'tracking_error_incep', 'jensen_incep'],
+			   [iif(is_id_integer, INT, SYMBOL), MONTH, SYMBOL,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, 
+				DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE] );
+}
+
+
+/*
  *  建表 xxx_performance_weekly
  */
 def create_entity_performance_weekly(is_id_integer=false) {
@@ -668,6 +715,59 @@ def generate_entity_style_stats(entity_info, indicators, isToMySQL, mutable enti
 
 
 /*
+ *   按照 XXX_bfi_bm_indicator 表结构准备数据记录
+ * 
+ *   TODO: why we need isToMySQL here?
+ *         其它的指标恐怕也要按这个改，因为私募可能会有近6月没有数据但近2年之类的周期有数据的情况！
+ */
+def generate_entity_bfi_indicator(entity_info, indicators, isToMySQL, mutable entity_bfi_indicator) {
+
+    t = null;
+
+	v_cols_from = ['upside_capture_ret', 'downside_capture_ret', 'upside_capture_ratio', 'downside_capture_ratio', 'alpha_a', 'winrate', 'beta', 'info_a', 'track_error_a', 'jensen_a'];
+	v_cols_to = ['upsidecapture_ret', 'downsidecapture_ret', 'upsidecapture_ratio', 'downsidecapture_ratio', 'alpha', 'winrate', 'beta', 'info_ratio', 'tracking_error', 'jensen'];
+	v_cols_useless = ['track_error', 'info', 'alpha', 'treynor', 'jensen', 'm2', 'm2_a']; // 标准指标中不被当前表覆盖的数据点
+
+    if(isToMySQL) {
+
+		t = lj(
+				lj(
+					lj(
+						lj(
+							lj(
+								lj(
+									lj(
+										lj(
+											lj(entity_info,
+											   indicators['BFI-6M'] AS t_6m, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+											indicators['BFI-1Y'] AS t_1y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+										indicators['BFI-2Y'] AS t_2y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+									indicators['BFI-3Y'] AS t_3y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+								indicators['BFI-4Y'] AS t_4y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+							indicators['BFI-5Y'] AS t_5y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+						indicators['BFI-10Y'] AS t_10y, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+					indicators['BFI-YTD'] AS t_ytd, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless),
+				indicators['BFI-INCEP'] AS t_incep, ['entity_id', 'benchmark_id', 'end_date']).dropColumns!(v_cols_useless);
+				
+		t.rename!(v_cols_from, v_cols_to + '_6m');
+		t.rename!('t_1y_' + v_cols_from, v_cols_to + '_1y');
+		t.rename!('t_2y_' + v_cols_from, v_cols_to + '_2y');
+		t.rename!('t_3y_' + v_cols_from, v_cols_to + '_3y');
+		t.rename!('t_4y_' + v_cols_from, v_cols_to + '_4y');
+		t.rename!('t_5y_' + v_cols_from, v_cols_to + '_5y');
+		t.rename!('t_10y_' + v_cols_from, v_cols_to + '_10y');
+		t.rename!('t_ytd_' + v_cols_from, v_cols_to + '_ytd');
+		t.rename!('t_incep_' + v_cols_from, v_cols_to + '_incep');
+		t.dropColumns!(['inception_date', 'ini_value']).rename!('benchmark_id', 'factor_id');
+
+        entity_bfi_indicator.tableInsert(t.reorderColumns!(entity_bfi_indicator.colNames()));
+
+    } else {
+        
+    }
+}
+
+/*
  *   按照 XXX_performance_weekly 表结构准备数据记录
  * 
  * 

modules/indicatorCalculator.dos

@@ -1,7 +1,9 @@
 module fundit::indicatorCalculator
 
 use fundit::sqlUtilities
-use fundit::dataPuller
+use fundit::operationDataPuller
+use fundit::performanceDataPuller
+use fundit::ms_dataPuller
 use fundit::returnCalculator
 use fundit::navCalculator
 
@@ -1029,8 +1031,8 @@ def cal_monthly_indicators(entity_type, indicator_type, monthly_returns) {
     // 取基金和基准的对照表
     if(indicator_type == 'BFI') {
 
-        benchmark = SELECT fund_id AS entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
-                    FROM get_fund_bfi_factors(v_entity_ids, oldest_date.temporalFormat('yyyy-MM'), end_day.temporalFormat('yyyy-MM'));
+        benchmark = SELECT entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
+                    FROM get_entity_bfi_factors(entity_type, v_entity_ids, oldest_date.month(), end_day.month());
 
     } else {
         // 主基准， 对应 xxx_info 中的 primary_benchmark_id
@@ -1329,8 +1331,8 @@ def ms_cal_monthly_indicators(entity_type, indicator_type, monthly_returns) {
     // 取基金和基准的对照表
     if(indicator_type == 'BFI') {
 
-        benchmark = SELECT fund_id AS entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
-                    FROM get_fund_bfi_factors(v_entity_ids, oldest_date.temporalFormat('yyyy-MM'), end_day.temporalFormat('yyyy-MM'));
+        benchmark = SELECT entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
+                    FROM get_entity_bfi_factors(entity_type, v_entity_ids, oldest_date.month(), end_day.month());
 
     } else if(indicator_type == 'CAI') {
 

modules/operationDataPuller.dos

@@ -300,51 +300,34 @@ def get_entity_primary_benchmark(entity_type, entity_ids, month_start, month_end
 	
 }
 
+
 /*
- *  取某时间段的基金BFI因子
+ *  取某时间段的基金组合BFI基准
+ *
  * 
- *  Example: get_fund_bfi_factors("'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
- *           get_fund_bfi_factors(['MF00003PW2', 'MF00003PW1', 'MF00003PXO'], '1990-01', '2024-06');
+ *  Example: get_entity_bfi_factors('MF', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", 1990.01M, 2024.06M);
+ *           get_entity_bfi_factors('PF', [166002,166114], 1990.01M, 2029.12M, 2024.10.01);
  */
-def get_fund_bfi_factors(fund_ids, month_start, month_end) {
-
-    s_entity_ids = ids_to_string(fund_ids);
-
-    if(s_entity_ids == null || s_entity_ids == '') return null;
-    
-    s_query = "SELECT fund_id, end_date, factor_id
-               FROM pfdb.pf_fund_factor_bfi_by_category_group
-               WHERE fund_id IN (" + s_entity_ids + ")
-                 AND end_date >= '" + month_start + "'
-                 AND end_date <= '" + month_end + "'
-                 AND isvalid = 1
-               ORDER BY fund_id, end_date, factor_id;";
-
-    conn = connect_mysql();
-
-    t = odbc::query(conn, s_query);
-
-    conn.close();
+def get_entity_bfi_factors(entity_type, entity_ids, month_start, month_end, updatetime=1990.01.01) {
 
-    return t;
+    tmp = get_bfi_by_category_group_table_description(entity_type);
 
-}
+    s_entity_ids = ids_to_string(entity_ids);
 
+    sql_entity_id = '';
 
-/*
- *  取某时间段的组合BFI因子
- * 
- *  Example: get_portfolio_bfi_factors("166002,166114", '1900-01', '2024-06');
- */
-def get_portfolio_bfi_factors(portfolio_ids, month_start, month_end) {
+    if(s_entity_ids != NULL) {
+    	sql_entity_id = " AND " + tmp.sec_id_col[0] + " IN (" + s_entity_ids + ")";
+    }
 
-    s_query = "SELECT portfolio_id, end_date, factor_id
-               FROM pfdb.pf_portfolio_factor_bfi_by_category_group
-               WHERE portfolio_id IN (" + portfolio_ids + ")
-                 AND end_date >= '" + month_start + "'
-                 AND end_date <= '" + month_end + "'
-                 AND isvalid = 1
-               ORDER BY portfolio_id, end_date, factor_id;";
+    s_query = "SELECT " + tmp.sec_id_col[0] + " AS entity_id, end_date, factor_id 
+               FROM " + tmp.table_name[0] + "
+               WHERE isvalid = 1 " +
+                 sql_entity_id + "
+                 AND end_date >= '" + month_start.temporalFormat('yyyy-MM') + "'
+                 AND end_date <= '" + month_end.temporalFormat('yyyy-MM') + "'
+                 AND updatetime >= '" + updatetime$STRING + "'
+               ORDER BY " + tmp.sec_id_col[0] + ", end_date, factor_id";
 
     conn = connect_mysql();
 
@@ -353,41 +336,8 @@ def get_portfolio_bfi_factors(portfolio_ids, month_start, month_end) {
     conn.close();
 
     return t;
-
 }
 
-
-/*
- *  取某时间段的基金组合BFI基准
- *
- * 
- *  Example: get_entity_bfi_factors('MF', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
- *           get_entity_bfi_factors('PF', [166002,166114], '1990-01', '2024-08');
- */
-def get_entity_bfi_factors(entity_type, entity_ids, month_start, month_end) {
-
-    t = null;
-
-    s_entity_ids = ids_to_string(entity_ids);
-
-    if(s_entity_ids == null || s_entity_ids == '') return null;
-
-    if(entity_type == 'MF' || entity_type == 'HF') {
-
-    	t = get_fund_bfi_factors(s_entity_ids, month_start, month_end);
-
-        t.rename!('fund_id', 'entity_id');
-
-    } else if(entity_type == 'PF') {
-
-    	t = get_portfolio_bfi_factors(s_entity_ids, month_start, month_end);
-
-        t.rename!('portfolio_id', 'entity_id');    
-    }
-
-	return t;
-	
-}
 /*
  *   取基金-经理关系表
  *   

modules/rankingCalculator.dos

@@ -1,7 +1,8 @@
 module fundit::rankingCalculator
 
 use fundit::sqlUtilities
-use fundit::dataPuller
+use fundit::operationDataPuller;
+use fundit::performanceDataPuller;
 use fundit::dataSaver
 
 /*