支持相关性计算及BFI matching

modules/bfiMatcher.dos

@@ -3,146 +3,307 @@ module fundit::bfiMatcher
 use fundit::sqlUtilities;
 use fundit::operationDataPuller;
 use fundit::performanceDataPuller;
+use fundit::dataSaver;
 
 
 /*
  *   返回预设的指标最小值
+ *   
+ *   NOTE: 对数据量的要求， Java 计算coe相关性表时用48，但计算bfi时用10，这里统一用10
  * 
- */
+ */  
 def get_min_threshold(data_name) {
 
-    ret = 0;
+    d = dict(STRING, FLOAT);
 
-    if(data_name == 'correlation') { ret = 0.64; }
-    else if(data_name == 'ret_count') { ret = 48; }
+    d['correlation'] = 0.64;
+    d['data_count'] = 10; 
+    d['beta'] = 0.64;
+    d['t_value'] = 2.58;
+    d['r2'] = 0.64;
+    d['r2_neutral'] = 0.04;
 
-    return ret;
+    return d[data_name];
 }
 
-
 /*
- *  取BFI所需要的指数/因子ID
+ *  T-value 的聚合函数版
  * 
- *  NOTE: Java使用的逻辑如下（除了FA)，暂时没有差别
- *                    SELECT a.fund_Id FROM mfdb.fund_performance AS a
-                      LEFT JOIN pfdb.cm_class_asset_index AS b ON a.fund_id = b.index_id 
-                      WHERE a.end_date = 'YYYY-MM' AND a.fund_id LIKE 'IN%' AND b.isvalid = 1 ORDER BY a.fund_id ASC
-
  */
-def get_bfi_index_list() {
-
-    return ['FA00000VML','FA00000VMM','FA00000VMN','FA00000VMO','FA00000WKG','FA00000WKH','IN00000008','IN0000000D','IN0000000M','IN0000000S',
-            'IN0000000T','IN0000000U','IN0000000V','IN0000000W','IN0000000X','IN0000000Y','IN0000000Z','IN00000010','IN00000011','IN00000012',
-            'IN00000013','IN00000014','IN00000015','IN00000016','IN00000017','IN00000077','IN00000078','IN00000079','IN0000007A','IN0000007B',
-            'IN0000007C','IN0000007D','IN0000007E','IN0000007F','IN0000007G','IN0000007M','IN0000007N','IN0000007O','IN00000080','IN00000088',
-            'IN0000008O','IN0000009M','IN0000028E','IN000002CM'];
-}
-
-
 defg regressionT(y, x) {
 
-	r = SELECT beta, tstat FROM  ols(y, x, true, 1) WHERE rowNo(beta) = 1;
+	r = SELECT beta, tstat FROM ols(y, x, true, 1) WHERE rowNo(beta) = 1;
 
 	return r[0]['tstat'];
 }
 
+
 /*
- *   计算 bfi-matching 所需要的数据指标（月度）
- * 
- * 
+ *   计算 correlation & bfi-matching 所需要的数据指标（周数据计算，返回月度结果）
+ *   
+ *   NOTE: 与 Java BFI 不同，这里为了与RBSA保持统一，用收益率来计算相关性；转化成月度时用了各周平均值
  */
-def cal_monthly_closity(ret1, ret2, win) {
+def cal_monthly_closity0(nav1, nav2, win) {
+
+    n1 = nav1;
+    n1.sortBy!(['entity_id', 'price_date'], [1, 1]);
+    n2 = nav2;
+    n2.sortBy!(['benchmark_id', 'price_date'], [1, 1]);
 
-    t0 = SELECT end_date.month() AS end_date, end_date as price_date, ret1.ret AS ret1, ret2.ret AS ret2, tmoving(count, end_date, end_date, win) AS ret_count
-         FROM ret1
-         INNER JOIN ret2 ON ret1.end_date = ret2.end_date
+    t0 = SELECT entity_id, end_date, n1.price_date, n1.nav AS nav1, n1.nav.ratios()-1 AS ret1,
+                n2.nav AS nav2, n2.nav.ratios()-1 AS ret2, tmoving(count, end_date, end_date, win) AS data_count
+         FROM n1
+         INNER JOIN n2 ON n1.end_date = n2.end_date
          ORDER BY end_date;
 
-    t = SELECT end_date, price_date,
+    t = SELECT entity_id, end_date, price_date,
                tmcorr(t0.end_date, ret1, ret2, win) AS corr,
-               iif(tmstd(end_date, ret1-ret2, win) == 0, null, tmavg(end_date, ret1-ret2, win)\tmstd(end_date, ret1-ret2, win)) AS info,
-               tmoving(regressionT, end_date, [ret1, ret2], win) AS t_value,
-               tmbeta(end_date, ret1, ret2, win) AS beta // 用 ols() 算的值和这个一样
+               iif(tmstd(end_date, ret1-ret2, win) == 0, null, tmavg(end_date, ret1-ret2, win)\tmstd(end_date, ret1-ret2, win)) AS info, // 貌似没用
+               iif(data_count >= get_min_threshold('data_count'), tmoving(regressionT, end_date, [ret1, ret2], win), double(NULL)) AS t_value,
+               iif(data_count >= get_min_threshold('data_count'), tmbeta(end_date, ret1, ret2, win), double(NULL)) AS beta // 用 ols() 算的值和这个一样
         FROM t0
-        WHERE ret_count >= get_min_threshold('ret_count')
-        ORDER BY end_date;
-
-    UPDATE t SET corr = NULL WHERE corr < get_min_threshold('correlation');
-
-    return SELECT end_date.month().last() AS end_date,
-                  corr.last() AS corr,
-                  info.last() * sqrt(get_annulization_multiple('w')) AS info, // annuulized info ratio
-                  t_value.last() AS t_value,
-                  beta.last() AS beta
+        ORDER BY price_date;
+
+	// 将每月各周的数据平均值作为月度数据返回
+    return SELECT entity_id, price_date.month().last() AS end_date, price_date.last() AS price_date,
+                  corr.avg() AS corr,
+                  info.avg() AS info,
+                  t_value.avg() AS t_value,
+                  beta.avg() AS beta
            FROM t
-           GROUP BY end_date.month();
+           GROUP BY entity_id, price_date.month();
 }
 
+
 /*
- *   计算目标和BFI所用指数因子的相关系数
+ *   计算 correlation & bfi-matching 所需要的数据指标（周数据计算，返回月度结果）
  *   
- *   @param entity_info <TABLE>: NEED COLUMNS entity_id, inception_date, price_date
+ *   NOTE: 与 Java BFI 不同，这里为了与RBSA保持统一，用收益率来计算相关性；转化成月度时用了各周平均值
+ */
+def cal_monthly_closity(entity, nav1, nav2, win) {
+
+    n1 = nav1;
+    n1.sortBy!(['entity_id', 'price_date'], [1, 1]);
+    n2 = nav2;
+    n2.sortBy!(['benchmark_id', 'price_date'], [1, 1]);
+
+    t_dates = SELECT entity_id, end_date FROM nav1 WHERE end_date >= entity.price_date.weekEnd();
+
+    t0 = SELECT entity_id, end_date, n1.price_date, n1.nav AS nav1, n1.nav.ratios()-1 AS ret1,
+                n2.nav AS nav2, n2.nav.ratios()-1 AS ret2, tmoving(count, end_date, end_date, win) AS data_count
+         FROM n1
+         INNER JOIN n2 ON n1.end_date = n2.end_date
+         ORDER BY end_date;
+
+	t_rt = table(100:0, ['entity_id', 'end_date', 'price_date', 'corr', 'info', 't_value', 'beta'], 
+				[entity.entity_id.type(), MONTH, DATE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
+				
+	for(dt in t_dates.end_date) {
+
+		if( (EXEC data_count FROM t0 WHERE end_date = dt)[0] >= get_min_threshold('data_count') ){
+
+			rets = EXEC ret1, ret2 FROM t0 WHERE end_date BETWEEN(dt.temporalAdd(duration('-' + win$STRING)):dt);
+		
+			cor = corr(rets.ret1, rets.ret2);
+			info = mean(rets.ret1 - rets.ret2) \ std(rets.ret1 - rets.ret2); // 貌似没用
+		    t_value = regressionT(rets.ret1, rets.ret2);
+	    	bta = beta(rets.ret1, rets.ret2); // 用 ols() 算的值和这个一样
+	
+			INSERT INTO t_rt VALUES (entity.entity_id, dt, entity.price_date, cor, info, t_value, bta);
+		}
+	}
+
+	// 将每月各周的数据平均值作为月度数据返回
+    return SELECT entity_id, price_date.month().last() AS end_date, price_date.last() AS price_date,
+                  corr.avg() AS corr,
+                  info.avg() AS info,
+                  t_value.avg() AS t_value,
+                  beta.avg() AS beta
+           FROM t_rt
+           GROUP BY entity_id, price_date.month();
+}
+
+
+/*
+ *   计算目标和各资产类别指数及BFI因子的累计净值相关系数
  *   
- *   TODO: correlation is OK; beta, info, t_value are way off!
+ *   @param entity_info <TABLE>: [COLUMNS] entity_id, price_date
  *   
- *   NOTE: 与Java把月末日期作为截止日期不同的是，这里用每月最后一个周五作为截止日，所以数值会与MySQL中存储的略为不同
+ *   NOTE: 整合 Java中 TampCalcCorrelationServiceImpl 和 BestFitIndexServiceImpl 做的计算, 但只支持周数据计算
  * 
+ *   Example: cal_entity_index_coe('MF', get_fund_info(['MF00003PW1', 'MF00003PW2', 'MF00003RZI']).join(take(2024.09.30, 3) AS price_date).rename!('fund_id', 'entity_id'));
+ *   		  cal_entity_index_coe('PF', get_portfolio_info([166002]).join(take(2024.09.30, 1) AS price_date).rename!('portfolio_id', 'entity_id'));
+ *   
  */
 def cal_entity_index_coe(entity_type, entity_info) {
 
-// entity_info = get_fund_info(['MF00003PW1', 'MF00003PWC']).join(take(2024.10.31, 2) AS price_date).rename!('fund_id', 'entity_id');
+// entity_type = 'PF'
+// entity_info = get_portfolio_info([166002]).join(take(2024.09.30, 1) AS price_date).rename!('portfolio_id', 'entity_id');
+
+// entity_type = 'MF'
+// entity_info =  get_fund_info(['MF00003PW1', 'MF00003PW2', 'MF00003RZI']).join(take(2024.09.30, 3) AS price_date).rename!('fund_id', 'entity_id');
+// entity_info = tb_cal_entity[i : min(size, i+batch_size)]
+
+// entity_type = 'HF'
+// entity_info =  get_fund_info(['HF0000A134', 'HF0000A12R', 'HF00017JVX']).join(take(2024.05.30, 3) AS price_date).rename!('fund_id', 'entity_id');
 
 
     if(entity_info.isVoid() || entity_info.size() == 0) return null;
 
-    // 简单起见，取数据集中最新日期作为截止日期
-    end_day = entity_info.price_date.max();
+    // 取数据集中最早日期作为因子的起始日期
+    start_day = entity_info.price_date.min();
 
-	ret_entity = get_entity_weekly_rets(entity_type, entity_info);
+    // 取数据集每个基金组合指定日期之前5年至今的周净值
+    s_json = (SELECT entity_id AS sec_id, price_date.temporalAdd(-5y) AS price_date FROM entity_info).toStdJson();
 
-	if(ret_entity.isVoid() || ret_entity.size() == 0) return null;
+	nav_entity = get_nav_for_return_calculation(entity_type, 'w', s_json);
 
-    // 取BFI用得到的指数/因子列表
-    v_indexes = get_bfi_index_list();
+	if(nav_entity.isVoid() || nav_entity.size() == 0) return null;
 
-    // 手搓一个带日期的数据表
-    index_info = table(v_indexes AS entity_id, take(end_day, v_indexes.size()) AS price_date);
-    ret_index = get_entity_weekly_rets('MI', index_info);
+    // 取相关性计算及BFI用得到的指数/因子列表
+    // 只有基金需要单独做相关性计算，目的是为基金推荐做数据准备
+    if(entity_type in ('MF', 'HF'))
+	    v_indexes = (get_bfi_index_list().factor_id <- get_correlation_index_list().entity_id).distinct();
+	else {
+		v_indexes = get_bfi_index_list().factor_id;
+		// Portfolio_id 改回整型
+		v_port_id = nav_entity.sec_id$INT;
+		nav_entity.replaceColumn!('sec_id', v_port_id);
+	}
 
-	if(ret_index.isVoid() || ret_index.size() == 0) return null;
+    s_json2 = table(v_indexes AS sec_id, take(start_day.temporalAdd(-5y), v_indexes.size()) AS price_date).toStdJson();
 
-    // 两次循环遍历所有entity和指数
-    entity_coe = table(1000:0, ['entity_id', 'index_id', 'end_date', 'coe_1y', 'coe_3y', 'coe_5y', 'info_ratio_1y', 'info_ratio_3y', 'info_ratio_5y',
-                                                                     't_value_1y', 't_value_3y', 't_value_5y', 'beta_1y', 'beta_3y', 'beta_5y'],
-                               [SYMBOL, SYMBOL, MONTH, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
-                                                       DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
+    // 取指数及因子周点位
+    nav_index = get_nav_for_return_calculation('FA', 'w', s_json2).unionAll(get_nav_for_return_calculation('MI', 'w', s_json2));
+
+	if(nav_index.isVoid() || nav_index.size() == 0) return null;
 
-    for(entity in entity_info.entity_id) {
+	// 按照SQL 建表
+    entity_coe = create_entity_index_coe();
+
+    // 两次循环遍历所有entity和指数
+    for(entity in entity_info) {
+//entity= entity_info[0]
+        nav1 = SELECT sec_id AS entity_id, weekEnd(price_date) AS end_date, price_date, nav
+               FROM nav_entity WHERE sec_id = entity.entity_id;
 
-        ret1 = SELECT fund_id AS entity_id, price_date.weekEnd(4) AS end_date, price_date.weekEnd(4) AS price_date, ret_1w AS ret 
-               FROM ret_entity WHERE fund_id = entity AND price_date.weekEnd(4) <= end_day;
-    	
     	for(index in v_indexes) {
+//index=v_indexes[0]
+            nav2 = SELECT sec_id AS benchmark_id, weekEnd(price_date) AS end_date, price_date, nav 
+                   FROM nav_index WHERE sec_id = index;
 
-            ret2 = SELECT index_id AS benchmark_id, price_date.weekEnd(4) AS end_date, price_date.weekEnd(4) AS price_date, ret_1w AS ret 
-                   FROM ret_index WHERE index_id = index AND price_date.weekEnd(4) <= end_day;
+            if(nav2.isVoid() || nav2.size() == 0) continue; // 忽略已经停止更新的指数，或者是特殊的无风险利率 IN0000000M
 
-            benchmarks = table(take(entity, ret1.size()) AS entity_id, take(index, ret1.size()) AS benchmark_id, ret1.price_date.weekEnd(4) AS end_date);
-            
-            closity_1y = cal_monthly_closity(ret1, ret2, 1y);
-            closity_3y = cal_monthly_closity(ret1, ret2, 3y);
-            closity_5y = cal_monthly_closity(ret1, ret2, 5y);
+            closity_1y = cal_monthly_closity(entity, nav1, nav2, 1y);
+            closity_3y = cal_monthly_closity(entity, nav1, nav2, 3y);
+            closity_5y = cal_monthly_closity(entity, nav1, nav2, 5y);
 
     		INSERT INTO entity_coe
-        	    SELECT entity, index, c1.end_date, c1.corr AS coe_1y, c3.corr AS coe_3y, c5.corr AS coe_5y, 
-        	                                       c1.info AS info_ratio_1y, c3.info AS info_ratio_3y, c5.info AS info_ratio_5y,
-        	                                       c1.t_value AS t_value_1y, c3.t_value AS t_value_3y, c5.t_value AS t_value_5y,
-        	                                       c1.beta AS beta_1y, c3.beta AS beta_3y, c5.beta AS beta_5y
+        	    SELECT c1.entity_id, c1.end_date, index,
+        	    	   c1.corr AS coe_1y, c3.corr AS coe_3y, c5.corr AS coe_5y, 
+        	    	   //c1.corr2 AS coe_1y_2, c3.corr2 AS coe_3y_2, c5.corr2 AS coe_5y_2, 
+        	           c1.info AS info_ratio_1y, c3.info AS info_ratio_3y, c5.info AS info_ratio_5y,
+        	           c1.t_value AS t_value_1y, c3.t_value AS t_value_3y, c5.t_value AS t_value_5y,
+        	           c1.beta AS beta_1y, c3.beta AS beta_3y, c5.beta AS beta_5y
                 FROM closity_1y c1
                 LEFT JOIN closity_3y  c3 ON c1.end_date = c3.end_date
                 LEFT JOIN closity_5y c5 ON c1.end_date = c5.end_date;
+
     	}
     }
 
 	return entity_coe;
 }
+
+/*
+ *   匹配BFI, 逻辑和 Java BestFitIndexServiceImpl 类似
+ * 
+ *   @param entity_info <TABLE>: [COLUMNS] entity_id, strategy
+ *   @param entity_coe <TABLE>: [COLUMNS] entity_id, end_date, index_id, coe_1y, t_value_1y, beta_1y
+ * 
+ *   NOTE: Java 中的 rule2 还包括 FA00000VN7 (100%中证全指 IN0000007N) 是不对的，而且漏掉了CTA和FOF。已将DEV数据库中此因子划入category_group 74；另外找r2最小的因子也离谱
+ *                  rule3 FA00000VMX （100%中证转债 中证转债）漏掉了公募债券（FOF, 相对价值（套利），多策略，公募混合是否要加？怕会和股票打架，待研究）
+ *                  
+          UPDATE pfdb.`cm_factor_information` 
+            SET category_group_id = 74, category='全市场', factor_name='全市场', category_group='规模', strategy=',101,102,103,107,', maximum_num=1, updaterid=123, updatetime='2024-11-25' 
+          WHERE factor_id = 'FA00000VN7';
+          
+          UPDATE pfdb.`cm_factor_information` 
+            SET category_group_id = 78, category_group='配置', maximum_num=1, updaterid=123, updatetime='2024-11-25' 
+          WHERE factor_id = 'FA00000VNB' AND category_group_id = 80;
+          
+          UPDATE pfdb.`cm_factor_information` 
+            SET category_group_id = 78, category_group='配置', maximum_num=1, updaterid=123, updatetime='2024-11-25' 
+          WHERE factor_id = 'FA00000VND' AND category_group_id = 76;
+ *         
+ */
+def match_entity_bfi(entity_type, entity_info, entity_coe) {
+
+	// 特殊因子：现金，可被应用于所有策略
+    v_factor_cash = ['FA000000MJ'];
+
+	//有一些特殊的因子只会被部分策略所用, 否则会引起歧义
+	v_factor_1 = ['FA00000VMY', 'FA00000VMZ', 'FA00000VN0', 'FA00000VN1', 'FA00000VN2', 'FA00000VN3', 'FA00000VN4', 'FA00000VN5', 'FA00000VN6'];
+	v_strategy_1 = [3, 7, 8, 105]; // 私募CTA, 私募FOF, 私募多策略, 公募商品
+
+	v_factor_2 = ['FA00000SMB', 'FA00000VMG'];
+	v_strategy_2 = [1, 3, 5, 7, 8]; // 私募股票（多空），CTA, 相对价值，私募FOF, 私募多策略
+
+	v_factor_3 = ['FA00000VMX'];
+	v_strategy_3 = [6, 103]; // 私募固收，公募债券
+
+	// 只需要BFI因子的相关性数据
+	coe = SELECT *
+	      FROM ej(entity_info, ej(entity_coe, get_bfi_index_list(), 'index_id', 'factor_id'), 'entity_id')
+		  ORDER BY entity_id, end_date, category_group_id, coe_1y DESC, order_id;
+
+	t_bfi_raw = table(1000:0, 
+					  ['entity_id', 'end_date', 'category_group_id', 'factor_id', 'rank', 'coe_1y', 'r2',
+					  //'rank2', 'coe_1y_2', 'r2_2',
+					   'performance_flag', 't_value_1y', 'beta_1y', 'maximum_num', 'order_id', 'factor_name'],
+					  [iif(entity_type=='PF', INT, SYMBOL), MONTH, SHORT, SYMBOL, SHORT, DOUBLE, DOUBLE,
+					  //SHORT, DOUBLE, DOUBLE,
+					   STRING, DOUBLE, DOUBLE, SHORT, SHORT, STRING]);				  
+
+	// 首先处理特殊情况 TODO: java treats rule2 differently by finding min R2 without checking t_value & corr
+	v_special_rule = [v_factor_1, v_factor_2, v_factor_3];
+	v_special_strategy = [v_strategy_1, v_strategy_2, v_strategy_3];
+	
+	for(i in 0..v_special_rule.size()-1) {
+		INSERT INTO t_bfi_raw
+			SELECT * FROM (
+				SELECT entity_id, end_date, category_group_id, index_id AS factor_id,
+					   coe_1y.rank(false) AS rank, coe_1y, square(coe_1y) AS r2, 
+					   'w', t_value_1y, beta_1y,
+					   maximum_num, order_id, factor_name
+				FROM entity_info ei
+				INNER JOIN coe ON ei.entity_id = coe.entity_id
+				WHERE ei.strategy IN v_special_strategy[i]
+				  AND coe.index_id IN v_special_rule[i].join(v_factor_cash)
+				  AND t_value_1y >= get_min_threshold('t_value') 
+	 			  AND coe_1y >= get_min_threshold('correlation')
+				  AND order_id IS NOT NULL
+				CONTEXT BY entity_id, end_date, category_group_id )
+			WHERE rank < maximum_num;
+
+		DELETE FROM coe WHERE index_id IN v_special_rule[i];
+	}
+	
+	INSERT INTO t_bfi_raw
+		SELECT * FROM (
+			SELECT entity_id, end_date, category_group_id, index_id AS factor_id,
+				   coe_1y.rank(false) AS rank, coe_1y, square(coe_1y) AS r2, 
+				   'w', t_value_1y, beta_1y,
+				   maximum_num, order_id, factor_name
+			FROM entity_info ei
+			INNER JOIN coe ON ei.entity_id = coe.entity_id
+			WHERE t_value_1y >= get_min_threshold('t_value')
+
+			  AND coe_1y >= get_min_threshold('correlation')
+			  AND order_id IS NOT NULL
+			CONTEXT BY entity_id, end_date, category_group_id )
+		WHERE rank < maximum_num;
+
+	return SELECT * FROM t_bfi_raw ORDER BY entity_id, end_date, category_group_id;
+
+}

modules/dataSaver.dos

@@ -308,14 +308,38 @@ def create_entity_latest_performance(is_id_integer=false) {
 def create_entity_index_coe(is_id_integer=false) {
 
 	return table(1000:0,
-	            ['entity_id', 'end_date',
+	            ['entity_id', 'end_date', 'index_id',
 	             'coe_1y', 'coe_3y', 'coe_5y', 'info_ratio_1y', 'info_ratio_3y', 'info_ratio_5y',
 	             't_value_1y', 't_value_3y', 't_value_5y', 'beta_1y', 'beta_3y', 'beta_5y'],
-	            [iif(is_id_integer, INT, SYMBOL), STRING,
+	            [iif(is_id_integer, INT, SYMBOL), MONTH, SYMBOL,
 	             DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
 	             DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
 }
 
+/*
+ *   建表 xxx_factor_bfi_max_r2
+ * 
+ */
+def create_entity_bfi_max_r2(is_id_integer=false) {
+
+	return table(1000:0,
+	            ['entity_id', 'end_date', 'factor_id', 'coe', 'r2', 'performance_flag', 'rz_portrait'],
+	            [iif(is_id_integer, INT, SYMBOL), MONTH, SYMBOL, DOUBLE, DOUBLE, STRING, STRING]);
+}
+
+
+/*
+ *   建表 xxx_factor_bfi
+ * 
+ */
+def create_entity_factor_bfi(is_id_integer=false) {
+
+		return table(1000:0,
+	            ['entity_id', 'end_date', 'factor_id',
+	             'coe', 'r2', 'performance_flag', 't_value_1y', 'beta_1y'],
+	            [iif(is_id_integer, INT, SYMBOL), MONTH, SYMBOL,
+	             DOUBLE, DOUBLE, STRING, DOUBLE, DOUBLE]);
+}
 
 /*
  *   建表 XXXX_indicator_ranking

modules/performanceDataPuller.dos

@@ -238,7 +238,7 @@ def get_fund_latest_nav_performance(fund_ids, isFromMySQL) {
 }
 
 /*
- * 通用取净值
+ * 通用取给定日期之后（含此日期）的净值
  * 
  * 
  * Create: 202408                                                    Joey
@@ -447,8 +447,8 @@ def get_benchmark_return(benchmarks, end_day) {
     if(benchmarks.isVoid() || benchmarks.size() == 0) { return null; }
 
     // 前缀为 IN 的 benchmark id
-    t_index_id = SELECT DISTINCT benchmark_id FROM benchmarks WHERE benchmark_id LIKE 'IN%';
-    s_index_ids = iif(isVoid(t_index_id), "", "'" + t_index_id.benchmark_id.concat("','") + "'");
+    t_index_id = SELECT DISTINCT benchmark_id FROM benchmarks WHERE benchmark_id LIKE 'IN%';;
+    s_index_ids = iif(isVoid(t_index_id), "",  "'" + t_index_id.benchmark_id.concat("','") + "'");
 
    // 前缀为 FA 的 benchmark id
    t_factor_id = SELECT DISTINCT benchmark_id FROM benchmarks WHERE benchmark_id LIKE 'FA%';
@@ -849,5 +849,55 @@ def get_fund_bfi_bm_indicator_ranking(fund_ids, end_date, factor_ids, isFromMySQ
     }
 
     return t;
+}
+
+/*
+ *  取算相关性所需要的指数/因子ID
+ * 
+ *  NOTE: 与 Java 使用的逻辑相同， 在cm_class_asset_index中有当月performance的指数加上 CSI5 和 Cap3Style 的 factor
+ *        唯一的不同是将历史已经停止维护的老指数筛掉（必须有当前月份的指数收益）
+ *
+ *  Example: get_correlation_index_list();
+ */
+def get_correlation_index_list() {
+
+    s_query = "SELECT a.fund_id AS entity_id
+    		   FROM mfdb.fund_performance AS a
+          	   LEFT JOIN pfdb.cm_class_asset_index AS b ON a.fund_id = b.index_id 
+	           WHERE a.end_date = LEFT(CURRENT_DATE, 7)
+	           AND a.fund_id LIKE 'IN%' 
+	           AND b.isvalid = 1 
+	           ORDER BY a.fund_id ASC";
+
+    conn = connect_mysql();
+ 
+    t = odbc::query(conn, s_query);
+ 
+    conn.close();
+
+	INSERT INTO t VALUES (['FA00000VML','FA00000VMM','FA00000VMN','FA00000VMO','FA00000WKG','FA00000WKH']);
+
+	return t;
+}
+
+/*
+ *  取BFI所需要的因子ID
+ * 
+ *  TODO: 大盘风格、小盘风格 (FA00000WKG, FA00000WKH) 没有category_group_id & order_id, 在哪里用？
+ */
+def get_bfi_index_list() {
+
+	s_query = "SELECT factor_id, factor_name, category_group_id, maximum_num, order_id
+    		   FROM pfdb.cm_factor_information
+	           WHERE isvalid = 1
+	             AND ( factor_type = 5 OR factor_id IN ('FA00000SMB', 'FA00000VMG') )
+	           ORDER BY order_id ASC";
+
+    conn = connect_mysql();
+ 
+    t = odbc::query(conn, s_query);
+ 
+    conn.close();
 
+	return t;
 }

modules/returnCalculator.dos

@@ -1,5 +1,6 @@
 module fundit::returnCalculator
 
+use fundit::sqlUtilities
 use fundit::operationDataPuller
 use fundit::performanceDataPuller
 
@@ -97,78 +98,6 @@ def mix_monthly_returns(entity_type, entity_info) {
 
 
 /*
- *  【作废】根据基金净值序列计算月收益序列（适合提供给指标运算）
- * 
- *  Create:  20240907                                                  Joey
- *                    TODO: missing pulling data from local
- *                    TODO: ONLY support month return now
- *                    
- *  @param entity_type <STRING>: NAV universe, 'HF','MF','PF','EQ'... defined in get_nav_table_description()
- *  @param fund_ids <STRING VECTOR>: 基金ID
- *  @param isFromMySQL <BOOL>: 净值来源 1 - 远程MySQL、 0 - 本地 DolphinDB
- *  
- *  Example: cal_fund_monthly_returns('HF', "'HF000004KN','HF000103EU','HF00018WXG'", true);
- *  
- */
-def cal_fund_monthly_returns(entity_type, fund_ids, isFromMySQL){
-
-    tb_rets = null;
-
-    // 暂时只支持公私募和组合
-    if(!(entity_type IN ['HF', 'MF', 'PF'])) return tb_rets;
-    
-    // 用于保证老基金也能取到所有历史净值
-    very_old_price_date = 1990.01.01;
-
-    // 基金基本信息，包括初始净值
-    tb_fund_info = get_fund_info(fund_ids);
-
-    // 基金净值
-    tb_nav = SELECT * FROM get_nav_by_price_date(entity_type, fund_ids, very_old_price_date, isFromMySQL);
-
-    tb_month_end = table(100:0, ['entity_id', 'price_date'], [STRING, DATE]);
-    // 填充好各基金有效期内所有月份的最后一天
-    for( f in tb_fund_info )
-    {
-        INSERT INTO tb_month_end SELECT fund_id AS entity_id, price_date FROM table(f.fund_id.take(1) AS fund_id).cj(table(temporalSeq(f.inception_date, today(), 'M') AS price_date)) ;
-    }
-
-    UPDATE tb_month_end SET end_date = price_date.month();
-    
-    tb_monthly_nav = SELECT entity_id, monthEnd(price_date).month().last() AS end_date, price_date.last() AS price_date, cumulative_nav.last() AS cumulative_nav
-                     FROM tb_nav
-                     GROUP BY entity_id, monthEnd(price_date);
-
-    // 完整月末日期的净值序列（包括缺失数据为NULL）
-    tb_monthly_nav = SELECT me.entity_id, me.end_date, n.price_date, n.cumulative_nav
-                     FROM tb_month_end me
-                     LEFT JOIN tb_monthly_nav n ON me.entity_id = n.entity_id AND me.end_date = n.end_date
-                     ORDER BY me.entity_id, me.end_date;
-
-    // 补一下成立日的初始净值
-    // NOTE: DolphinDB 遇见 EXISTS 语句时，似乎主表的 alias 失效，只好用全名
-    INSERT INTO tb_monthly_nav 
-        SELECT fund_id AS entity_id, inception_date.month(), inception_date, ifNull(ini_value, 1)
-        FROM tb_fund_info fi
-        WHERE NOT EXISTS ( SELECT * FROM tb_monthly_nav n WHERE entity_id = tb_fund_info.fund_id AND n.price_date = tb_fund_info.inception_date);
-
-    if(tb_monthly_nav.isVoid() || tb_monthly_nav.size() == 0) { return tb_rets; }
-
-    // 算 ratios 之前先把时间顺序排好
-    tb_monthly_nav.sortBy!(['entity_id', 'end_date', 'price_date'], [1, 1, 1]);
-
-    // 计算月收益
-    tb_rets = SELECT entity_id AS fund_id, end_date, price_date, cumulative_nav, cumulative_nav.ratios() - 1 AS ret
-              FROM tb_monthly_nav
-              CONTEXT BY entity_id;
-
-
-    // the records without return calculated but do have nav are still useful for some calculations
-    return ( SELECT * FROM tb_rets WHERE cumulative_nav > 0 );
-}
-
-
-/*
  *  根据基金净值序列计算周收益序列
  * 
  *  Create:  20240907                                                  Joey
@@ -192,8 +121,7 @@ def cal_weekly_returns(entity_type, entity_info){
      tb_nav = get_nav_for_return_calculation(entity_type, 'w', s_json);
      tb_nav.rename!('sec_id', 'entity_id');
 
-    UPDATE tb_nav  // 当12月31日是周四、五、六时，该周为第52周，所以次年前几天有可能是上一年的第52，53周
-    SET year_week = iif(price_date.weekOfYear() >= 52 && price_date.monthOfYear() == 1, price_date.year()-1, price_date.year())$STRING + (price_date.weekOfYear()$STRING).lpad(2, "0");
+    UPDATE tb_nav SET year_week = get_year_week(price_date);
 
     tb_weekly_nav = SELECT entity_id, year_week, price_date.last() AS price_date, cumulative_nav.last() AS cumulative_nav
                     FROM tb_nav n
@@ -211,104 +139,6 @@ def cal_weekly_returns(entity_type, entity_info){
     return tb_rets_1w;
 }
 
-/*
- *  [RETIRED] 月末 fund_performance 表计算
- *  
- *  @param fund_ids <STRING>: 逗号分隔的ID
- *  @param end_date <STRING>: YYYY-MM 
- * 
- *  Example: cal_fund_performance("'HF000004KN','HF00018WXG','HF000103EU'", '2024-06', true);
- */
-def cal_fund_performance(fund_ids, month_end) {
-
-    // 获取必要的基金月度净值
-    tb_nav = get_nav_for_hedge_fund_performance(fund_ids, month_end);
-
-    tb_rets = SELECT fund_id, price_date.month() AS end_date, price_date, cumulative_nav,
-                     cumulative_nav \ nav_1m - 1 AS ret_1m,
-                     cumulative_nav \ nav_3m - 1 AS ret_3m,
-                     cumulative_nav \ nav_6m - 1 AS ret_6m,
-                     cumulative_nav \ nav_1y - 1 AS ret_1y,
-                     cumulative_nav \ nav_2y - 1 AS ret_2y,
-                     cumulative_nav \ nav_3y - 1 AS ret_3y,
-                     cumulative_nav \ nav_4y - 1 AS ret_4y,
-                     cumulative_nav \ nav_5y - 1 AS ret_5y,
-                     cumulative_nav \ nav_10y - 1 AS ret_10y,
-                     cumulative_nav \ nav_ytd - 1 AS ret_ytd,
-                     cumulative_nav \ nav_incep - 1 AS ret_incep, inception_date
-              FROM tb_nav;
-
-    // NOTE: this is to keep consistance with MySQL, even it is NOT complied with GIPS standard
-    UPDATE tb_rets SET ret_1m_a = (1 + ret_1m).pow(12\1) - 1, ret_3m_a = (1 + ret_3m).pow(12\3) - 1, ret_6m_a = (1 + ret_6m).pow(12\6) - 1,
-                       ret_1y_a= ret_1y, ret_2y_a = (1 + ret_2y).pow(12\24) - 1, ret_3y_a = (1 + ret_3y).pow(12\36) - 1,
-                       ret_4y_a = (1 + ret_4y).pow(12\48) - 1, ret_5y_a = (1 + ret_5y).pow(12\60) - 1, ret_10y_a = (1 + ret_10y).pow(12\120) - 1,
-                       ret_ytd_a = (1 + ret_ytd).pow(12\int(temporalFormat(end_date, 'MM')))-1,
-                       ret_incep_a = (1 + ret_incep).pow(12\(end_date - inception_date.month())) - 1,
-                       ret_incep_a_all = (1 + ret_incep).pow(12\(end_date - inception_date.month()))- 1,
-                       ret_incep_a_gips = iif(end_date - inception_date.month() < 12, ret_incep,
-                                             (1 + ret_incep).pow(12\(end_date - inception_date.month()))- 1);
-
-    return tb_rets;
-}
-
-
-
-/*
- *  [RETIRED] 批量计算公募历史基金月度收益(fund_performance)
- *  NOTE: 任何数据频率快于或等于月度的净值数据都可以用此函数一次性计算完整历史记录。双月频、季频甚至更低频率的基金只能按月计算
- *
- *  cal_mutual_fund_performance("'HF000004KN','HF00018WXG','HF000103EU'", true)
- *
- */
-def cal_mutual_fund_performance(fund_ids, isFromMySQL) {
-
-    // 计算月收益
-    tb_tmp = cal_fund_monthly_returns('MF', fund_ids, isFromMySQL);
-
-    tb_rets = SELECT fund_id, end_date, ret_1m,
-                     (1 + ret_1m).mprod(3) - 1 AS ret_3m, (1 + ret_1m).mprod(6) - 1 AS ret_6m, (1 + ret_1m).mprod(12) - 1 AS ret_1y,
-                     (1 + ret_1m).mprod(24) - 1 AS ret_2y, (1 + ret_1m).mprod(36) - 1 AS ret_3y, (1 + ret_1m).mprod(48) - 1 AS ret_4y,
-                     (1 + ret_1m).mprod(60) - 1 AS ret_5y, (1 + ret_1m).mprod(120) - 1 AS ret_10y
-              FROM tb_tmp
-              CONTEXT BY fund_id;
-
-    // NOTE: this is to keep consistance with MySQL, even it is NOT complied with GIPS standard
-    UPDATE tb_rets SET ret_1m_a = (1 + ret_1m).pow(12) - 1, ret_3m_a = (1 + ret_3m).pow(4) - 1, ret_6m_a = (1 + ret_6m).pow(2) - 1, ret_1y_a= ret_1y,
-                       ret_2y_a = (1 + ret_2y).pow(1\2) - 1, ret_3y_a = (1 + ret_3y).pow(1\3) - 1, ret_4y_a = (1 + ret_4y).pow(1\4) - 1, 
-                       ret_5y_a = (1 + ret_5y).pow(1\5) - 1, ret_10y_a = (1 + ret_10y).pow(1\10) - 1;
-    
-    // ytd 不会用上面的CONTEXT BY语句实现
-    tb_ret_ytd = SELECT a.fund_id, a.end_date, a.price_date, a.cumulative_nav, -1 + a.cumulative_nav \ b.cumulative_nav AS ret_ytd,
-                       (a.cumulative_nav \ b.cumulative_nav).pow(12\(a.end_date - b.end_date)) - 1 AS ret_ytd_a
-                 FROM tb_rets a
-                 INNER JOIN tb_rets b ON a.fund_Id = b.fund_id 
-                   AND b.end_date = a.price_date.yearEnd().datetimeAdd(-1y).month()
-    
-    // since inception 不会用上面的CONTEXT BY语句实现
-    tb_ret_incep = SELECT a.fund_id, a.end_date, a.price_date,  cumulative_nav, -1 + cumulative_nav \ ini_value AS ret_incep
-                   FROM tb_rets a
-                   INNER JOIN tb_fund_info fi ON a.fund_id = fi.fund_id
-
-    
-    UPDATE tb_ret_incep SET ret_incep_a = (1 + ret_incep).pow(12\(end_date - end_date.first())) - 1 CONTEXT BY fund_Id
-    UPDATE tb_ret_incep SET ret_incep_a_gips = iif( end_date - end_date.first() < 12, ret_incep, ret_incep_a ), ret_incep_a_all = ret_incep_a CONTEXT BY fund_id
-    
-    // 只选需要更新的记录
-    tb_fund_performance = SELECT a.fund_id, a.end_date.datetimeFormat("yyyy-MM") AS end_date, c.price_date, c.cumulative_nav,
-                                 a.ret_1m, a.ret_1m_a, a.ret_3m, a.ret_3m_a, a.ret_6m, a.ret_6m_a,
-                                 a.ret_1y, a.ret_1y_a, a.ret_2y, a.ret_2y_a, a.ret_3y, a.ret_3y_a,
-                                 a.ret_4y, a.ret_4y_a, a.ret_5y, a.ret_5y_a, a.ret_10y, a.ret_10y_a,
-                                 b.ret_ytd, b.ret_ytd_a, c.ret_incep, c.ret_incep_a, c.ret_incep_a_all, c.ret_incep_a_gips
-                                 // , 123 AS creatorid, now() AS createtime, 123 AS updaterid, now() AS updatetime, 1 AS isvalid
-                          FROM tb_rets a 
-                          LEFT JOIN tb_ret_ytd b ON a.fund_id = b.fund_id AND a.end_date = b.end_date
-                          LEFT JOIN tb_ret_incep c ON a.fund_id = c.fund_id AND a.end_date = c.end_date
-                          WHERE c.price_date IS NOT NULL
-                          ORDER BY a.fund_id, c.price_date
-
-    return tb_fund_performance
-}
-
 
 /*
  *  批量计算区间收益

modules/task_fundPerformance.dos

@@ -14,12 +14,13 @@ use fundit::ms_dataPuller;
  *   [定时任务]：最新净值触发的业绩指标计算
  * 
  *   @param entityType <STRING>: 'MF', 'HF'...
- *   @param date <DATETIME>: 净值更新时间, 为空时缺省为当前时间-1天；为1990.01.01或更早日期时代表初始化，指标会被存入本地数据库
+ *   @param date <DATETIME>: 净值更新时间, 为空时缺省为当前时间-1天；为1989.01.01或更早日期时代表初始化，指标会被存入本地数据库
  *   
  *   NOTE: 与Java不同的是当月indicator计算每日触发，不必等到Month-end production
  *   
  *   Example: calFundPerformanceTask('MF', 2024.10.28);
  *            calFundPerformanceTask('MI', 2024.10.28);
+ *            calFundPerformanceTask('MF', get_ini_data_const()['date']);  -- 【初始化数据专用】(70min)
  */
 def calFundPerformanceTask(entityType, date) {
 
@@ -125,13 +126,15 @@ def calFundPerformanceTask(entityType, date) {
 
             save_and_sync(tb_fund_latest_performance, 'raw_db.fund_latest_performance', 'raw_db.fund_latest_nav_performance');
 
-            // 数据初始化时将指标存入本地，做排名之用
-            if(end_day <= 1990.01.01) {
-            	save_table(tb_fund_performance, 'raw_db.fund_performance', false);
-            	save_table(tb_fund_indicator, 'raw_db.fund_indicator', false);
-            	save_table(tb_fund_risk_stats, 'raw_db.fund_risk_stats', false);
-            	save_table(tb_fund_riskadjret_stats, 'raw_db.fund_riskadjret_stats', false);
-            	save_table(tb_fund_style_stats, 'raw_db.fund_style_stats', false);
+            // 数据初始化时将指标存入本地
+            if(end_day <= get_ini_data_const['date']) {
+            	save_table(tb_fund_performance, 'mfdb.fund_performance', false);
+            	save_table(tb_fund_indicator, 'mfdb.fund_indicator', false);
+            	save_table(tb_fund_risk_stats, 'mfdb.fund_risk_stats', false);
+            	save_table(tb_fund_riskadjret_stats, 'mfdb.fund_riskadjret_stats', false);
+            	save_table(tb_fund_style_stats, 'mfdb.fund_style_stats', false);
+            	save_table(tb_fund_performance_weekly, 'mfdb.fund_performance_weekly', false);
+            	save_table(tb_fund_latest_performance, 'mfdb.fund_latest_performance', false);
             }
 
         } catch(ex) {
@@ -145,6 +148,16 @@ def calFundPerformanceTask(entityType, date) {
 	
 }
 
+/*
+ *   [定时任务] 匹配BFI并存入数据库
+ * 
+ * 
+ */
+def matchEntityBFI(entityType, date) {
+
+
+}
+
 
 /*
  *   [定时任务] 计算BFI指标并存入数据库
@@ -234,8 +247,8 @@ def calEntityBfiIndicatorTask(entityType, date) {
             save_and_sync(tb_bfi_indicator, t_desc.table_name[0].strReplace(db_name, 'raw_db'), t_desc.table_name[0].strReplace(db_name, 'raw_db'));
 
             // 数据初始化时将指标存入本地，做排名之用
-            if(end_day <= 1990.01.01) {
-            	save_table(tb_bfi_indicator, t_desc.table_name[0].strReplace(db_name, 'raw_db'), false);
+            if(end_day <= get_ini_data_const['date']) {
+            	save_table(tb_bfi_indicator, t_desc.table_name[0], false);
             }
 
         } catch(ex) {
@@ -415,22 +428,96 @@ def ms_calFundReturns() {
 }
 
 /*
- *   实验性质的API
+ *   [定时任务] 计算基金和组合的BFI
  * 
  * 
+ *   TODO: max_r2 表在哪里被用到了？应该和基金推荐有关系
  */
-def calFundIndexCorrelation(entityType, date) {
+def MatchEntityBFI(entityType, date) {
+//entityType = 'MF'
+//date = 2024.11.20
+
+	rt = '';
 
-    if(find(['HF', 'MF'], entityType) < 0) return null;
+    if(find(['HF', 'MF', 'PF'], entityType) < 0) return null;
 
     // 取有最新净值变动的基金列表 (1s)
-    tb_cal_funds = get_entity_list_by_nav_updatetime(entityType, NULL, date, true);
+    tb_cal_entity = get_entity_list_by_nav_updatetime(entityType, NULL, date, true);
 
-    if(tb_cal_funds.isVoid() || tb_cal_funds.size() == 0 ) return;
+    if(tb_cal_entity.isVoid() || tb_cal_entity.size() == 0 ) return;
+
+	i = 0;
+	size = tb_cal_entity.size();
+	batch_size = 1000;
+	entity_index_coe = create_entity_index_coe();
+	
+	do {
+		// 4 min per 1000 funds
+	    coe = cal_entity_index_coe(entityType, tb_cal_entity[i : min(size, i+batch_size)]);
+
+	    if(coe.isVoid() || coe.size() == 0) continue;
+
+		entity_info = get_entity_info(entityType, tb_cal_entity[i : min(size, i+batch_size)].entity_id);
+		
+		bfi_raw = match_entity_bfi(entityType, entity_info, coe);
+
+		// 先存到数据库，落袋为安
+		try {
+		    
+			// 筛掉 correlation 绝对值不够阈值的记录
+	    	t_coe = SELECT entity_id, end_date, index_id,
+	    				   iif(coe_1y.abs() < get_min_threshold('correlation'), double(NULL), coe_1y) AS coe_1y,
+	    				   iif(coe_3y.abs() < get_min_threshold('correlation'), double(NULL), coe_3y) AS coe_3y,
+	    				   iif(coe_5y.abs() < get_min_threshold('correlation'), double(NULL), coe_5y) AS coe_5y,
+	    				   info_ratio_1y, info_ratio_3y, info_ratio_5y, t_value_1y, t_value_3y, t_value_5y, beta_1y, beta_3y, beta_5y
+					FROM coe;
+
+			DELETE FROM t_coe WHERE coe_1y IS NULL AND coe_3y IS NULL AND coe_5y IS NULL;
+
+			// 候选因子
+			t_bfi_candidates = SELECT entity_id, end_date, index_id AS factor_id, coe_1y AS coe, coe_1y.square() AS r2, 'w' AS performance_flag, t_value_1y, beta_1y
+			                   FROM t_coe WHERE index_id LIKE 'FA%';
+			
+			chg_columns_for_mysql(t_coe, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
+
+			// 只有基金需要存 index_coe 表
+		    if(entityType IN ['MF', 'HF']) save_and_sync(t_coe, 'raw_db.pf_fund_index_coe', );
+
+			// 所有的 factors 存到 xxx_factor_bfi 表；NOTE: Java 把所有 factor 的数据都存起来，这里只存 correlation 达标的记录 （反正这个表没啥用？）
+			chg_columns_for_mysql(t_bfi_candidates, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
+	        save_and_sync(t_bfi_candidates, iif(entityType == 'PF', 'raw_db..pf_portfolio_factor_bfi', 'raw_db.cm_fund_factor_bfi'), );
+
+			if(bfi_raw.isVoid() || bfi_raw.size() == 0) continue;
+
+			// 有效因子
+	        t_bfi = SELECT entity_id, end_date, factor_id, coe_1y AS coe, r2, performance_flag, t_value_1y, beta_1y 
+	                FROM bfi_raw ORDER BY entity_id, end_date, r2 DESC;
+
+	        // 最大R2因子及所有有效因子标签
+			t_max_r2 = SELECT entity_id, factor_id.first() AS factor_id, end_date,
+			                  int(NULL) AS performance_flag, coe.first() AS coe, r2.first() AS r2, concat(factor_name, ",") AS rz_portrait
+			           FROM ej(t_bfi, get_bfi_index_list(), 'factor_id')
+			           GROUP BY entity_id, end_date;
 
-//    tb_fund_index_coe = create_entity_index_coe();
-    // (7m)
-    coe = cal_entity_index_coe(entityType, tb_cal_funds[0:1000]);
+			// 有效 factors 存到 xxx_factor_bfi_by_category_group 表
+			chg_columns_for_mysql(t_bfi, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
+			save_and_sync(t_bfi_candidates, iif(entityType == 'PF', 'raw_db.pf_portfolio_factor_bfi_by_category_group', 'raw_db.pf_fund_factor_bfi_by_category_group'), );
 
-    return coe;
+			// 有效因子中 R2 最大的因子存 xxx_max_r2 
+			chg_columns_for_mysql(t_max_r2, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
+			save_and_sync(t_max_r2, iif(entityType == 'PF', 'raw_db.pf_portfolio_factor_bfi_max_r2', 'raw_db.pf_fund_factor_bfi_by_category_group_max_r2'), );
+	        
+
+		} catch (ex) {
+
+            //TODO: Log errors
+            rt += ex + '\n';
+        }
+	    
+	    i = i + batch_size;
+	
+	} while (i<size)
+
+	
+    return rt;
 }