小修小补

modules/bfiMatcher.dos

@@ -1,7 +1,23 @@
 module fundit::bfiMatcher
 
-use fundit::sqlUtilities
-use fundit::dataPuller
+use fundit::sqlUtilities;
+use fundit::operationDataPuller;
+use fundit::performanceDataPuller;
+
+
+/*
+ *   返回预设的指标最小值
+ * 
+ */
+def get_min_threshold(data_name) {
+
+    ret = 0;
+
+    if(data_name == 'correlation') { ret = 0.64; }
+    else if(data_name == 'ret_count') { ret = 48; }
+
+    return ret;
+}
 
 
 /*
@@ -22,34 +38,60 @@ def get_bfi_index_list() {
             'IN0000008O','IN0000009M','IN0000028E','IN000002CM'];
 }
 
-/*
- *   计算收益月度相关性
- * 
- *   @param ret1 <TABLE>: NEED COLUMN entity_id, price_date, ret
- *   @param ret2 <TABLE>: NEED COLUMN entity_id, price_date, ret
- *   @param win <DURATION>: 1y, 3y, 5y
- *   
- *   NOTE: price_date 应统一为各周中的同一天（比如周收益都用 price_date.weekEnd(4) 转化为周五， 月收益都用 price_date.monthEnd() 转为月末日
- */
-def cal_monthly_correlation(ret1, ret2, win) {
 
-    t =  SELECT r1.price_date, tmcorr(r1.price_date, r1.ret, r2.ret, win) AS corr
-         FROM ret1 r1 
-         LEFT JOIN ret2 r2 ON r1.price_date = r2.price_date
-         ORDER BY r1.price_date;
+defg regressionT(y, x) {
+
+	r = SELECT beta, tstat FROM  ols(y, x, true, 1) WHERE rowNo(beta) = 1;
 
-    return SELECT price_date.month()[0] AS end_date, corr.last() AS corr FROM t  WHERE corr IS NOT NULL GROUP BY price_date.month();
+	return r[0]['tstat'];
+}
 
+/*
+ *   计算 bfi-matching 所需要的数据指标（月度）
+ * 
+ * 
+ */
+def cal_monthly_closity(ret1, ret2, win) {
+
+    t0 = SELECT end_date.month() AS end_date, end_date as price_date, ret1.ret AS ret1, ret2.ret AS ret2, tmoving(count, end_date, end_date, win) AS ret_count
+         FROM ret1
+         INNER JOIN ret2 ON ret1.end_date = ret2.end_date
+         ORDER BY end_date;
+
+    t = SELECT end_date, price_date,
+               tmcorr(t0.end_date, ret1, ret2, win) AS corr,
+               iif(tmstd(end_date, ret1-ret2, win) == 0, null, tmavg(end_date, ret1-ret2, win)\tmstd(end_date, ret1-ret2, win)) AS info,
+               tmoving(regressionT, end_date, [ret1, ret2], win) AS t_value,
+               tmbeta(end_date, ret1, ret2, win) AS beta // 用 ols() 算的值和这个一样
+        FROM t0
+        WHERE ret_count >= get_min_threshold('ret_count')
+        ORDER BY end_date;
+
+    UPDATE t SET corr = NULL WHERE corr < get_min_threshold('correlation');
+
+    return SELECT end_date.month().last() AS end_date,
+                  corr.last() AS corr,
+                  info.last() * sqrt(get_annulization_multiple('w')) AS info, // annuulized info ratio
+                  t_value.last() AS t_value,
+                  beta.last() AS beta
+           FROM t
+           GROUP BY end_date.month();
 }
 
 /*
  *   计算目标和BFI所用指数因子的相关系数
  *   
+ *   @param entity_info <TABLE>: NEED COLUMNS entity_id, inception_date, price_date
+ *   
+ *   TODO: correlation is OK; beta, info, t_value are way off!
+ *   
  *   NOTE: 与Java把月末日期作为截止日期不同的是，这里用每月最后一个周五作为截止日，所以数值会与MySQL中存储的略为不同
  * 
  */
 def cal_entity_index_coe(entity_type, entity_info) {
 
+// entity_info = get_fund_info(['MF00003PW1', 'MF00003PWC']).join(take(2024.10.31, 2) AS price_date).rename!('fund_id', 'entity_id');
+
 
     if(entity_info.isVoid() || entity_info.size() == 0) return null;
 
@@ -70,27 +112,35 @@ def cal_entity_index_coe(entity_type, entity_info) {
 	if(ret_index.isVoid() || ret_index.size() == 0) return null;
 
     // 两次循环遍历所有entity和指数
-    entity_coe = table(1000:0, ['entity_id', 'index_id', 'end_date', 'coe_1y', 'coe_3y', 'coe_5y'],
-                               [SYMBOL, SYMBOL, MONTH, DOUBLE, DOUBLE, DOUBLE]);
+    entity_coe = table(1000:0, ['entity_id', 'index_id', 'end_date', 'coe_1y', 'coe_3y', 'coe_5y', 'info_ratio_1y', 'info_ratio_3y', 'info_ratio_5y',
+                                                                     't_value_1y', 't_value_3y', 't_value_5y', 'beta_1y', 'beta_3y', 'beta_5y'],
+                               [SYMBOL, SYMBOL, MONTH, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+                                                       DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
 
     for(entity in entity_info.entity_id) {
 
+        ret1 = SELECT fund_id AS entity_id, price_date.weekEnd(4) AS end_date, price_date.weekEnd(4) AS price_date, ret_1w AS ret 
+               FROM ret_entity WHERE fund_id = entity AND price_date.weekEnd(4) <= end_day;
+    	
     	for(index in v_indexes) {
 
-            ret1 = SELECT fund_id, price_date.weekEnd(4) AS price_date, ret_1w AS ret FROM ret_entity WHERE fund_id = entity AND price_date.weekEnd(4) <= end_day;
-            ret2 = SELECT index_id, price_date.weekEnd(4) AS price_date, ret_1w AS ret FROM ret_index WHERE index_id = index AND price_date.weekEnd(4) <= end_day;
-
-    		corr_1y = cal_monthly_correlation(ret1, ret2, 1y);
-
-    		corr_3y = cal_monthly_correlation(ret1, ret2, 3y);
+            ret2 = SELECT index_id AS benchmark_id, price_date.weekEnd(4) AS end_date, price_date.weekEnd(4) AS price_date, ret_1w AS ret 
+                   FROM ret_index WHERE index_id = index AND price_date.weekEnd(4) <= end_day;
 
-    		corr_5y = cal_monthly_correlation(ret1, ret2, 5y);
+            benchmarks = table(take(entity, ret1.size()) AS entity_id, take(index, ret1.size()) AS benchmark_id, ret1.price_date.weekEnd(4) AS end_date);
+            
+            closity_1y = cal_monthly_closity(ret1, ret2, 1y);
+            closity_3y = cal_monthly_closity(ret1, ret2, 3y);
+            closity_5y = cal_monthly_closity(ret1, ret2, 5y);
 
     		INSERT INTO entity_coe
-        	    SELECT entity, index, corr_1y.end_date, corr_1y.corr AS coe_1y, corr_3y.corr AS coe_3y, corr_5y.corr AS coe_5y
-                FROM corr_1y
-                LEFT JOIN corr_3y ON corr_1y.end_date = corr_3y.end_date
-                LEFT JOIN corr_5y ON corr_1y.end_date = corr_5y.end_date;
+        	    SELECT entity, index, c1.end_date, c1.corr AS coe_1y, c3.corr AS coe_3y, c5.corr AS coe_5y, 
+        	                                       c1.info AS info_ratio_1y, c3.info AS info_ratio_3y, c5.info AS info_ratio_5y,
+        	                                       c1.t_value AS t_value_1y, c3.t_value AS t_value_3y, c5.t_value AS t_value_5y,
+        	                                       c1.beta AS beta_1y, c3.beta AS beta_3y, c5.beta AS beta_5y
+                FROM closity_1y c1
+                LEFT JOIN closity_3y  c3 ON c1.end_date = c3.end_date
+                LEFT JOIN closity_5y c5 ON c1.end_date = c5.end_date;
     	}
     }
 

modules/indicatorCalculator.dos

@@ -34,7 +34,7 @@ defg aggCVaR(returns, method, confidenceLevel) {
 }
 
 /*
- *   回撤
+ *   最大回撤
  * 
  * 
  */
@@ -43,6 +43,14 @@ defg maxDrawdown(navs) {
 	return max(1 - navs \ cummax(navs));
 }
 
+/*
+ *   几何平均值
+ * 
+ */
+defg geometricMean(x){
+	
+    return x.log().avg().exp()
+}
 
 /*
  *   Trailing Monthly Return, Standard Deviation, Skewness, Kurtosis, Max Drawdown, VaR, CVaR, Calmar Ratio

modules/navCalculator.dos

@@ -1,7 +1,9 @@
 module fundit::navCalculator
 
 use fundit::sqlUtilities
-use fundit::dataPuller
+use fundit::operationDataPuller
+use fundit::performanceDataPuller
+use fundit::portfolioDataPuller
 
 
 /*

modules/rbsaCalculator.dos

@@ -0,0 +1,221 @@
+module fundit::rbsaCalculator
+
+use fundit::performanceDataPuller
+use fundit::operationDataPuller
+
+/*
+ *   RBSA 计算
+ *   @param: ret <TABLE>: historical return (double) vector which contains the same number of return as index
+ *           index_ret <TABLE>: historical index return table  which each row is an index
+ *           is_long <BOOL>: true - long-only, false - long-short
+ *   @return: table
+ * 
+ *   Create  20240703  模仿python代码在Dolphin中实现，具体计算逻辑完全不懂                          Joey
+ *                     原代码见： http://gogs.fundit.cn/FundIt/FinanceCalcPython/src/dev36/pf_scical/v1/calc_rbsa_use_osqp.py
+ *                     Python官方示例见：https://osqp.org/docs/examples/least-squares.html
+ *                     Dolphin官方示例见：https://docs.dolphindb.cn/zh/funcs/o/osqp.html
+ *
+ */
+defg cal_rbsa(ret, index_ret, is_long) {
+
+    // 窗口长度
+    m = ret.size()
+    // 指数个数
+    n = index_ret.cols()
+    
+    P0 = matrix(float, n, m+n)
+    P1 = concatMatrix([matrix(float, m, n), eye(m)])
+    P = concatMatrix([P0, P1], false)
+    q = array(float, m+n, (m+n)*10, 0)
+    
+    A0 = concatMatrix( [matrix(index_ret), -eye(m)])
+    A1 = concatMatrix( [matrix(take(1, n)).transpose(), matrix(float, 1, m)])
+    A2 = concatMatrix( [eye(n), matrix(float, n, m)])
+    A = concatMatrix( [A0, A1, A2], false)
+    
+    // join 1 是为了限制所有权重加总为100%
+    // 下限
+    lb =(ret join 1) join array(float, n, n*10, iif(is_long == true, 0, -2))
+    // 上限
+    ub=(ret join 1) join array(float, n, n*10, iif(is_long == true, 1, 2))
+    
+    res = osqp( q, P, A, lb, ub)
+    
+    return res
+ }
+
+
+/*
+ * 滚动 rbsa
+ * @param ret <TABLE>: return table, at least with "effective_date" and "ret" as columns
+ * @param index_ret <TABLE>: index return table, with "effective_date" and all index ids as columns
+ * @param is_long <BOOL>: boolean. true means weightings could be negative values
+ * @param window <INT>: number of return in a window
+ * @param step <INT>: rolling step
+ * 
+ * TODO: use rolling()
+ * 
+ * @return <TABLE> with "effective_date", "index_id" and "weights" columns
+ */
+def cal_rolling_rbsa(ret, index_ret, is_long, window, step) {
+
+	// 找到所有指数全有数据的最早日期
+    v_start_date = EXEC effective_date.max() AS start_date 
+                   FROM (SELECT entity_id, effective_date.min() AS effective_date FROM index_ret WHERE ret IS NOT NULL GROUP BY entity_id);
+
+	m_index_ret = SELECT ret FROM index_ret WHERE effective_date >= v_start_date PIVOT BY effective_date, entity_id;
+	
+    t = SELECT * FROM ej(ret, m_index_ret, 'effective_date') ORDER BY ret.effective_date;
+    t.nullFill!(0)
+
+    // not sure why this doesn't work
+    // rolling(cal_rbsa{,,is_long}, (t.ret, t.slice(, ret.cols():).matrix()), window, step)
+
+    // 指数个数
+    n = m_index_ret.cols() - 1
+
+    // 计算起始位置
+    i = (t.size() - window) % step
+
+    // 运行rbsa计算次数
+    cnt = (t.size() - i - window) / step;
+    
+    tb = table(max(cnt,1):0, ["effective_date", "price_date", "index_id", "weights", "alpha", "r2", "adj_r2"], [STRING, DATE, STRING, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
+
+    if(t.size() >= max(window, step) && cnt > 0) {
+    	
+        do {
+
+			alpha = 0;
+        	r2 = 0;
+        	adj_r2 = 0;
+
+            v_ret = t.ret[i:(i+window)];
+            
+            t_index_ret = t.slice( i:(i+window), ret.cols(): );
+        
+            // 传入window个收益
+            res = cal_rbsa(v_ret, t_index_ret, is_long);
+
+            if(res[0] == 'solved') {
+
+				m_predict_ret = t_index_ret.matrix() ** res[1][0:n];
+				
+				alpha = v_ret.mean() - m_predict_ret.mean();
+				SSR = sum2(m_predict_ret - v_ret.mean());
+				SST = sum2(v_ret - v_ret.mean());
+				if(SST == 0) {
+					// 当SST=0, 先计算SSE再计算SST
+					SSE = sum2(v_ret - m_predict_ret);
+					SST = SSE + SSR;
+				}
+				
+				if(SST != 0) {
+					r2 = SSR/SST;
+					adj_r2 = 1 - (1 - r2) * (window - 1) / (window - n - 1);
+				}
+				
+				for(j in 1..n) {
+	    	        tb.tableInsert(t.effective_date[i+window-1], t.price_date[i+window-1], m_index_ret.colNames()[j], res[1][j-1].round(4), alpha, r2, adj_r2);
+				}
+            }
+
+            // 往前推进step个收益
+            i = i + step
+            
+            cnt -= 1
+    
+        } while( cnt >= 0)
+        
+    } else {
+    
+        tb.tableInsert(null, "error", "The number of returns must be greater than window size.")
+    }
+
+    return tb
+
+}
+
+/*
+ *  计算单基金或组合的RBSA
+ * 
+ *  @param entity_type <STRING>: 目标基金/组合的类型
+ *  @param entity_id <STRING>: 目标基金/组合的ID
+ *  @param index_ids <VECTOR>: 基准指数IDs
+ *  @param freq <STRING>: m, w, d
+ *  @param start_day <DATE>
+ *  @param end_day <DATE>
+ *  @param is_long <BOOL>: 是否只考虑纯多头
+ *  @param window <INT>: 窗口（必须多于基准指数个数）
+ *  @param step <INT>: 步长
+ *  
+ *  @return <TABLE>: entity_id, effective_date, price_date, index_id, weights, alternative_id, level, alpha, r2, adj_r2
+ *  
+ *  Example: cal_entity_RBSA('MF', 'MF00003PW1', ['IN00000008', 'IN00000077', 'IN0000007G', 'IN0000009M'], 'w', 1900.01.01, 2024.11.15, true, 24, 24);
+ *           cal_entity_RBSA('PF', 166002, ['FA00000VML', 'FA00000VMM', 'FA00000VMN', 'FA00000VMO', 'IN0000007G'], 'w', 2020.01.01, 2024.11.08, true, 24, 24);
+ *           cal_entity_RBSA('MF', 'MF000200KQ', ['IN00000008', 'IN00000077', 'IN0000007G', 'IN0000009M'], 'w', 1900.01.01, 2024.11.16, true, 24, 24);
+ */
+def cal_entity_RBSA(entity_type, entity_id, index_ids, freq='w', start_day=1900.01.01, end_day=2099.12.31, is_long=true, window=24, step=24) {
+// entity_type='MF'
+// entity_id= 'MF00003PW1'
+// index_ids=['IN00000008', 'IN00000077', 'IN0000007G', 'IN0000009M']
+// freq='w'
+// start_day=2001.01.19
+// end_day=2024.11.16
+// is_long=true
+// window=48
+// step=13
+
+    tb_result = table(100:0, ["entity_id", "effective_date", "index_id", "weights", "alpha", "r2", "adj_r2"], 
+                             [iif(entity_type=='PF', INT, STRING), STRING, STRING, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
+
+    v_entity = array(iif(entity_type=='PF', INT, STRING));
+    v_entity.append!(entity_id);
+    
+	entity_ret = get_entity_return(entity_type, v_entity, freq, start_day, end_day, true);
+
+	// 数据长度不够，按照顺序依次分别用母基金(4), 指数(3)的数据来代替
+	level = 1
+	alternative_id = NULL;
+	if(entity_ret.isVoid() || entity_ret.size() < window) {
+		if(entity_type IN ['MF', 'HF']) {
+
+			fund_info = get_fund_info(v_entity);
+			p_fund_id = fund_info.p_fund_id[0];
+			primary_benchmark_id = fund_info.benchmark_id[0];
+			if(p_fund_id != NULL) {
+				entity_ret = get_entity_return(entity_type, v_entity.replace(entity_id, p_fund_id) , freq, start_day, end_day, true);
+				alternative_id = p_fund_id;
+				level = 4;
+			} else if(primary_benchmark_id != NULL) {
+				entity_ret = get_entity_return(entity_type, v_entity.replace(entity_id, primary_benchmark_id) , freq, start_day, end_day, true);
+				alternative_id = primary_benchmark_id;
+				level = 3;
+			} else {
+				return tb_result;
+			}
+		} else if(entity_type == 'PF'){
+
+			portfolio_info = get_portfolio_info(v_entity);
+			primary_benchmark_id = portfolio_info.benchmark_id[0];
+			if(primary_benchmark_id != NULL) {
+				entity_ret = get_entity_return(entity_type, v_entity.replace(entity_id, primary_benchmark_id) , freq, start_day, end_day, true);
+				alternative_id  = primary_benchmark_id;
+				level = 3;
+			} else
+				return tb_result;
+		}
+	}
+
+    // 因为用来做基准指数的可能是指数、因子、基金等等任何时间序列数据，所以不用填 entity_type
+    index_ret = get_entity_return(NULL, index_ids, freq, start_day, end_day, true);
+
+	if(index_ret.isVoid() || index_ret.size() == 0) return tb_result;
+
+	tb_result = SELECT entity_id, effective_date, price_date, index_id, weights, alternative_id, level, alpha, r2, adj_r2
+	            FROM cal_rolling_rbsa(entity_ret, index_ret, is_long, window, step);
+
+	return tb_result;
+	
+}
+

modules/returnCalculator.dos

@@ -1,7 +1,7 @@
 module fundit::returnCalculator
 
-use fundit::fundCalculator
-use fundit::dataPuller
+use fundit::operationDataPuller
+use fundit::performanceDataPuller
 
 
 /*
@@ -208,8 +208,6 @@ def cal_weekly_returns(entity_type, entity_info){
                  ORDER BY entity_id, year_week;
     
     return tb_rets_1w;
-
-
 }
 
 /*