完善一下

modules/indicatorCalculator.dos

@@ -118,11 +118,10 @@ def cal_basic_performance(entity_info, ret, trailing_month) {
     // accumulate 版的 skewness, kurtosis, var, cvar 似乎都不对劲，只好找个笨办法来实现
     if(trailing_month == 'incep') {
         // 需要至少6个数才计算标准差、峰度、偏度
-        t0 = SELECT 
+        t0 = SELECT price_date.max() AS price_date,
                     prod(1+ret)-1 AS trailing_ret, 
                     prod(1+ret)-1 AS trailing_ret_a,
                     iif(count(entity_id) > 5, std(ret), null) AS std_dev,
-                    iif(count(entity_id) > 5, std(ret)*sqrt(12), null) AS std_dev_a,
                     iif(count(entity_id) > 5, skew(ret, false), null) AS skewness,
                     iif(count(entity_id) > 5, kurtosis(ret, false), null)-3 AS kurtosis,
                     min(ret) AS wrst_month
@@ -156,12 +155,12 @@ def cal_basic_performance(entity_info, ret, trailing_month) {
 
     } else if(trailing_month == 'ytd') {
 
-        t1 = SELECT entity_id, end_date,
+        t1 = SELECT entity_id, end_date, price_date.cummax() AS price_date,
                     cumprod(1+ret)-1 AS trailing_ret,
                     cumprod(1+ret)-1 AS trailing_ret_a, // no need annulization for ytd
                     iif(cumcount(entity_id) > 5, cumstd(ret), null) AS std_dev,
                     iif(cumcount(entity_id) > 5, tmoving(skew{, false}, end_date, ret, 12), null) AS skewness,
-                    iif(cumcount(entity_id) > 5, tmoving(kurtosis{, false}, end_date, ret, 12), null) AS kurtosis,
+                    iif(cumcount(entity_id) > 5, tmoving(kurtosis{, false}, end_date, ret, 12)-3, null) AS kurtosis,
                     cummin(ret) AS wrst_month,
                     maxDrawdown(nav) AS drawdown
         FROM ret WHERE ret > -1
@@ -173,7 +172,7 @@ def cal_basic_performance(entity_info, ret, trailing_month) {
         	
         win = trailing_month$INT;
         
-        t1 = SELECT entity_id, end_date, 
+        t1 = SELECT entity_id, end_date, price_date.mmax(win) AS price_date,
                     mprod(1+ret, win)-1 AS trailing_ret,
                     iif(trailing_month > 12,
                         mprod(1+ret, win).pow(12\win)-1,
@@ -269,6 +268,7 @@ def cal_omega_sortino_kappa(ret, risk_free, trailing_month) {
               FROM ret t
               INNER JOIN lpm l ON t.entity_id = l.entity_id AND t.end_date = l.end_date
               INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+              WHERE t.ret > -1
               CONTEXT BY t.entity_id;
 
     } else if(trailing_month == 'ytd') {
@@ -281,6 +281,7 @@ def cal_omega_sortino_kappa(ret, risk_free, trailing_month) {
               FROM ret t
               INNER JOIN lpm l ON t.entity_id = l.entity_id AND t.end_date = l.end_date
               INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+              WHERE t.ret > -1
               CONTEXT BY t.entity_id, t.end_date.year();
 
     } else {
@@ -295,6 +296,7 @@ def cal_omega_sortino_kappa(ret, risk_free, trailing_month) {
               FROM ret t
               INNER JOIN lpm l ON t.entity_id = l.entity_id AND t.end_date = l.end_date
               INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+              WHERE t.ret > -1
               CONTEXT BY t.entity_id;
     }
 
@@ -304,6 +306,9 @@ def cal_omega_sortino_kappa(ret, risk_free, trailing_month) {
 
 /*
  *    Winning Ratio, Tracking Error, Information Ratio
+ *    
+ *    NOTE: mcount is very unique in mFun, because it doesn't support minPeriods(BUG?), while others default minPeriods = window.
+ *          As a result, we have to live with lots of records having winrate but no tracking error and info ratio
  *   
  *    TODO: Win Rate incept is off, because Java incorrectly takes all end_date as denominator even when benchmark has no price
  *          Information Ratio is way off!
@@ -326,7 +331,7 @@ def cal_benchmark_tracking(ret, benchmarks, bmk_ret, trailing_month) {
         t = SELECT entity_id, end_date, benchmark_id,
                    cumcount(iif(exc_ret >= 0, 1, null)) \ cnt AS winrate,
                    exc_ret.cumstd() AS track_error, 
-                   iif(exc_ret.cumstd() == 0, null, exc_ret.cumavg() / exc_ret.cumstd()) AS info
+                   iif(exc_ret.cumstd() == 0, null, exc_ret.cumavg() \ exc_ret.cumstd()) AS info
             FROM t0 
             CONTEXT BY entity_id, benchmark_id
             ORDER BY entity_id, end_date, benchmark_id;
@@ -346,7 +351,7 @@ def cal_benchmark_tracking(ret, benchmarks, bmk_ret, trailing_month) {
         t = SELECT entity_id, end_date, benchmark_id,
                    cumcount(iif(exc_ret >= 0, 1, null)) \ cnt AS winrate,
                    exc_ret.cumstd() AS track_error, 
-                   iif(exc_ret.cumstd() == 0, null, exc_ret.cumavg() / exc_ret.cumstd()) AS info
+                   iif(exc_ret.cumstd() == 0, null, exc_ret.cumavg() \ exc_ret.cumstd()) AS info
             FROM t0 
             CONTEXT BY entity_id, benchmark_id, end_date.year()
             ORDER BY entity_id, end_date, benchmark_id;
@@ -365,9 +370,9 @@ def cal_benchmark_tracking(ret, benchmarks, bmk_ret, trailing_month) {
              CONTEXT BY t.entity_id, bm.benchmark_id;
 
         t = SELECT entity_id, end_date, benchmark_id,
-                   mcount(iif(exc_ret >= 0, 1, null), win) \ cnt AS winrate,
-                   exc_ret.mstd(win) AS track_error, 
-                   iif(exc_ret.mstd(win) == 0, null, exc_ret.mavg(win) / exc_ret.mstd(win)) AS info
+                   mcount(iif(exc_ret >= 0, 1, null), win) \ cnt AS winrate, 
+                   mstd(exc_ret, win) AS track_error, 
+                   iif(mstd(exc_ret, win) == 0, null, mavg(exc_ret, win) \ mstd(exc_ret, win)) AS info
             FROM t0 
             CONTEXT BY entity_id, benchmark_id
             ORDER BY entity_id, end_date, benchmark_id;
@@ -442,7 +447,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
 
     if(trailing_month == 'incep') {
 
-        t1 = SELECT t.entity_id, t.end_date, (1+t.ret).cumprod() AS upside_ret, (1+bmk.ret).cumprod() AS bmk_upside_ret, bmk.end_date.cumcount() AS bmk_upside_cnt, bm.benchmark_id
+        t1 = SELECT t.entity_id, t.end_date, 
+                    (1+t.ret).cumprod() AS upside_ret, (1+bmk.ret).cumprod() AS bmk_upside_ret, bmk.end_date.cumcount() AS bmk_upside_cnt, bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -450,7 +456,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
                AND bmk.ret >= 0
              CONTEXT BY t.entity_id, bm.benchmark_id;
     
-        t2 = SELECT t.entity_id, t.end_date, (1+t.ret).cumprod() AS downside_ret, (1+bmk.ret).cumprod() AS bmk_downside_ret, bmk.end_date.cumcount() AS bmk_downside_cnt, bm.benchmark_id
+        t2 = SELECT t.entity_id, t.end_date, 
+                    (1+t.ret).cumprod() AS downside_ret, (1+bmk.ret).cumprod() AS bmk_downside_ret, bmk.end_date.cumcount() AS bmk_downside_cnt, bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -460,7 +467,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
 
     } else if(trailing_month == 'ytd') {
 
-        t1 = SELECT t.entity_id, t.end_date, (1+t.ret).cumprod() AS upside_ret, (1+bmk.ret).cumprod() AS bmk_upside_ret, bmk.end_date.cumcount() AS bmk_upside_cnt, bm.benchmark_id
+        t1 = SELECT t.entity_id, t.end_date,
+                    (1+t.ret).cumprod() AS upside_ret, (1+bmk.ret).cumprod() AS bmk_upside_ret, bmk.end_date.cumcount() AS bmk_upside_cnt, bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -468,7 +476,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
                AND bmk.ret >= 0
              CONTEXT BY t.entity_id, bm.benchmark_id, t.end_date.year();
     
-        t2 = SELECT t.entity_id, t.end_date, (1+t.ret).cumprod() AS downside_ret, (1+bmk.ret).cumprod() AS bmk_downside_ret, bmk.end_date.cumcount() AS bmk_downside_cnt, bm.benchmark_id
+        t2 = SELECT t.entity_id, t.end_date,
+                    (1+t.ret).cumprod() AS downside_ret, (1+bmk.ret).cumprod() AS bmk_downside_ret, bmk.end_date.cumcount() AS bmk_downside_cnt, bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -480,8 +489,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
 
         win = trailing_month$INT;
 
-        t1 = SELECT t.entity_id, t.end_date, bm.benchmark_id,
-                    (1+t.ret).mprod(win) AS upside_ret, (1+bmk.ret).mprod(win) AS bmk_upside_ret, bmk.end_date.mcount(win) AS bmk_upside_cnt
+        t1 = SELECT t.entity_id, t.end_date,
+                    (1+t.ret).mprod(win) AS upside_ret, (1+bmk.ret).mprod(win) AS bmk_upside_ret, bmk.end_date.mcount(win) AS bmk_upside_cnt, bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -489,8 +498,8 @@ def cal_capture_ratio(ret, benchmarks, bmk_ret, trailing_month) {
                AND bmk.ret >= 0
              CONTEXT BY t.entity_id, bm.benchmark_id;
     
-        t2 = SELECT t.entity_id, t.end_date, bm.benchmark_id,
-                    (1+t.ret).mprod(win) AS downside_ret, (1+bmk.ret).mprod(win) AS bmk_downside_ret, bmk.end_date.mcount(win) AS bmk_downside_cnt
+        t2 = SELECT t.entity_id, t.end_date,
+                    (1+t.ret).mprod(win) AS downside_ret, (1+bmk.ret).mprod(win) AS bmk_downside_ret, bmk.end_date.mcount(win) AS bmk_downside_cnt,bm.benchmark_id
              FROM ret t
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id AND t.end_date = bmk.end_date
@@ -525,7 +534,7 @@ def cal_sharpe(ret, std_dev, risk_free, trailing_month) {
                  FROM ret t
                  INNER JOIN std_dev std ON t.entity_id = std.entity_id AND t.end_date = std.end_date
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
-                 WHERE std.std_dev <> 0
+                 WHERE std.std_dev <> 0 AND t.ret > -1
                  CONTEXT BY t.entity_id;
                  
     } else if(trailing_month == 'ytd') {
@@ -534,7 +543,7 @@ def cal_sharpe(ret, std_dev, risk_free, trailing_month) {
                  FROM ret t
                  INNER JOIN std_dev std ON t.entity_id = std.entity_id AND t.end_date = std.end_date
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
-                 WHERE std.std_dev <> 0
+                 WHERE std.std_dev <> 0 AND t.ret > -1
                  CONTEXT BY t.entity_id, t.end_date.year();
     } else {
 
@@ -544,7 +553,7 @@ def cal_sharpe(ret, std_dev, risk_free, trailing_month) {
                  FROM ret t
                  INNER JOIN std_dev std ON t.entity_id = std.entity_id AND t.end_date = std.end_date
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
-                 WHERE std.std_dev <> 0
+                 WHERE std.std_dev <> 0 AND t.ret > -1
                  CONTEXT BY t.entity_id;
     }
 
@@ -621,6 +630,7 @@ def cal_jensen(ret, bmk_ret, risk_free, beta, trailing_month) {
                  INNER JOIN beta beta ON t.entity_id = beta.entity_id AND t.end_date = beta.end_date
                  INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND beta.benchmark_id = bmk.benchmark_id
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+                 WHERE t.ret > -1
                  CONTEXT BY t.entity_id, beta.benchmark_id;
 
     } else if(trailing_month == 'ytd') {
@@ -630,6 +640,7 @@ def cal_jensen(ret, bmk_ret, risk_free, beta, trailing_month) {
                  INNER JOIN beta beta ON t.entity_id = beta.entity_id AND t.end_date = beta.end_date
                  INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND beta.benchmark_id = bmk.benchmark_id
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+                 WHERE t.ret > -1
                  CONTEXT BY t.entity_id, beta.benchmark_id, t.end_date.year();
 
     } else {
@@ -641,6 +652,7 @@ def cal_jensen(ret, bmk_ret, risk_free, beta, trailing_month) {
                  INNER JOIN beta beta ON t.entity_id = beta.entity_id AND t.end_date = beta.end_date
                  INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND beta.benchmark_id = bmk.benchmark_id
                  INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+                 WHERE t.ret > -1
                  CONTEXT BY t.entity_id, beta.benchmark_id;
 
     }
@@ -662,6 +674,7 @@ def cal_m2(ret, benchmarks, bmk_ret, risk_free, trailing_month) {
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id
              INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+             WHERE t.ret > -1
              CONTEXT BY t.entity_id, bm.benchmark_id;
 
     } else if(trailing_month == 'ytd') {
@@ -671,6 +684,7 @@ def cal_m2(ret, benchmarks, bmk_ret, risk_free, trailing_month) {
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id
              INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+             WHERE t.ret > -1
              CONTEXT BY t.entity_id, bm.benchmark_id, t.end_date.year();
 
     } else {
@@ -682,6 +696,7 @@ def cal_m2(ret, benchmarks, bmk_ret, risk_free, trailing_month) {
              INNER JOIN benchmarks bm ON t.entity_id = bm.entity_id AND t.end_date = bm.end_date
              INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date AND bm.benchmark_id = bmk.benchmark_id
              INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+             WHERE t.ret > -1
              CONTEXT BY t.entity_id, bm.benchmark_id;
 
     }
@@ -704,39 +719,101 @@ def cal_ms_return(ret, risk_free, trailing_month) {
 
     win = trailing_month$INT;
 
-    r = SELECT t.entity_id, t.end_date, t.price_date.mmax(win) AS price_date, t.price_date.mmin(win) AS min_date,
+    r = SELECT t.entity_id, t.end_date,
                ((1 + t.ret)\(1 + rfr.ret)).mprod(win).pow(12\(t.end_date.mmax(win) - t.end_date.mmin(win)))-1 AS ms_ret_a,
                (1 + t.ret).pow(-2).mavg(win).pow(-12/2)-1 AS ms_rar_a
         FROM ret t
         INNER JOIN risk_free rfr ON t.end_date = rfr.end_date
+        WHERE t.ret > -1
         CONTEXT BY t.entity_id;
 
     return r;
 }
 
 /*
+ *   有效主体-基准对应表
+ * 
+ *   @param benchmarks <TABLE>: entity-benchmark 的对应关系表 NEED COLUMNS: entity_id, end_date, benchmark_id
+ *   @param end_day <DATE>:
+ *   @param trailing_month <STRING>: 
+ *   @param isEffectiveOnly <BOOL>: false时与Java相同； true：多了个限制条件：如果区间内有效基准数少于1/2，不做计算
+ *                                  比如过去12个月中某BFI只出现2次，小于需要的6次，此BFI不参与 trailing 1 year 计算
+ * 
+ */
+def get_effective_benchmarks(benchmarks, end_day, trailing_month, isEffectiveOnly) {
+
+    min_pct = 0.5;
+
+    if(isEffectiveOnly) {
+
+      	t_dates = SELECT DISTINCT entity_id, end_date FROM benchmarks WHERE end_date <= end_day.month();
+
+        if(trailing_month == 'incep') {
+
+        	t = SELECT entity_id, end_date, end_date.cumcount() AS cnt FROM t_dates CONTEXT BY entity_id;
+        	
+        	bmk = SELECT bmk.* FROM benchmarks bmk
+        	      INNER JOIN t ON bmk.entity_id = t.entity_id AND bmk.end_date = t.end_date
+        	      CONTEXT BY bmk.entity_id, bmk.benchmark_id
+        	      HAVING bmk.end_date.cumcount() >= t.cnt * min_pct;
+
+        } else if(trailing_month == 'ytd') {
+
+        	t = SELECT entity_id, end_date, end_date.cumcount() AS cnt FROM t_dates CONTEXT BY entity_id, end_date.year();
+        	
+        	bmk = SELECT bmk.* FROM benchmarks bmk
+        	      INNER JOIN t ON bmk.entity_id = t.entity_id AND bmk.end_date = t.end_date
+        	      CONTEXT BY entity_id, benchmark_id, end_date.year()
+        	      HAVING bmk.end_date.cumcount() >= t.cnt * min_pct;
+        } else {
+
+        	win = trailing_month$INT;
+
+        	t = SELECT entity_id, end_date, end_date.mcount(win) AS cnt FROM t_dates CONTEXT BY entity_id;
+        	
+        	bmk = SELECT bmk.* FROM benchmarks bmk
+        	      INNER JOIN t ON bmk.entity_id = t.entity_id AND bmk.end_date = t.end_date
+        	      CONTEXT BY entity_id, benchmark_id
+        	      HAVING bmk.end_date.mcount(win) >= t.cnt * min_pct;
+        }
+
+    } else {
+
+    	bmk = SELECT * FROM benchmarks WHERE end_date <= end_day.month();
+    }
+
+    return bmk;
+}
+
+
+/*
  *   Calculation for monthly indicators which need benchmark
  *   
  *   @param entity_info <TABLE>: xxx_information表，NEED COLUMNS entity_id, inception_date
+ *   @param benchmark_mapping <TABLE>: entity-benchmark mapping table, NEED COLUMNS entity_id, end_date, benchmark_id
+ *   @param end_day <DATE>;
  *   @param tb_ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav
- *   @param benchmarks <TABLE>: entity-benchmark mapping table 
  *   @param index_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
  *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
  *   @param month <INT>: trailing x month
  *   
  *   @return: indicators table
  *    
- *    
  *   
  *   Create  20240904  模仿Java & python代码在Dolphin中实现，具体计算逻辑可能会有不同                          Joey
  *                     TODO: some datapoints require more data, we need a way to disable calculation for them
  *
  */
-def cal_indicators_with_benchmark(entity_id, tb_ret, benchmarks, index_ret, risk_free, month) {
+def cal_indicators_with_benchmark(entity_info, benchmark_mapping, end_day, tb_ret, index_ret, risk_free, month) {
 
     // sorting for correct first() and last() value
-    ret = SELECT * FROM tb_ret ORDER BY entity_id, price_date;
-    
+    ret = SELECT * FROM tb_ret WHERE ret > -1 AND end_date <= end_day.month() ORDER BY entity_id, price_date;
+
+    // get the effective benchmarks
+    benchmarks = get_effective_benchmarks(benchmark_mapping, end_day, month, true);
+
+    if(ret.isVoid() || ret.size() == 0 || benchmarks.isVoid() || benchmarks.size() == 0) return null;
+
     // alpha, beta
     alpha_beta = cal_alpha_beta(ret, benchmarks, index_ret, risk_free, month);
 
@@ -771,33 +848,37 @@ def cal_indicators_with_benchmark(entity_id, tb_ret, benchmarks, index_ret, risk
                 [DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
 
     UPDATE r 
-      SET alpha_a = alpha * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1),
-          jensen_a = jensen * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1),
-          track_error_a = track_error * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
-          info_a = info * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
-          m2_a = m2 * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1);
+      SET alpha_a = alpha * iif(end_date - inception_date.month() > 12, plainAnnu, 1),
+          jensen_a = jensen * iif(end_date - inception_date.month() > 12, plainAnnu, 1),
+          track_error_a = track_error * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1),
+          info_a = info * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1),
+          m2_a = m2 * iif(end_date - inception_date.month() > 12, plainAnnu, 1)
+    FROM ej(r, entity_info, 'entity_id');
     
-    return r.dropColumns!(['price_date', 'min_date']);
+    return r;
 }
 
 /*
  *   Monthly standard indicator calculation
- *   @param tb_ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
+ *   
+ *   @param entity_info <TABLE>:
  *   @param benchmarks <TABLE>: entity-benchmark mapping table
+ *   @param end_day <DATE>:
+ *   @param tb_ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
  *   @param benchmark_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
  *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
+ *   @param month <STRING>:
  *   
  *   @return: indicators table
  *   
  *   
  *   Create  20240904  模仿Java & python代码在Dolphin中实现，具体计算逻辑可能会有不同                          Joey
- *                     TODO: some datapoints require more data, we need a way to disable calculation for them
  *
  */
-def cal_indicators(entity_info, tb_ret, benchmarks, benchmark_ret, risk_free, month) {
+def cal_indicators(entity_info, benchmarks, end_day, tb_ret, benchmark_ret, risk_free, month) {
 
     // sorting for correct first() and last() value
-    ret = SELECT * FROM tb_ret ORDER BY entity_id, price_date;
+    ret = SELECT * FROM tb_ret WHERE end_date <= end_day.month() ORDER BY entity_id, price_date;
 
     // 收益、标准差、偏度、峰度、最大回撤、VaR, CVaR、卡玛比率
     rtn = cal_basic_performance(entity_info, ret, month);
@@ -809,13 +890,21 @@ def cal_indicators(entity_info, tb_ret, benchmarks, benchmark_ret, risk_free, mo
     lpms = cal_omega_sortino_kappa(ret, risk_free, month);
 
     // 需要基准的指标们
-    indicator_with_benchmark = cal_indicators_with_benchmark(entity_info, ret, benchmarks, benchmark_ret, risk_free, month);
+    indicator_with_benchmark = cal_indicators_with_benchmark(entity_info, benchmarks, end_day, ret, benchmark_ret, risk_free, month);
 
     r = SELECT * FROM rtn a1
                  LEFT JOIN sharpe ON a1.entity_id = sharpe.entity_id AND a1.end_date = sharpe.end_date
                  LEFT JOIN lpms ON a1.entity_id = lpms.entity_id AND a1.end_date = lpms.end_date
                  LEFT JOIN indicator_with_benchmark bmk ON a1.entity_id = bmk.entity_id AND a1.end_date = bmk.end_date;
 
+
+    // 晨星收益和风险
+    if(month$STRING in ['36', '60', '120']) {
+        ms = cal_ms_return(ret, risk_free, month);
+
+        r = SELECT * FROM r LEFT JOIN ms ON r.entity_id = ms.entity_id AND r.end_date = ms.end_date;
+    }
+
     // 年化各数据点
     // GIPS RULE: NO annulization for data less than 1 year
     plainAnnu = get_annulization_multiple('m');
@@ -825,95 +914,15 @@ def cal_indicators(entity_info, tb_ret, benchmarks, benchmark_ret, risk_free, mo
                 [DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
 
     UPDATE r 
-      SET std_dev_a = std_dev * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
-          ds_dev_a = ds_dev * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
-          sharpe_a = sharpe * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
-          sortino_a = sortino * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1);
-    
-    return r;
-}
-
-
-/*
- *   Monthly BFI indicator calculation
- *   @param ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
- *   @param benchmarks <TABLE>: entity-benchmark mapping table
- *   @param benchmark_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
- *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
- *   
- *   @return: BFI indicators table
- *   
- *   【NOTE】JAVA IMPLEMENTATION ASSUMES BFI FACTORS OF CURRENT MONTH ARE SAME AS HISTORICAL ONES!   
- *   
- *   Create  20240914                                                                 Joey
- *
- */
-def cal_bfi_indicators(entity_info, ret, benchmarks, benchmark_ret, risk_free, month) {
-
-    // 需要基准的指标们
-    r = cal_indicators_with_benchmark(entity_info, ret, benchmarks, benchmark_ret, risk_free, month);
+      SET std_dev_a = std_dev * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1),
+          ds_dev_a = ds_dev * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1),
+          sharpe_a = sharpe * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1),
+          sortino_a = sortino * iif(end_date - inception_date.month() > 12, sqrtAnnu, 1)
+    FROM ej(r, entity_info, 'entity_id');
     
     return r;
 }
 
-/*
- *   Monthly Morningstar indicator calculation
- *   
- *   @param ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
- *   @param benchmarks <USELESS>:
- *   @param benchmark_ret <USELESS>:
- *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
- * 
- */
-def cal_ms_indicators(ret, benchmarks, benchmark_ret, risk_free, month) {
-
-	r = cal_ms_return(ret, risk_free, month);
-
-	return r;
-}
-
-/*
- *   Calculate Trailing x month indicators
- * 
- *   @param: func <FUNCTION>: the calculation function
- *   @param: entity_info <TABLE>: basic information of entity, NEED COLUMNS entity_id, inception_date
- *   @param benchmarks <TABLE>: entity-benchmark mapping table
- *   @param: ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
- *   @param: end_day <DATE>: 计算截止日期
- *   @param bmk_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
- *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
- *   @param month <INT>: X month
- *   @param isCal <BOOL>: 是否计算此区间
- *   
- *   NOTE: 与Java不同，这里多了个限制条件：如果区间内有效基准数少于1/2，不做计算；
- *         比如过去12个月中某BFI只出现2次，小于需要的6次，此BFI不参与 trailing 1 year 计算
- *   
- * 
- */
-def cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, index_ret, risk_free_rate, isCal, month) {
-
-    ret = null;
-
-    ret = SELECT * FROM tb_ret r 
-          INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-          WHERE r.end_date > end_day.month() - month 
-            AND end_day.month() - ei.inception_date.month() >= month
-          ORDER BY entity_id, end_date;
-
-    bmk = SELECT * FROM benchmarks 
-          WHERE end_date > end_day.month() - month
-          CONTEXT BY entity_id, benchmark_id
-          HAVING count(end_date) >= month/2
-          ORDER BY entity_id, end_date, benchmark_id;
-
-    bmk_ret = SELECT * FROM index_ret WHERE end_date > end_day.month() - month;
-
-    if(ret.size() > 0 && bmk.size() > 0 && bmk_ret.size() > 0 && isCal == 1) {
-        ret = func(ret, bmk, bmk_ret, risk_free_rate);
-    }
-
-	return ret;
-}
 
 /*
  *   Calculate trailing 6m, ytd, 1y, 2y, 3y, 4y, 5y, 10y and since inception datapoints
@@ -921,15 +930,15 @@ def cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, index_ret
  *   @param: func <FUNCTION>: the calculation function
  *   @param: entity_info <TABLE>: basic information of entity, NEED COLUMNS entity_id, inception_date
  *   @param benchmarks <TABLE>: entity-benchmark mapping table
- *   @param: ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
  *   @param: end_day <DATE>: 计算截止日期
+ *   @param: ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
  *   @param bmk_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
  *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
  *   @param periods <BOOL VECTOR>: 是否计算的区间向量，分别对应 incep, ytd, 6m, 1y, 2y, 3y, 4y, 5y, 10y
  *   
  * 
  */
-def cal_trailing(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods) {
+def cal_trailing2(func, entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate) {
 
     r_incep = null;
     r_ytd = null;
@@ -941,92 +950,39 @@ def cal_trailing(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_f
     r_5y = null;
     r_10y = null;
 
-    // since inception
-    // TODO: 怎么给since inception加上是否计算的限制条件，暂时用3
-    if(tb_ret.size() > 0 && benchmarks.size() > 0 && periods[0] == 1) {
-
-        bmk = SELECT * FROM benchmarks 
-              CONTEXT BY entity_id, benchmark_id
-              HAVING count(end_date) >= 3
-              ORDER BY entity_id, end_date, benchmark_id;
-
-        r_incep = func(tb_ret, bmk, bmk_ret, risk_free_rate);
-    }
+    // incep
+    r_incep = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, 'incep');
 
     // ytd
-    r_ytd = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[1], end_day.month()-end_day.yearBegin().month()+1);
+    r_ytd = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, 'ytd');
 
-    // trailing 6m
-    r_6m = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[2], 6);
+    // 6m
+    r_6m = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '6');
 
-   // trailing 1y
-    r_1y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[3], 12);
-   
-   // trailing 2y
-    r_2y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[4], 24);
+    // 1y
+    r_1y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '12');
 
-   // trailing 3y
-    r_3y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[5], 36);
+    // 2y
+    r_2y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '24');
 
-   // trailing 4y
-    r_4y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[6], 48);
+    // 3y
+    r_3y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '36');
 
-   // trailing 5y
-    r_5y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[7], 60);
+    // 4y
+    r_4y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '48');
 
-   // trailing 10y
-    r_10y = cal_trailing_month(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, periods[8], 120);
+    // 5y
+    r_5y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '60');
 
-/*
-    // trailing 1y
-    tb_ret_1y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-12 AND (end_day.month() - ei.inception_date.month()) >= 12;
-    if(tb_ret_1y.size() > 0 && periods[3] == 1) {
-        r_1y = func(tb_ret_1y, benchmarks, bmk_ret, risk_free_rate, 12/2);
-    }
-    
-    // trailing 2y
-    tb_ret_2y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-24 AND (end_day.month() - ei.inception_date.month()) >= 24;
-    if(tb_ret_2y.size() > 0 && periods[4] == 1) {
-        r_2y = func(tb_ret_2y, benchmarks, bmk_ret, risk_free_rate, 24/2);
-    }
-    
-    // trailing 3y
-    tb_ret_3y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-36 AND (end_day.month() - ei.inception_date.month()) >= 36;
-    if(tb_ret_3y.size() > 0 && periods[5] == 1) {
-        r_3y = func(tb_ret_3y, benchmarks, bmk_ret, risk_free_rate, 36/2);
-    }
+    // 10y
+    r_10y = func(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate, '120');
 
-    // trailing 4y
-    tb_ret_4y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-48 AND (end_day.month() - ei.inception_date.month()) >= 48;
-    if(tb_ret_4y.size() > 0 && periods[6] == 1) {
-        r_4y = func(tb_ret_4y, benchmarks, bmk_ret, risk_free_rate, 48/2);
-    }
-    
-    // trailing 5y
-    tb_ret_5y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-60 AND (end_day.month() - ei.inception_date.month()) >= 60;
-    if(tb_ret_5y.size() > 0 && periods[7] == 1) {
-        r_5y = func(tb_ret_5y, benchmarks, bmk_ret, risk_free_rate, 60/2);
-    }
-
-    // trailing 10y
-    tb_ret_10y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
-                WHERE r.end_date > end_day.month()-120 AND (end_day.month() - ei.inception_date.month()) >= 120;
-    if(tb_ret_10y.size() > 0 && periods[8] == 1) {
-        r_10y = func(tb_ret_10y, benchmarks, bmk_ret, risk_free_rate, 120/2);
-    }
-*/
     return r_incep, r_ytd, r_6m, r_1y, r_2y, r_3y, r_4y, r_5y, r_10y;
 }
 
 
-
 /*
- *   Calculate trailing 6m, ytd, 1y, 2y, 3y, 4y, 5y, 10y and since inception standard indicators
+ *   Calculate trailing ytd, 6m, 1y, 2y, 3y, 4y, 5y, 10y and since inception standard indicators
  *   
  *   @param: entity_info <TABLE>: basic information of entity, NEED COLUMNS entity_id, inception_date
  *   @param benchmarks <TABLE>: entity-benchmark mapping table
@@ -1036,9 +992,9 @@ def cal_trailing(func, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_f
  *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
  * 
  */
-def cal_trailing_indicators(entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate) {
-
-    return cal_trailing(cal_indicators, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, [1,1,1,1,1,1,1,1,1]);
+def cal_trailing_indicators2(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate) {
+	
+    return cal_trailing2(cal_indicators, entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate);
 
 }
 
@@ -1054,20 +1010,11 @@ def cal_trailing_indicators(entity_info, benchmarks, tb_ret, end_day, bmk_ret, r
  *   
  * 
  */
-def cal_trailing_bfi_indicators(entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate) {
-
-    return cal_trailing(cal_bfi_indicators, entity_info, benchmarks, tb_ret, end_day, bmk_ret, risk_free_rate, [1,1,1,1,1,1,1,1,1]);
+def cal_trailing_bfi_indicators2(entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate) {
 
+    return cal_trailing2(cal_indicators_with_benchmark, entity_info, benchmarks, end_day, tb_ret, bmk_ret, risk_free_rate);
 }
 
-/*
- *   Calculate trailing 3y, 5y, 10y Morningstar Return, Risk-Adjested Return and Risk
- * 
- */
-def cal_trailing_ms_indicators(entity_info, tb_ret, end_day, risk_free_rate) {
-	
-    return cal_trailing(cal_ms_indicators,  entity_info, , tb_ret, end_day, , risk_free_rate, periods=[0,0,0,0,0,1,0,1,1]);
-}
 
 /*
  *   Calculate fund indicators for one date
@@ -1117,15 +1064,17 @@ def cal_fund_indicators(entity_type, fund_ids, end_day, isFromNav) {
     risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(very_old_date, end_day);
 
     // 标准的指标
-    t0 = cal_trailing_indicators(fund_info, primary_benchmark, tb_ret, end_day, bmk_ret, risk_free_rate);
+    t0 = cal_trailing_indicators2(fund_info, primary_benchmark, end_day, tb_ret, bmk_ret, risk_free_rate);
 
     // Morningstar 指标
-    t1 = cal_trailing_ms_indicators(fund_info, tb_ret, end_day, risk_free_rate);
+    //t1 = cal_trailing_ms_indicators(fund_info, tb_ret, end_day, risk_free_rate);
 
     // PBI stands for "Primary Benchmark Index", MS stands for "MorningStar"
-    v_table_name = ['PBI-INCEP', 'PBI-YTD', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y', 'MS-3Y', 'MS-5Y', 'MS-10Y'];
+    v_table_name = ['PBI-INCEP', 'PBI-YTD', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y'/*, 'MS-3Y', 'MS-5Y', 'MS-10Y'*/];
+
+    return dict(v_table_name, t0);
+//    return dict(v_table_name, t0 <- t1[5] <- t1[7] <- t1[8]);
     
-    return dict(v_table_name, t0 <- t1[5] <- t1[7] <- t1[8]);
 }
 
 /*
@@ -1177,7 +1126,7 @@ def cal_fund_bfi_indicators(entity_type, fund_ids, end_day, isFromNav) {
 
     risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(very_old_date, end_day);
 
-    t0 = cal_trailing_bfi_indicators(fund_info, bfi_benchmark, tb_ret, end_day, bmk_ret, risk_free_rate);
+    t0 = cal_trailing_bfi_indicators2(fund_info, bfi_benchmark, end_day, tb_ret, bmk_ret, risk_free_rate);
 
     // BFI stands for "Best Fit Index"
     v_table_name = ['BFI-INCEP', 'BFI-YTD', 'BFI-6M', 'BFI-1Y', 'BFI-2Y', 'BFI-3Y', 'BFI-4Y', 'BFI-5Y', 'BFI-10Y'];
@@ -1231,7 +1180,7 @@ def cal_portfolio_indicators(portfolio_ids, end_day, cal_method, isFromNav) {
 
     risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(very_old_date, end_day);
 
-    t0 = cal_trailing_indicators(portfolio_info, primary_benchmark, tb_ret, end_day, bmk_ret, risk_free_rate);
+    t0 = cal_trailing_indicators2(portfolio_info, primary_benchmark, end_day, tb_ret, bmk_ret, risk_free_rate);
 
     v_table_name = ['PBI-INCEP', 'PBI-YTD', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y'];