小修小补

modules/dataPuller.dos

@@ -746,3 +746,211 @@ def get_benchmark_return(benchmarks, end_day) {
 
 	return t_bmk;
 }
+
+
+/*
+ *  【Morningstar Integration】取某时间后净值更新的公募基金列表
+ *
+ *  @param entity_ids <STRING|VECTOR>:
+ *  @param update_time <DATETIME>: all updates after this time
+ *
+ *  Example： ms_get_fund_list_by_nav_createtime(['MF00003PW1','MF00003PWC'], 2024.10.26);
+ */
+def ms_get_fund_list_by_nav_createtime(entity_ids, createtime) {
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+    sql_entity_id = '';
+
+    if(s_entity_ids != NULL) {
+    	sql_entity_id = " AND fund_id IN (" + s_entity_ids + ")";
+    }
+
+    s_query = "SELECT fund_id AS entity_id, MIN(price_date) AS price_date 
+               FROM raw_db.public_nav2
+               WHERE isvalid = 1 " +
+                 sql_entity_id + "
+                 AND cumulative_nav > 0
+                 AND createtime >= '" + createtime$STRING + "'
+               GROUP BY fund_id
+               ORDER BY fund_id, price_date";
+    
+    conn = connect_mysql();
+
+    t = odbc::query(conn, s_query);
+    
+    conn.close();
+
+    return t
+
+}
+
+
+/*
+ * 【Morningstar Integration】 
+ *
+ * Example: ms_get_fund_info("'MF00003PW1','MF00003PWC'");
+ * 
+ */
+def ms_get_fund_info(fund_ids) {
+
+    s_entity_ids = ids_to_string(fund_ids);
+    
+    if(s_entity_ids == NULL || s_entity_ids == '') return null;
+
+    s_query = "SELECT fi.fund_id, fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value, fs.strategy, fs.substrategy
+               FROM raw_db.fund_information2 fi
+               INNER JOIN raw_db.fund_strategy2 fs ON fi.fund_id = fs.fund_id AND fs.isvalid = 1
+               WHERE fi.fund_id IN (" + s_entity_ids + ")
+               AND fi.isvalid = 1
+               ORDER BY fi.fund_id"
+
+    conn = connect_mysql()
+
+    t = odbc::query(conn, s_query)
+
+    conn.close()
+
+    return t
+
+}
+
+
+/*
+ * 【Morningstar Integration】 
+ *
+ * Example: ms_get_fund_monthly_nav(['MF00003PW1','MF00003PWC']);
+ * 
+ */
+def ms_get_fund_monthly_nav(fund_ids) {
+
+    s_entity_ids = ids_to_string(fund_ids);
+    
+    if(s_entity_ids == NULL || s_entity_ids == '') return null;
+
+    s_query = "SELECT n.fund_id AS entity_id, n.price_date, n.cumulative_nav
+               FROM raw_db.public_nav2 n INNER JOIN (
+                   SELECT fund_id, max(price_date) AS monthend_date
+                   FROM raw_db.public_nav2
+                   WHERE fund_id IN (" + s_entity_ids + ")
+                   AND isvalid = 1
+                   AND cumulative_nav > 0
+                   GROUP BY fund_id, DATE_FORMAT(price_date, '%Y-%m')
+                 ) t ON n.fund_id = t.fund_id AND n.price_date = t.monthend_date
+               UNION
+               SELECT fi.fund_id, fi.inception_date, IFNULL(fi.initial_unit_value, 1)
+               FROM raw_db.fund_information2 fi
+               WHERE fi.fund_id IN (" + s_entity_ids + ")
+               ORDER BY entity_id, price_date;"
+
+    conn = connect_mysql()
+
+    t = odbc::query(conn, s_query)
+
+    conn.close()
+
+    return t
+
+}
+
+/*
+ *  【Morningstar Integration】取某时间段的基金主基准
+ *  NOTE: 目前数据库里只存最新的基准，以后很可能会支持时间序列
+ * 
+ *  Example: ms_get_fund_primary_benchmark("'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ */
+def ms_get_fund_primary_benchmark(fund_ids, month_start, month_end) {
+
+    s_query = "SELECT fund_id, primary_benchmark_id AS benchmark_id, inception_date
+               FROM raw_db.fund_information2
+               WHERE fund_id IN (" + fund_ids + ")
+                 AND isvalid = 1;";
+
+    conn = connect_mysql();
+
+    t = odbc::query(conn, s_query);
+
+    conn.close();
+
+    t.addColumn('end_date', MONTH);
+    m_start = temporalParse(month_start, 'yyyy-MM');
+    m_end = temporalParse(month_end, 'yyyy-MM');
+    tb_end_date = table(m_start..m_end AS end_date);
+
+    return (SELECT t.fund_id, d.end_date, t.benchmark_id FROM t JOIN tb_end_date d WHERE d.end_date >= t.inception_date.month());
+
+}
+
+
+/*
+ *  【Morningstar Integration】取某时间段的基金组合主基准
+ *
+ * 
+ *  Example: ms_get_entity_primary_benchmark('MF', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ *           ms_get_entity_primary_benchmark('PF', [166002,166114], '1990-01', '2024-08');
+ */
+def ms_get_entity_primary_benchmark(entity_type, entity_ids, month_start, month_end) {
+
+    t = null;
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
+
+    if(entity_type == 'MF' || entity_type == 'HF') {
+
+    	t = ms_get_fund_primary_benchmark(s_entity_ids, month_start, month_end);
+
+        t.rename!('fund_id', 'entity_id');
+
+    } else if(entity_type == 'PF') {
+
+    	t = get_portfolio_primary_benchmark(s_entity_ids, month_start, month_end);
+
+        t.rename!('portfolio_id', 'entity_id');    
+    }
+
+	return t;
+	
+}
+
+/*
+ * 【Morningstar Integration】取无风险月度利率
+ *
+ * ms_get_risk_free_rate(1990.01.01, today())
+ */
+def ms_get_risk_free_rate(start_date, end_date) {
+    
+    return get_monthly_ret('MI', "'IN000002EI'", start_date, end_date, true);
+}
+
+/*
+ *  【Morningstar Integration】取某时间段的基金同类平均指数
+ *  NOTE: 目前数据库里只存最新的基准，以后很可能会支持时间序列
+ * 
+ *  Example: ms_get_fund_category_average("'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ */
+def ms_get_fund_category_average(fund_ids, month_start, month_end) {
+
+    s_query = "SELECT fi.fund_id, ip.index_id AS benchmark_id, fi.inception_date
+               FROM raw_db.fund_information2 fi
+               INNER JOIN mfdb.indexes_profile ip ON ip.index_code = concat('MSMWCA.2.', fi.pub_sub_fund_type)
+               WHERE fi.fund_id IN (" + fund_ids + ")
+                 AND fi.isvalid = 1
+                 AND ip.isvalid = 1;";
+
+    conn = connect_mysql();
+
+    t = odbc::query(conn, s_query);
+
+    conn.close();
+
+    t.addColumn('end_date', MONTH);
+    m_start = temporalParse(month_start, 'yyyy-MM');
+    m_end = temporalParse(month_end, 'yyyy-MM');
+    tb_end_date = table(m_start..m_end AS end_date);
+
+    return (SELECT t.fund_id, d.end_date, t.benchmark_id FROM t JOIN tb_end_date d WHERE d.end_date >= t.inception_date.month());
+
+}
+

modules/indicatorCalculator.dos

@@ -1289,3 +1289,83 @@ def cal_portfolio_bfi_indicators(portfolio_ids, end_day, cal_method, isFromNav)
     return dict(v_table_name, t0);
 
 }
+
+
+
+/*
+ *   【Morningstar Integration】通用月度指标计算
+ * 
+ *   @param entity_type <STRING>:
+ *   @param indicator_type <STRING>: PBI, BFI
+ *   @param monthly_returns <TABLE>: NEED COLUMN: entity_id, end_date, price_date, nav, ret
+ * 
+ *   @return <DICT TABLE>: ['PBI-INCEP', 'PBI-YTD', 'PBI-3M', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y']
+ * 
+ */
+def ms_cal_monthly_indicators(entity_type, indicator_type, monthly_returns) {
+
+    if(find(['MF', 'HF', 'PF'], entity_type) < 0) return null;
+
+    if(monthly_returns.isVoid() || monthly_returns.size() < 1) return null;
+
+    oldest_date = EXEC price_date.min() FROM monthly_returns;
+
+    v_entity_ids = (SELECT DISTINCT entity_id FROM monthly_returns).entity_id;
+    
+    entity_info = get_entity_info(entity_type, v_entity_ids);
+    
+    if(entity_info.isVoid() || entity_info.size() == 0) { return null };
+    
+    end_day = today();
+
+    // 取基金和基准的对照表
+    if(indicator_type == 'BFI') {
+
+        benchmark = SELECT fund_id AS entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
+                    FROM get_fund_bfi_factors(v_entity_ids, oldest_date.temporalFormat('yyyy-MM'), end_day.temporalFormat('yyyy-MM'));
+
+    } else if(indicator_type == 'CAI') {
+
+        benchmark = SELECT fund_id AS entity_id, end_date.temporalParse('yyyy-MM') AS end_date, factor_id AS benchmark_id 
+                    FROM ms_get_fund_category_average(v_entity_ids, oldest_date.temporalFormat('yyyy-MM'), end_day.temporalFormat('yyyy-MM'));
+
+    } else {
+        // 主基准， 对应 xxx_info 中的 primary_benchmark_id
+        benchmark = SELECT entity_id, end_date, iif(benchmark_id.isNull(), 'IN00000008', benchmark_id) AS benchmark_id 
+                    FROM ms_get_entity_primary_benchmark(entity_type, v_entity_ids, oldest_date.temporalFormat('yyyy-MM'), end_day.temporalFormat('yyyy-MM')) ;
+    	
+    }
+
+    // 取所有出现的基准月收益
+    bmk_ret = get_benchmark_return(benchmark, end_day);
+
+    if(bmk_ret.isVoid() || bmk_ret.size() == 0) { return null; }
+
+    // TODO: risk free指数月收益存在fund_performance表，所以先将就用 fund_id 表示。之后统一改为更准确的名字
+    risk_free_rate = SELECT entity_id AS fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM ms_get_risk_free_rate(oldest_date, end_day);
+
+    if(risk_free_rate.isVoid() || risk_free_rate.size() == 0) { return null; }
+
+    // 指标计算
+    if(indicator_type == 'BFI') {
+
+        t0 = cal_trailing_bfi_indicators(entity_info, benchmark, end_day, monthly_returns, bmk_ret, risk_free_rate);
+
+        v_table_name = ['BFI-INCEP', 'BFI-YTD', 'BFI-3M', 'BFI-6M', 'BFI-1Y', 'BFI-2Y', 'BFI-3Y', 'BFI-4Y', 'BFI-5Y', 'BFI-10Y'];
+
+    } else if(indicator_type == 'CAI') {
+
+        t0 = cal_trailing_bfi_indicators(entity_info, benchmark, end_day, monthly_returns, bmk_ret, risk_free_rate);
+
+        v_table_name = ['CAI-INCEP', 'CAI-YTD', 'CAI-3M', 'CAI-6M', 'CAI-1Y', 'CAI-2Y', 'CAI-3Y', 'CAI-4Y', 'CAI-5Y', 'CAI-10Y'];
+    	
+    } else {
+
+        t0 = cal_trailing_indicators(entity_info, benchmark, end_day, monthly_returns, bmk_ret, risk_free_rate);
+
+        v_table_name = ['PBI-INCEP', 'PBI-YTD', 'PBI-3M', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y'];
+    }
+
+    return dict(v_table_name, t0);
+	
+}

modules/returnCalculator.dos

@@ -26,14 +26,19 @@ def cal_monthly_returns_by_nav(entity_info, mutable nav) {
     tb_monthly_nav = SELECT n.entity_id, end_date, price_date, cumulative_nav 
                      FROM tb_monthly_nav n
                      INNER JOIN entity_info ei ON n.entity_id = ei.entity_id
-                     WHERE n.price_date >= ei.inception_date
-                     ORDER BY n.entity_id, n.end_date, n.price_date;
+                     WHERE n.price_date >= ei.inception_date;
+
+    // 补回来成立日初始净值
+    INSERT INTO tb_monthly_nav
+        SELECT entity_id, inception_date.month(), inception_date, ini_value
+        FROM entity_info
+        WHERE inception_date IS NOT NULL;
 
     if(tb_monthly_nav.isVoid() || tb_monthly_nav.size() == 0) { return tb_rets; }
 
     // 计算月收益
     tb_rets = SELECT entity_id, end_date, price_date, cumulative_nav, cumulative_nav.ratios() - 1 AS ret
-              FROM tb_monthly_nav
+              FROM tb_monthly_nav.sortBy!(['entity_id', 'price_date'], [1, 1])
               CONTEXT BY entity_id;
 
     // the records without return calculated but do have nav are still useful for some calculations (e.g. max drawdown)
@@ -92,7 +97,7 @@ def mix_monthly_returns(entity_type, entity_info) {
 
 
 /*
- *  根据基金净值序列计算月收益序列（适合提供给指标运算）
+ *  【作废】根据基金净值序列计算月收益序列（适合提供给指标运算）
  * 
  *  Create:  20240907                                                  Joey
  *                    TODO: missing pulling data from local

modules/task_fundPerformance.dos

@@ -19,6 +19,8 @@ def generate_entity_performance(entity_info, indicators, isToMySQL, mutable enti
 
     if(isToMySQL) {
 
+       if(indicators['PBI-3M'].isVoid() || indicators['PBI-3M'].size() == 0) return;
+
         t = SELECT entity_id, end_date, price_date, nav AS cumulative_nav, ret AS ret_1m, ret AS ret_1m_a, trailing_ret AS ret_3m, trailing_ret_a AS ret_3m_a
             FROM indicators['PBI-3M'] AS ind
             INNER JOIN entity_info fi ON ind.entity_id = fi.entity_id
@@ -77,6 +79,8 @@ def generate_entity_risk_stats(entity_info, indicators, isToMySQL, mutable entit
 
     t = null;
 
+   if(indicators['PBI-6M'].isVoid() || indicators['PBI-6M'].size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT entity_id, end_date, std_dev_a AS stddev_6m, ds_dev_a AS downsidedev_6m, alpha_a AS alpha_6m, winrate AS winrate_6m, beta AS beta_6m,
@@ -141,6 +145,8 @@ def generate_entity_riskadjret_stats(entity_info, indicators, isToMySQL, mutable
 
     t = null;
 
+   if(indicators['PBI-6M'].isVoid() || indicators['PBI-6M'].size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT entity_id, end_date,
@@ -206,6 +212,8 @@ def generate_entity_indicator(entity_info, indicators, isToMySQL, mutable entity
 
     t = null;
 
+   if(indicators['PBI-6M'].isVoid() || indicators['PBI-6M'].size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT entity_id, end_date, info_a AS info_ratio_6m, m2_a AS m2_6m, track_error_a AS tracking_error_6m
@@ -261,6 +269,8 @@ def generate_entity_style_stats(entity_info, indicators, isToMySQL, mutable enti
 
     t = null;
 
+   if(indicators['PBI-6M'].isVoid() || indicators['PBI-6M'].size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT entity_id, end_date, upside_capture_ret AS upsidecapture_ret_6m, downside_capture_ret AS downsidecapture_ret_6m,
@@ -326,6 +336,8 @@ def generate_entity_performance_weekly(entity_info, ret_w, isToMySQL, mutable en
 
     t = null;
 
+   if(ret_w.isVoid() || ret_w.size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT entity_id, year_week, year_week.left(4)$INT AS end_year, year_week.right(2)$INT AS week_of_year, price_date,
@@ -350,6 +362,8 @@ def generate_entity_latest_performance(entity_info, perf_latest, isToMySQL, muta
 
     t = null;
 
+   if(perf_latest.isVoid() || perf_latest.size() == 0) return;
+
     if(isToMySQL) {
 
         t = SELECT r.*
@@ -487,6 +501,132 @@ def calFundPerformance(entityType, date) {
 }
 
 
+
+/*
+ *   【临时】用于数据初始化：只计算收益
+ * 
+ *   @param entityType <STRING>: 'MF', 'HF'...
+ *   @param date <DATETIME>: 净值更新时间
+ * 
+ */
+def ms_calFundReturns() {
+
+    rt = '';
+
+    very_old_date = 1990.01.01;
+
+    // 取基金列表 (27s)
+    tb_cal_funds = ms_get_fund_list_by_nav_createtime(NULL, very_old_date);
+
+    if(tb_cal_funds.isVoid() || tb_cal_funds.size() == 0 ) return;
+
+    tb_fund_performance = create_entity_performance();
+
+    tb_fund_indicator = create_entity_indicator();
+    tb_fund_risk_stats = create_entity_risk_stats();
+    tb_fund_riskadjret_stats = create_entity_riskadjret_stats();
+    tb_fund_style_stats = create_entity_style_stats();
+
+    tb_fund_performance_weekly = create_entity_performance_weekly();
+    tb_fund_latest_performance = create_entity_latest_performance();
+
+    // 分批跑
+    i = 0;
+    batch_size = 1000;
+
+
+    do {
+
+        funds = tb_cal_funds[i : min(tb_cal_funds.size(), i+batch_size)];
+
+        if(funds.isVoid() || funds.size() == 0) break;
+
+        // 200ms
+        fund_info = SELECT entity_id, price_date, inception_date, benchmark_id, ini_value 
+                    FROM ej(funds, ms_get_fund_info(funds.entity_id), 'entity_id', 'fund_id');
+
+        // 计算月收益 (19s)
+        tb_nav = ms_get_fund_monthly_nav(fund_info.entity_id);
+
+        rets = cal_monthly_returns_by_nav(fund_info, tb_nav);
+
+        if(!rets.isVoid() && rets.size() > 0) {
+
+            // 计算月度指标 (67s)
+            rets.rename!('cumulative_nav', 'nav');
+            indicators = cal_monthly_indicators('MF', 'PBI', rets);
+
+            // 仿照MySQL的表结构准备好记录 （1s)
+            generate_entity_performance(fund_info, indicators, true, tb_fund_performance);
+
+            generate_entity_indicator(fund_info, indicators, true, tb_fund_indicator);
+            generate_entity_risk_stats(fund_info, indicators, true, tb_fund_risk_stats);
+            generate_entity_riskadjret_stats(fund_info, indicators, true, tb_fund_riskadjret_stats);
+            generate_entity_style_stats(fund_info, indicators, true, tb_fund_style_stats);
+
+        }
+        
+        // 计算周收益 (49s)
+        rets_w = cal_weekly_returns('MF', fund_info);
+
+        if(! rets_w.isVoid() && rets_w.size() > 0) {
+            generate_entity_performance_weekly(fund_info, rets_w, true, tb_fund_performance_weekly);
+        }
+
+        // 计算最新收益 (23s)
+        perf_latest = cal_latest_performance('MF', fund_info, true);
+
+        if(! perf_latest.isVoid() && perf_latest.size() > 0) {
+            generate_entity_latest_performance(fund_info, perf_latest, true, tb_fund_latest_performance);
+        }
+
+        i += batch_size;
+
+//    } while (i < batch_size);
+    } while (i <= tb_cal_funds.size());
+
+
+    if(! tb_fund_performance.isVoid() && tb_fund_performance.size() > 0) {
+
+        // save data to MySQL  (26m)
+        try {
+
+            chg_columns_for_mysql(tb_fund_performance, 'fund_id');
+            save_and_sync(tb_fund_performance, 'raw_db.fund_performance', 'raw_db.fund_performance');
+
+            chg_columns_for_mysql(tb_fund_indicator, 'fund_id');
+            save_and_sync(tb_fund_indicator, 'raw_db.fund_indicator', 'raw_db.fund_indicator');
+
+            chg_columns_for_mysql(tb_fund_risk_stats, 'fund_id');
+            // mfdb.fund_performance 表中 maxdrawdown_6m & maxdrawdown_ytd 是虚拟列，这里用数据列顺序强行写入真实列 6m_maxdrawdown & ytd_maxdrawdown (DolphinDB 不允许字段名以数字开头)
+            save_and_sync(tb_fund_risk_stats, 'raw_db.fund_risk_stats', 'raw_db.fund_risk_stats');
+
+            chg_columns_for_mysql(tb_fund_riskadjret_stats, 'fund_id');
+            save_and_sync(tb_fund_riskadjret_stats, 'raw_db.fund_riskadjret_stats', 'raw_db.fund_riskadjret_stats');
+
+            chg_columns_for_mysql(tb_fund_style_stats, 'fund_id');
+            save_and_sync(tb_fund_style_stats, 'raw_db.fund_style_stats', 'raw_db.fund_style_stats');
+
+            save_and_sync(tb_fund_performance_weekly, 'raw_db.fund_performance_weekly', 'raw_db.fund_performance_weekly');
+
+            save_and_sync(tb_fund_latest_performance, 'raw_db.fund_latest_performance', 'raw_db.fund_latest_nav_performance');
+
+        } catch(ex) {
+
+            //TODO: Log errors
+            rt = ex;
+        }
+    }
+    
+    return rt;
+	
+}
+
+/*
+ *   实验性质的API
+ * 
+ * 
+ */
 def calFundIndexCorrelation(entityType, date) {
 
     if(find(['HF', 'MF'], entityType) < 0) return null;
@@ -501,4 +641,4 @@ def calFundIndexCorrelation(entityType, date) {
     coe = cal_entity_index_coe(entityType, tb_cal_funds[0:1000]);
 
     return coe;
-}
+}