支持指标计算定时任务

modules/dataPuller.dos

@@ -97,16 +97,16 @@ def get_portfolio_weekly_rets(portfolio_ids, start_date, end_date, isFromMySQL)
     return t
 }
 
+/*
+ *  取月收益
+ * 
+ *  Example: get_monthly_ret('FD', ['HF000004KN','HF000103EU','HF00018WXG'], 2000.01.01, 2024.03.01, true);
+ */
 def get_monthly_ret(entity_type, entity_ids, start_date, end_date, isFromMySQL) {
 
-    s_entity_ids = '';
-    
-    // 判断输入的 fund_ids 是字符串标量还是向量
-    if ( entity_ids.form() == 0 ) {
-        s_entity_ids = entity_ids;
-    } else {
-        s_entity_ids = "'" + entity_ids.concat("','") + "'";
-    }
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
 
     tmp = get_performance_table_description(entity_type);
 
@@ -204,25 +204,21 @@ def get_fund_latest_nav_performance(fund_ids, isFromMySQL) {
 }
 
 /*
- * 取私募基金净值
+ * 取基金净值
  * 
  * 
  * Create: 202408                                                    Joey
  *                 TODO: add isvalid and nav > 0 for local version
  * 
  *
- * Example: get_nav_by_price_date('HF', "'HF000004KN','HF00018WXG'", 2024.05.01, true)
+ * Example: get_nav_by_price_date('HF', "'HF000004KN','HF00018WXG'", 2024.05.01, true);
+ *          get_nav_by_price_date('IN', "'IN00000008','IN0000000M'", 2024.05.01, true);
  */
 def get_nav_by_price_date(entity_type, entity_ids, price_date, isFromMySQL) {
 
-    s_entity_ids = '';
-    
-    // 判断输入的 fund_ids 是字符串标量还是向量
-    if ( entity_ids.form() == 0 ) {
-        s_entity_ids = entity_ids;
-    } else {
-        s_entity_ids = "'" + entity_ids.concat("','") + "'";
-    }
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
 
     tmp = get_nav_table_description(entity_type);
 
@@ -305,15 +301,20 @@ def get_index_nav_by_price_date(index_ids, price_date) {
 /*
  * 取有效基金基本信息
  *
- * Example: get_fund_info("'HF000004KN','HF00018WXG'")
+ * Example: get_fund_info("'HF000004KN','HF00018WXG'");
+ *          get_fund_info(['HF000004KN','HF00018WXG']);
  * 
  */
 def get_fund_info(fund_ids) {
 
+    s_entity_ids = ids_to_string(fund_ids);
+    
+    if(s_entity_ids == NULL || s_entity_ids == '') return null;
+
     s_query = "SELECT fi.fund_id, fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value, fs.strategy, fs.substrategy
                FROM mfdb.fund_information fi
                INNER JOIN mfdb.fund_strategy fs ON fi.fund_id = fs.fund_id AND fs.isvalid = 1
-               WHERE fi.fund_id IN (" + fund_ids + ")
+               WHERE fi.fund_id IN (" + s_entity_ids + ")
                AND fi.isvalid = 1
                ORDER BY fi.fund_id"
 
@@ -335,6 +336,10 @@ def get_fund_info(fund_ids) {
  */
 def get_portfolio_info(portfolio_ids) {
 
+    s_entity_ids = ids_to_string(portfolio_ids);
+    
+    if(s_entity_ids == NULL || s_entity_ids == '') return null;
+
     s_query = "SELECT cpm.id AS portfolio_id, cpm.userid, cpm.customer_id, cpm.inception_date, cpm.portfolio_source, cpm.portfolio_type
                FROM pfdb.`pf_customer_portfolio_map` cpm
                INNER JOIN pfdb.cm_user u ON cpm.userid = u.userid
@@ -353,15 +358,47 @@ def get_portfolio_info(portfolio_ids) {
 }
 
 /*
- * 取私募基金净值更新信息, 返回基金及其净值更新的最早净值日期
+ *   取基金组合基础有效信息
+ * 
+ *   Example: get_entity_info('FD', ['HF000004KN','HF000103EU','HF00018WXG']);
+ *            get_entity_info('PF', '166002,166114');
+ */
+def get_entity_info(entity_type, entity_ids) {
+
+    t = null;
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
+
+    if(entity_type == 'FD') {
+
+    	t = get_fund_info(s_entity_ids);
+
+        t.rename!('fund_id', 'entity_id');
+
+    } else if(entity_type == 'PF') {
+
+    	t = get_portfolio_info(s_entity_ids);
+
+        t.rename!('portfolio_id', 'entity_id');    
+    }
+
+	return t;
+}
+
+
+/*
+ * 取基金净值更新信息, 返回基金及其净值更新的最早净值日期
  *
  * @param fund_ids: fund_id STRING VECTOR
  * @param update_time: all updates after this time
  *
- * Example: get_fund_list_by_nav_updatetime(null, 2024.07.19T10:00:00)
+ * Example: get_fund_list_by_nav_updatetime('MF', null, 2024.09.26);
+ *          get_fund_list_by_nav_updatetime('HF', null, 2024.07.19T10:00:00)
  * 
  */
-def get_fund_list_by_nav_updatetime(fund_ids, updatetime) {
+def get_fund_list_by_nav_updatetime(entity_type, fund_ids, updatetime) {
 
     s_fund_sql = '';
     // 这里要用 isVoid， 因为 isNull对向量返回的是布尔向量
@@ -369,11 +406,17 @@ def get_fund_list_by_nav_updatetime(fund_ids, updatetime) {
         s_fund_ids = fund_ids.concat("','");
         s_fund_sql = " AND fi.fund_id IN ('" + s_fund_ids + "')";
     }
+
+    if(entity_type == 'HF') {
+    	nav_table = 'mfdb.nav';
+    } else {
+    	nav_table = 'mfdb.public_nav';
+    }
     
     s_query = "SELECT fi.fund_id, MIN(nav.price_date) AS price_date,
                       fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value
                FROM mfdb.fund_information fi
-               INNER JOIN mfdb.nav ON fi.fund_id = nav.fund_id
+               INNER JOIN " + nav_table + " nav ON fi.fund_id = nav.fund_id
                WHERE fi.isvalid = 1" +
                  s_fund_sql + "
                  AND nav.cumulative_nav > 0
@@ -471,6 +514,37 @@ def get_portfolio_primary_benchmark(portfolio_ids, month_start, month_end) {
 
 }
 
+/*
+ *  取某时间段的基金组合主基准
+ *
+ * 
+ *  Example: get_entity_primary_benchmark('FD', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ *           get_entity_primary_benchmark('PF', [166002,166114], '1990-01', '2024-08');
+ */
+def get_entity_primary_benchmark(entity_type, entity_ids, month_start, month_end) {
+
+    t = null;
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
+
+    if(entity_type == 'FD') {
+
+    	t = get_fund_primary_benchmark(s_entity_ids, month_start, month_end);
+
+        t.rename!('fund_id', 'entity_id');
+
+    } else if(entity_type == 'PF') {
+
+    	t = get_portfolio_primary_benchmark(s_entity_ids, month_start, month_end);
+
+        t.rename!('portfolio_id', 'entity_id');    
+    }
+
+	return t;
+	
+}
 
 /*
  *  取某时间段的基金BFI因子
@@ -523,6 +597,39 @@ def get_portfolio_bfi_factors(portfolio_ids, month_start, month_end) {
 
 }
 
+
+/*
+ *  取某时间段的基金组合BFI基准
+ *
+ * 
+ *  Example: get_entity_bfi_factors('FD', "'MF00003PW2', 'MF00003PW1', 'MF00003PXO'", '1990-01', '2024-06');
+ *           get_entity_bfi_factors('PF', [166002,166114], '1990-01', '2024-08');
+ */
+def get_entity_bfi_factors(entity_type, entity_ids, month_start, month_end) {
+
+    t = null;
+
+    s_entity_ids = ids_to_string(entity_ids);
+
+    if(s_entity_ids == null || s_entity_ids == '') return null;
+
+    if(entity_type == 'FD') {
+
+    	t = get_fund_bfi_factors(s_entity_ids, month_start, month_end);
+
+        t.rename!('fund_id', 'entity_id');
+
+    } else if(entity_type == 'PF') {
+
+    	t = get_portfolio_bfi_factors(s_entity_ids, month_start, month_end);
+
+        t.rename!('portfolio_id', 'entity_id');    
+    }
+
+	return t;
+	
+}
+
 /*
  * 取组合交易表
  *
@@ -548,13 +655,16 @@ def get_portfolio_holding_history(portfolio_ids) {
 }
 
 /*
- *  取基金证券从某日期后的所有净值
- *  @param json_query <JSON>: [{sec_id:xxx, holding_date: yyyy-mm-dd}]
+ *  取基金证券从某日期后的所有净值及前值
+ *  
+ *  @param entity_type <STRING>: MF, HF, EQ, CF, MI, TI, CI, FA, PF
+ *  @param freq <STRING>: m, w, d
+ *  @param json_query <JSON>: [{sec_id:xxx, price_date: yyyy-mm-dd}]
  * 
  */
-def get_holding_nav(json_query) {
+def get_nav_for_return_calculation(entity_type, freq, json_query) {
 
-    s_query = "CALL pfdb.sp_get_nav_after_date('" + json_query + "')";
+    s_query = "CALL pfdb.sp_get_nav_after_date('" + entity_type + "', '" + freq + "', '" + json_query + "')";
 
     conn = connect_mysql();
 
@@ -564,3 +674,37 @@ def get_holding_nav(json_query) {
 
     return t;
 }
+
+/*
+ *   取主基准和BFI的历史月收益率
+ *   
+ *   @param benchmarks <TABLE>: entity-benchmark 的对应关系表
+ *   @param end_day <DATE>: 收益的截止日期
+ * 
+ *   @return <TABLE>: benchmark_id, end_date, ret
+ *   
+ */
+def get_benchmark_return(benchmarks, end_day) {
+
+    s_index_ids = '';
+    s_factor_ids = '';
+
+    if(benchmarks.isVoid() || benchmarks.size() == 0) { return null; }
+
+    // 前缀为 IN 的 benchmark id
+    t_index_id = SELECT DISTINCT benchmark_id FROM benchmarks WHERE benchmark_id LIKE 'IN%';
+    s_index_ids = iif(isVoid(t_index_id), "", "'" + t_index_id.benchmark_id.concat("','") + "'");
+
+   // 前缀为 FA 的 benchmark id
+   t_factor_id = SELECT DISTINCT benchmark_id FROM benchmarks WHERE benchmark_id LIKE 'FA%';
+   s_factor_ids = iif(isVoid(t_factor_id), "",  "'" + t_factor_id.benchmark_id.concat("','") + "'");
+
+
+    // 目前指数的月度业绩存在 fund_performance 表
+    t_bmk = SELECT fund_id AS benchmark_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('IX', s_index_ids, 1990.01.01, end_day, true);
+
+    // 而因子的月度业绩存在 cm_factor_performance 表
+    INSERT INTO t_bmk SELECT factor_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('FA', s_factor_ids, 1990.01.01, end_day, true);
+
+	return t_bmk;
+}

modules/navCalculator.dos

@@ -45,7 +45,7 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
 }
 
 /*
- *  计算FOF类组合收益
+ *  计算FOF类组合净值
  *  NOTE: 与MySQL逻辑一致，用户界面输入的交易净值会被暂时忽略，因为我们无法确保同一基金同一时间被输入的净值是相同的；
  *        忽略手工净值会导致收益不精确或无法计算的问题，但可能错误的净值将导致错误的结果，两害取其轻。
  *  
@@ -58,9 +58,9 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
  *  
  *  
  */
- def cal_portfolio_return(portfolio_ids, start_date, cal_method) {
+ def cal_portfolio_nav(portfolio_ids, start_date, cal_method) {
 
-    // 取持仓截面
+    // 取持仓截面get_nav_for_return_calculation
     tb_snapshot = convert_transaction_to_snapshot(portfolio_ids, today()).rename!('fund_id', 'sec_id');
 
     // 取涉及到的所有基金证券最早持仓日期
@@ -68,7 +68,7 @@ def convert_transaction_to_snapshot(portfolio_ids, end_day) {
 
     // 取涉及到的所有基金证券有用净值
     // TODO: need consider inception date nav
-    tb_nav = get_holding_nav(s_json);
+    tb_nav = get_nav_for_return_calculation('PF', 'd', s_json);
 
     // 补一下最新界面
     tb_latest_snapshot = SELECT sec_id, holding_date, nav.mean().round(6) AS nav

modules/returnCalculator.dos

@@ -1,8 +1,52 @@
 module fundit::returnCalculator
+
 use fundit::fundCalculator
 use fundit::dataPuller
 
 /*
+ *  通用月收益计算
+ * 
+ *  @param entity_type <STRING>: MF, HF, EQ, CF, MI, TI, CI, FA, PF
+ *  @param entity_info <TABLE>: COLUMN NEED entity_id, price_date, inception_date, ini_value
+ * 
+ */
+def cal_entity_monthly_returns(entity_type, entity_info) {
+
+    tb_rets = null;
+
+    // 取涉及到的所有基金证券最早持仓日期
+    s_json = (SELECT entity_id AS sec_id, price_date.min() AS price_date FROM entity_info GROUP BY entity_id).toStdJson();
+
+	// 取涉及到的所有基金证券有用净值(前值+某日期后所有净值)
+    // TODO: need consider inception date nav
+    tb_nav = get_nav_for_return_calculation(entity_type, 'm', s_json);
+
+
+    // 填充好各基金有效期内所有月份的最后一天
+    tb_monthly_nav = SELECT sec_id, price_date.month().last() AS end_date, price_date.last() AS price_date, cumulative_nav.last() AS cumulative_nav
+                     FROM tb_nav.sortBy!(['sec_id', 'price_date'], [1, 1])
+                     GROUP BY sec_id, price_date.month();
+
+    // 删掉成立日之前的净值
+    tb_monthly_nav = SELECT sec_id AS entity_id, end_date, price_date, cumulative_nav 
+                     FROM tb_monthly_nav n // ej(tb_monthly_nav, entity_info, 'sec_id', 'entity_id')
+                     INNER JOIN entity_info ei ON n.sec_id = ei.entity_id
+                     WHERE n.price_date >= ei.inception_date
+                     ORDER BY n.sec_id, n.end_date, n.price_date;
+
+    if(tb_monthly_nav.isVoid() || tb_monthly_nav.size() == 0) { return tb_rets; }
+
+    // 计算月收益
+    tb_rets = SELECT entity_id, end_date, price_date, cumulative_nav, cumulative_nav.ratios() - 1 AS ret
+              FROM tb_monthly_nav
+              CONTEXT BY entity_id;
+
+
+    // the records without return calculated but do have nav are still useful for some calculations (e.g. max drawdown)
+    return ( SELECT * FROM tb_rets WHERE cumulative_nav > 0 );
+}
+
+/*
  *  根据基金净值序列计算月收益序列（适合提供给指标运算）
  * 
  *  Create:  20240907                                                  Joey

modules/task_fundPerformance.dos

@@ -0,0 +1,148 @@
+module fundit::task_fundPerformance
+
+use fundit::fundCalculator
+use fundit::dataPuller
+use fundit::returnCalculator
+use fundit::indicatorCalculator
+
+
+/*
+ *  通用收益计算
+ * 
+ * 
+ */
+def calEntityReturns(entityType, freq, entities) {
+
+    ret = null;
+
+    if(freq == 'm') { ret = fundit::returnCalculator::cal_entity_monthly_returns(entityType, entities); }
+
+    return ret;
+
+}
+
+
+
+/*
+ *   通用指标计算
+ * 
+ * 
+ * 
+ *   @return <DICT TABLE>: ['PBI-INCEP', 'PBI-YTD', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y', 'MS-3Y', 'MS-5Y', 'MS-10Y']
+ * 
+ */
+def calIndicators(entityType, monthlyReturns) {
+
+    if(find(['FD', 'PF'], entityType) < 0) return null;
+
+    if(monthlyReturns.isVoid() || monthlyReturns.size() < 1) return null;
+
+    oldest_date = EXEC price_date.min() FROM monthlyReturns;
+
+    v_entity_ids = (SELECT DISTINCT entity_id FROM monthlyReturns).entity_id;
+    
+    entity_info = get_entity_info(entityType, v_entity_ids);
+    
+    if(entity_info.isVoid() || entity_info.size() == 0) { return null };
+    
+    if(monthlyReturns.isVoid() || monthlyReturns.size() == 0) { return null; }
+
+    end_day = today();
+
+    // 取基金和基准的对照表
+    primary_benchmark = SELECT entity_id, end_date, iif(benchmark_id.isNull(), 'IN00000008', benchmark_id) AS benchmark_id 
+                        FROM get_entity_primary_benchmark(entityType, v_entity_ids, oldest_date.month().temporalFormat('yyyy-MM'), end_day.month().temporalFormat('yyyy-MM')) ;
+
+    // 取所有出现的基准月收益
+    bmk_ret = get_benchmark_return(primary_benchmark, end_day);
+
+    if(bmk_ret.isVoid() || bmk_ret.size() == 0) { return null; }
+
+    risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(oldest_date, end_day);
+
+    if(risk_free_rate.isVoid() || risk_free_rate.size() == 0) { return null; }
+
+    // 标准的指标
+    t0 = cal_trailing_indicators(entity_info, primary_benchmark, end_day, monthlyReturns, bmk_ret, risk_free_rate);
+
+    // PBI stands for "Primary Benchmark Index"
+    v_table_name = ['PBI-INCEP', 'PBI-YTD', 'PBI-6M', 'PBI-1Y', 'PBI-2Y', 'PBI-3Y', 'PBI-4Y', 'PBI-5Y', 'PBI-10Y'];
+
+    // BFI指标
+
+    return dict(v_table_name, t0);
+	
+}
+
+/*
+ *   基金指标计算
+ * 
+ * 
+ */
+def calFundIndicators(rets) {
+
+	t = calIndicators('FD', rets);
+
+	return t;
+}
+
+
+/*
+ *   定时任务：最新净值触发的业绩指标计算
+ * 
+ *   TODO: 目前收益表在MySQL中，所以需要将计算的最新收益与MySQL中的历史数据合并
+ */
+def calFundPerformance(entityType, date) {
+
+    very_old_date = 1990.01.01;
+
+    if(find(['HF', 'MF'], entityType) < 0) return null;
+
+    // 取有最新净值变动的私募基金列表
+    tb_cal_funds = get_fund_list_by_nav_updatetime('MF', NULL, date);
+
+    // 分批跑
+    i = 0;
+    batch_size = 1000;
+    do {
+
+        funds = tb_cal_funds[i:batch_size]
+        funds.rename!('fund_id', 'entity_id');
+
+        // 计算月收益
+        rets = calEntityReturns(entityType, 'm', funds);
+
+        // 最新更新的收益存入数据库
+
+        // 取完整历史收益用于指标计算
+        all_rets = get_fund_monthly_ret(funds.entity_id, very_old_date, today().month(), true);
+        all_rets.rename!('fund_id', 'entity_id');
+
+        // 将MySQL中的 YYYY-MM 格式改为 YYYY.MM
+        end_dates = EXEC end_date.temporalParse('yyyy-MM') FROM all_rets;
+        all_rets.replaceColumn!('end_date', end_dates);
+
+        // 用新计算的收益覆盖旧的
+        UPDATE all_rets
+            SET all_rets.ret = rets.ret, all_rets.nav = rets.cumulative_nav
+        FROM ej(all_rets, rets, ['entity_id', 'end_date']);
+
+        // 用新计算的收益补充旧的， ret_ytd_a, ret_incep_a 没有用
+        INSERT INTO all_rets 
+            SELECT entity_id, end_date, price_date, ret, cumulative_nav, null AS ret_ytd_a, null AS ret_incep_a
+            FROM rets
+            WHERE NOT EXISTS ( SELECT * FROM all_rets WHERE entity_id = rets.entity_id AND end_date = rets.end_date);
+
+        // 计算月度指标
+        indicators = calFundIndicators(all_rets);
+
+        //INSERT INTO tb_indicators SELECT * FROM indicators;
+
+        i += batch_size;
+
+    } while (i < batch_size);
+//    } while (i <= tb_cal_funds.size());
+
+    return indicators;
+	
+}