小修小补

modules/rankingCalculator.dos

@@ -16,7 +16,8 @@ def get_indicator_info() {
            14, 15, 16, 17, 18, 19, 40, 58,
            37, 38, 41, 42, 43, 44, 45, 46, 47, 48, 49,
            33, 34, 35, 36,
-           66, 53, 54, 55, 56, 57
+           66, 53, 54, 55, 56, 57,
+           71, 72, 73
           ];
 
     names = ['ret',
@@ -24,7 +25,8 @@ def get_indicator_info() {
              'kapparatio', 'treynorratio', 'jensen', 'omegaratio', 'sharperatio', 'sortinoratio_MAR', 'calmarratio', 'sortinoratio',
              'per_con', 'info_ratio', 'var', 'cvar', 'smddvar', 'smddcvar', 'smdd_lpm1', 'smdd_lpm2', 'smdd_downside_dev', 'tracking_error', 'm2',
              'upsidecapture_ret', 'downsidecapture_ret', 'upsidecapture_ratio', 'downsidecapture_ratio',
-             'stability', 'jc_stddev', 'gzstyle_stddev', 'gzstrategy_stddev', 'zz_stddev', 'zx_stddev'
+             'stability', 'jc_stddev', 'gzstyle_stddev', 'gzstrategy_stddev', 'zz_stddev', 'zx_stddev',
+             'ms_return', 'ms_rar', 'ms_risk'
             ];
 
     is_ASCs = [false,
@@ -32,7 +34,8 @@ def get_indicator_info() {
                false, false, false, false, false, false, false, false,
                false, false, true, true, true, true, true, true, true, true, false,
                false, false, false, true,
-               true, true, true, true, true, true
+               true, true, true, true, true, true,
+               false, false, true
               ];
 
     return table(names AS name, ids AS id, is_ASCs AS is_ASC);
@@ -73,9 +76,13 @@ def gen_ranking_sql(entity_type, data_table, indicator_table) {
 	for(indicator in indicator_table) {
 
         // 与 MySQL 不同，这里统一把近4年和成立以来的排名去掉
-		if(indicator.id == 1)
+		if(indicator.id == 1) {
 			v_trailing = ['1m', '3m', '6m', '1y', '2y', '3y', '5y', '10y', 'ytd'];
-		else {
+		// 晨星指标只有3,5,10年
+		} else if(indicator.id in (71, 72, 73)) {
+			v_trailing = ['3y', '5y', '10y'];
+			v_missing_trailing = ['1m', '3m', '6m', '1y', '2y', 'ytd'];
+		} else {
 			v_trailing = ['6m', '1y', '2y', '3y', '5y', '10y', 'ytd'];
 			v_missing_trailing = ['1m', '3m'];
 		}
@@ -268,11 +275,18 @@ def prepare_data_for_ranking(ranking_by, entity_type, entity_info, end_date, isF
 
 		tb_data = lj(tb_data, tb_data_capture_stats, matchingCols);
 
+		// morningstar data
+		table_desc = get_ms_stats_table_description(entity_type);
+		tb_data_ms_stats = get_monthly_indicator_data(table_desc.table_name[0], end_date, isFromMySQL);
+		tb_data = lj(tb_data, tb_data_ms_stats, matchingCols);
+
+
     	v_indicator_id = [1,                                // 对应 fund_performance, 取消39（年化收益) 因为没有意义
     					  38, 41, 42, 48, 49,               // 对应 fund_indicator, 取消37 (per_con), 43, 44, 45, 46, 47 (smdd模型) 因为dolphin 未计算
         	              2, 6, 9, 10, 11, 12, 21, 59,      // 对应 fund_risk_stats, 取消50, 52 因为 dolphin 未计算
             	          14, 15, 16, 17, 18, 40, 58,       // 对应 fund_riskadjret_stats 取消19 (MAR Sortino ratio) 因为 dolphin 未计算
-                	      33, 34, 35, 36                    // 对应 fund_style_stats 
+                	      33, 34, 35, 36,                   // 对应 fund_style_stats
+                	      71, 72, 73                        // 对应 fund_ms_stats
                     	 ];
 
     }
@@ -338,6 +352,11 @@ def run_transformation_sql(entity_type, data_table, ranking_by, indicator_info)
 	    // 只有收益需要1m, 3m
     	if(indicator.id == 1)
     		v_trailing = ['1m', '3m', '6m', '1y', '2y', '3y', '5y', '10y', 'ytd'];
+		// 晨星指标只有3,5,10年
+    	else if(indicator.id in (71, 72, 73)) {
+    		v_trailing = ['3y', '5y', '10y'];
+    		v_missing = ['1m', '3m', '6m', '1y', '2y', 'ytd'];
+    	}
     	else {
     		v_trailing = ['6m', '1y', '2y', '3y', '5y', '10y', 'ytd'];
     		v_missing = ['1m', '3m'];
@@ -349,7 +368,7 @@ def run_transformation_sql(entity_type, data_table, ranking_by, indicator_info)
                 from = data_table
         	   ).eval();
 
-		// 给非收益指标补上1m, 3m的三套指标
+		// 给非收益指标补上缺失的指标
         if(indicator.id != 1 )
         {
         	v_tmp_col = ['indicator_' + v_missing, 'absrank_' + v_missing, 'perrank_' + v_missing].flatten();

modules/sqlUtilities.dos

@@ -31,9 +31,9 @@ def connect_mysql(user='pf_user') {
  */
 def get_local_database(server_name, db_name) {
 
-    db = database(directory="D:/Program Files/DolphinDB/server/database/" + server_name + "/" + db_name + "/")
+    db = database(directory="dfs://" + server_name + "/" + db_name + "/");
     
-    return db
+    return db;
 }
 
 /*
@@ -304,6 +304,27 @@ def get_capture_style_table_description(entity_type) {
 
 
 /*
+ *  根据不同类型的主体返回其晨星指标的表名、字段名
+ * 
+ *  Example: get_ms_stats_table_description('MF');
+ */
+def get_ms_stats_table_description(entity_type) {
+
+    tmp_universe = table(100:0, 
+                         ['type', 'table_name', 'sec_id_col'],
+                         [STRING, STRING, STRING]);
+
+    // 分别对应：私募，公募，私有基金，市场指数，图译指数，私有指数，图译因子，组合，基金经理，基金公司
+    INSERT INTO tmp_universe VALUES ( 
+        ['HF', 'MF', 'CF', 'MI', 'FI', 'CI', 'FA', 'PF', 'PL', 'CO'],
+        ['mfdb.fund_ms_stats', 'mfdb.fund_ms_stats', 'pfdb.pf_cus_fund_ms_stats', 'mfdb.fund_ms_stats', 'mfdb.fund_ms_stats', 'pfdb.cm_udf_index_ms_stats', 'pfdb.cm_factor_ms_stats', 'pfdb.pf_portfolio_ms_stats', 'mfdb.manager_ms_stats', 'mfdb.company_ms_stats'],
+        ['fund_id', 'fund_id', 'fund_id', 'fund_id', 'fund_id', 'index_id', 'factor_id', 'portfolio_id', 'manager_id', 'company_id'] );
+
+    return (SELECT * FROM tmp_universe u WHERE u.type = entity_type);
+
+}
+
+/*
  *  根据不同类型的主体返回其BFI指标的表名、字段名
  * 
  *  Example: get_bfi_indicator_table_description('PL');

modules/task_fundPerformance.dos

@@ -8,6 +8,70 @@ use fundit::returnCalculator;
 use fundit::indicatorCalculator;
 use fundit::ms_dataPuller;
 
+
+/*
+ *  [定时任务]: 同步公募，私募，市场指数的净值并存入本地数据库
+ * 
+ *  @param date <DATETIME>
+ * 
+ *  Example: GetEntityNavTask(2024.09.01);  // 【初始化专用】
+ */
+def GetEntityNavTask(date) {
+
+	for(entity_type in ['HF', 'MI', 'MF']) {
+
+		des = get_nav_table_description(entity_type)[0];
+
+		t_local_table = load_table_from_local('fundit', des.table_name);
+
+		entity_date = get_entity_list_by_nav_updatetime(entity_type, null, date, true);
+	
+		if(entity_date.isVoid() || entity_date.size() == 0) return;
+	
+		v_entity_id = entity_date.entity_id.distinct();
+	
+		i = 0;
+		batch_size = 1000;
+		max_cnt = v_entity_id.size();
+//		cnt_nums = 0:0;
+		cnt_num = 0;
+
+		do {
+			
+			tb_entity = SELECT * FROM entity_date WHERE entity_id IN v_entity_id[i : min(max_cnt, i+batch_size)];
+	
+			s_json = tb_entity.toStdJson();
+
+			tb_nav = sync_entity_nav_by_date(entity_type, s_json);
+
+			try {
+
+				// 建个键值表来更新数据
+//				t_local_table = keyedTable([des.sec_id_col, 'price_date'], load_table_from_local('fundit', des.table_name));
+
+//				cnt_nums += t_local_table.tableUpsert(tb_nav, keyColNames=['entity_id', 'price_date']);
+				t_local_table.append!(tb_nav);
+
+	
+				// 存入本地
+//				save_table(t_local_table, des.table_name, false);
+
+
+
+			} catch(ex) {
+				
+				writeLogLevel(ERROR, ex);
+			}
+
+			writeLogLevel(INFO, des.table_name + ': UPDATE ' + tb_nav.size()$STRING);
+
+			i += batch_size;
+			
+		} while (i < max_cnt);
+	}
+
+}
+
 /*
  *   [定时任务]：最新净值触发的业绩指标计算
  * 
@@ -16,7 +80,7 @@ use fundit::ms_dataPuller;
  *   
  *   NOTE: 与Java不同的是当月indicator计算每日触发，不必等到Month-end production
  *   
- *   Example: calFundPerformanceTask('MF', 2024.10.28);
+ *   Example: calFundPerformanceTask('HF', 2024.10.28);
  *            calFundPerformanceTask('MI', 2024.10.28);
  *            calFundPerformanceTask('FI', 2024.10.28);
  *            calFundPerformanceTask('MF', get_ini_data_const()['date']);  -- 【初始化数据专用】(70min)
@@ -32,7 +96,7 @@ def calFundPerformanceTask(entityType, date) {
     else
     	end_day = date;
 
-    // 取有最新净值变动的基金列表 (1s)
+    // 取有最新净值变动的基金列表 (2.5 min)
     tb_cal_funds = get_entity_list_by_nav_updatetime(entityType, NULL, end_day, true);
 
     if(tb_cal_funds.isVoid() || tb_cal_funds.size() == 0 ) return;
@@ -43,6 +107,7 @@ def calFundPerformanceTask(entityType, date) {
     tb_fund_risk_stats = create_entity_risk_stats();
     tb_fund_riskadjret_stats = create_entity_riskadjret_stats();
     tb_fund_style_stats = create_entity_style_stats();
+    tb_fund_ms_stats = create_entity_ms_stats();
 
     tb_fund_performance_weekly = create_entity_performance_weekly();
     tb_fund_latest_performance = create_entity_latest_performance();
@@ -77,7 +142,7 @@ def calFundPerformanceTask(entityType, date) {
             generate_entity_risk_stats(fund_info, indicators, true, tb_fund_risk_stats);
             generate_entity_riskadjret_stats(fund_info, indicators, true, tb_fund_riskadjret_stats);
             generate_entity_style_stats(fund_info, indicators, true, tb_fund_style_stats);
-
+			generate_entity_ms_stats(fund_info, indicators, true, tb_fund_ms_stats);
         }
         
         // 计算周收益 (8s)
@@ -121,17 +186,21 @@ def calFundPerformanceTask(entityType, date) {
             chg_columns_for_mysql(tb_fund_style_stats, 'fund_id');
             save_and_sync(tb_fund_style_stats, 'raw_db.fund_style_stats', 'raw_db.fund_style_stats');
 
+            chg_columns_for_mysql(tb_fund_ms_stats, 'fund_id');
+            save_and_sync(tb_fund_ms_stats, 'raw_db.fund_ms_stats', 'mfdb.fund_ms_stats'); // new table, write into directly
+
             save_and_sync(tb_fund_performance_weekly, 'raw_db.fund_performance_weekly', 'raw_db.fund_performance_weekly');
 
             save_and_sync(tb_fund_latest_performance, 'raw_db.fund_latest_performance', 'raw_db.fund_latest_nav_performance');
 
             // 数据初始化时将指标存入本地
-            if(end_day <= get_ini_data_const['date']) {
+            if(end_day <= get_ini_data_const()['date']) {
             	save_table(tb_fund_performance, 'mfdb.fund_performance', false);
             	save_table(tb_fund_indicator, 'mfdb.fund_indicator', false);
             	save_table(tb_fund_risk_stats, 'mfdb.fund_risk_stats', false);
             	save_table(tb_fund_riskadjret_stats, 'mfdb.fund_riskadjret_stats', false);
             	save_table(tb_fund_style_stats, 'mfdb.fund_style_stats', false);
+            	save_table(tb_fund_ms_stats, 'mfdb.fund_ms_stats', false);
             	save_table(tb_fund_performance_weekly, 'mfdb.fund_performance_weekly', false);
             	save_table(tb_fund_latest_performance, 'mfdb.fund_latest_performance', false);
             }

modules/task_monthlyPerformance.dos

@@ -17,12 +17,14 @@ use fundit::navCalculator;
  *   @param end_date <MONTH>:
  *   @param isFromMySQL <BOOL>: false 时读取dolphin本地的收益及指标表，用于初始化数据
  *   
+ *   NOTE: 与 MySQL 不同，这里的 rank 最小值是0，不是1
  * 
  *   Example: CalEntityRankingTask('MF', 2024.09M, true);
+ *   		  CalEntityRankingTask('CO', 2024.10M, true);
  */
 def CalEntityRankingTask(entityType, endDate, isFromMySQL=true) {
-//entityType='PL'
-//endDate = 2024.10M
+//entityType='MF'
+//endDate = 2024.11M
 //isFromMySQL = true
 	if(!(entityType in ['MF', 'HF', 'PL', 'CO'])) return NULL;