小修小补

modules/CalculationJobs.dos

@@ -0,0 +1,94 @@
+module fundit::CalculationJobs
+// login(`admin, `123456);
+// clearCachedModules();
+
+use fundit::task_fundPerformance;
+use fundit::task_portfolioPerformance;
+use fundit::task_monthlyPerformance;
+use fundit::task_weeklyPerformance;
+
+/* 
+ *  定时任务
+ *  
+ * 查询任务运行； getRecentJobs()；
+ * 查询任务列表：getScheduledJobs('weekly_%');
+ * 删除任务：deleteScheduledJob('monthly_manager_nav');
+ * 取消任务：cancelJob('daily_portfolio_performance20250120');
+ * 
+ * 任务全删除（用于重新Load本文件):
+ *  v_jobId = EXEC jobId from getScheduledJobs('daily_%').append!(getScheduledJobs('weekly_%')).append!(getScheduledJobs('monthly_%'));
+ *  for(jobId in v_jobId) deleteScheduledJob(jobId);
+ *  
+ *  
+ */
+
+
+/* -------------  DAILY JOBS ---------------------------*/
+
+// nav sync
+scheduleJob('daily_nav_sync', "Sync NAV for fund and index", GetEntityNavTask{now().temporalAdd(-3d)}, 20:00m, today(), today()+30, 'D');
+
+// index and factor calculations go first
+scheduleJob('daily_market_index_performance', "Market Index return and indicator calculation", calFundPerformanceTask{'MI', now().temporalAdd(-3d)}, 22:00m, today(), today()+30, 'D');
+scheduleJob('daily_fundit_index_performance', "FundIT Index return and indicator calculation", calFundPerformanceTask{'FI', now().temporalAdd(-3d)}, 22:00m, today(), today()+30, 'D');
+scheduleJob('daily_factor_performance', "Factor nav, return and indicator calculation", CalFactorPerformanceTask{now().temporalAdd(-3d)}, 23:00m, today(), today()+30, 'D');
+
+// standard return and indicator calculation
+scheduleJob('daily_mutual_fund_performance', "Mutual fund return and indicator calculation", calFundPerformanceTask{'MF', now().temporalAdd(-3d)}, 00:00m, today(), today()+30, 'D');
+scheduleJob('daily_hedge_fund_performance', "Hedge fund return and indicator calculation", calFundPerformanceTask{'HF', now().temporalAdd(-3d)}, 00:30m, today(), today()+30, 'D');
+scheduleJob('daily_portfolio_performance', "Portfolio nav, return and indicator calculation", CalPortfolioPerformanceTask{now().temporalAdd(-3d)}, 01:00m, today(), today()+30, 'D');
+
+
+/* -------------  WEEKLY JOBS ---------------------------*/
+
+// RBSA calculation 
+scheduleJob('weekly_fund_rbsa', "Fund RBSA calculation", CalEntityRBSATask{'MF', NULL, now().temporalAdd(-7d)}, 07:00m, today(), today()+30, 'W', [6, 0]);
+scheduleJob('weekly_portfolio_rbsa', "Portfolio RBSA calculation", CalEntityRBSATask{'PF', NULL, now().temporalAdd(-7d)}, 07:30m, today(), today()+30, 'W', [6, 0]);
+
+// Category Average index weekly NAV
+scheduleJob('weekly_category_avg_nav', "Category Average NAV calculation", CalCategoryAverageNavTask{now().temporalAdd(-7d)}, 08:00m, today(), today()+30, 'W', [6, 0]);
+
+// manager & company nav
+scheduleJob('weekly_manager_nav', "Manager NAV calculation", CalMCWeeklyNavTask{'PL', now().temporalAdd(-7d)}, 08:30m, today(), today()+30, 'W', [6, 0]);
+scheduleJob('weekly_company_nav', "Company NAV calculation", CalMCWeeklyNavTask{'CO', now().temporalAdd(-7d)}, 09:00m, today(), today()+30, 'W', [6, 0]);
+
+
+// BFI MATCHING HERE
+scheduleJob('weekly_fund_bfi_matching', "Fund bfi matching", MatchEntityBFITask{'MF', now().temporalAdd(-3d)}, 01:30m, today(), today()+30, 'W', [6, 0]);
+scheduleJob('weekly_portfolio_bfi_matching', "Portfolio bfi matching", MatchEntityBFITask{'PF', now().temporalAdd(-3d)}, 06:00m, today(), today()+30, 'W', [6, 0]);
+
+// BFI indicator calculation
+scheduleJob('weekly_fund_bfi_indicator', "Fund BFI indicator calculation", calEntityBfiIndicatorTask{'MF', now().temporalAdd(-3d)}, 07:00m, today(), today()+30, 'W', [6, 0]);
+scheduleJob('weekly_portfolio_bfi_indicator', "Portfolio BFI indicator calculation", calEntityBfiIndicatorTask{'PF', now().temporalAdd(-3d)}, 08:00m, today(), today()+30, 'W', [6, 0]);
+
+/* -------------  MONTHLY JOBS ---------------------------*/
+
+// manager & company nav
+scheduleJob('monthly_manager_nav', "Manager NAV calculation", CalMCMonthlyNavTask{'PL', now().temporalAdd(-1m).temporalAdd(-2d)}, 18:00m, today(), today()+30, 'M', [2, 3]);
+scheduleJob('monthly_company_nav', "Company NAV calculation", CalMCMonthlyNavTask{'CO', now().temporalAdd(-1m).temporalAdd(-2d)}, 20:00m, today(), today()+30, 'M', [2, 3]);
+
+// manager & company indicator
+scheduleJob('monthly_manager_indicator', "Manager indicator calculation", CalMCIndicatorTask{'PL', now().temporalAdd(-1m).temporalAdd(-2d)}, 22:00m, today(), today()+30, 'M', [2, 3]);
+scheduleJob('monthly_company_indicator', "Company indicator calculation", CalMCIndicatorTask{'CO', now().temporalAdd(-1m).temporalAdd(-2d)}, 23:00m, today(), today()+30, 'M', [2, 3]);
+
+// manager BFI matching
+scheduleJob('monthly_manager_bfi_matching', "Manager bfi matching", MatchManagerBFITask{now().temporalAdd(-1m).temporalAdd(-2d)}, 00:00m, today(), today()+30, 'M', [3, 4]);
+
+// manager BFI indicator
+scheduleJob('monthly_manager_bfi_indicator', "Manager bfi indicator calculation", CalManagerBfiIndicatorTask{now().temporalAdd(-1m).temporalAdd(-2d)}, 02:00m, today(), today()+30, 'M', [3, 4]);
+
+// manager & company rankings
+scheduleJob('monthly_company_ranking', "Company PBI ranking calculation", CalEntityRankingTask{'CO', now().temporalAdd(-1m).month(), true}, 03:00m, today(), today()+30, 'M', [3, 4]);
+scheduleJob('monthly_manager_ranking', "Manager PBI ranking calculation", CalEntityRankingTask{'PL', now().temporalAdd(-1m).month(), true}, 04:00m, today(), today()+30, 'M', [3, 4]);
+
+// manager bfi ranking
+scheduleJob('monthly_manager_bfi_ranking', "Manager BFI ranking calculation", CalEntityBfiRankingTask{'PL', now().temporalAdd(-1m).month(), true}, 06:00m, today(), today()+30, 'M', [3, 4]);
+
+
+// fund rankings
+scheduleJob('monthly_fund_ranking', "Fund PBI ranking calculation", CalEntityRankingTask{'MF', now().temporalAdd(-1m).month(), true}, 19:00m, today(), today()+30, 'M', [5, 6]);
+scheduleJob('monthly_fund_bfi_ranking', "Fund BFI ranking calculation", CalEntityBfiRankingTask{'MF', now().temporalAdd(-1m).month(), true}, 20:00m, today(), today()+30, 'M', [5, 6]);
+
+// portfolio rankings (strategy, substrategy, bfi included)
+scheduleJob('monthly_portfolio_ranking', "Portfolio ranking calculation", CalRelativeRankingTask{'PF', NULL, now().temporalAdd(-1m).month(), true}, 21:00m, today(), today()+30, 'M', [5, 6]);
+

modules/performanceDataPuller.dos

@@ -314,7 +314,7 @@ def get_entity_list_by_latest_return_updatetime(entity_type, entity_ids, updatet
  *             
  * 
  *
- * Example: get_nav_by_price_date('HF', "'HF000004KN','HF00018WXG'", 2024.05.01, true);
+ * Example: get_nav_by_price_date('MF', "'MF00003PW1','MF00003PW2'", 2024.05.01, false);
  *          get_nav_by_price_date('MI', "'IN00000008','IN0000000M'", 2024.05.01, true);
  *          get_nav_by_price_date('PL', NULL, 2024.11.01, true);
  */
@@ -353,14 +353,15 @@ def get_nav_by_price_date(entity_type, entity_ids, price_date, isFromMySQL) {
 
     } else {
     
-        tb_local = load_table_from_local("fundit", tmp.table_name[0])
+        tb_local = load_table_from_local("fundit", tmp.table_name[0]);
 
-        s_col = sqlCol("*")
+        pd = price_date;
 
-        // TODO: how to make the "fund_id" dynamicly decided by tmp.sec_id_col[0]?
-        s_where = [expr(<fund_id>, in, s_entity_ids.strReplace("'", "").split(",")), <price_date >= price_date>]
-        
-        t = sql(s_col, tb_local, s_where).eval()
+		t = <SELECT entity_id, price_date, cumulative_nav, nav FROM tb_local
+		     WHERE entity_id IN s_entity_ids.strReplace("'", "").split(",")
+		       AND isvalid = 1
+		       AND price_date >= pd
+		     ORDER BY entity_id, price_date>.eval();
     
     }
 
@@ -471,8 +472,11 @@ def get_entity_list_by_nav_updatetime(entity_type, entity_ids, updatetime, isFro
  *  @param freq <STRING>: m, w, d
  *  @param pre_nav_incld <INT>: 0- no pre_nav; 1- pre_nav only; 2- pre_nav + afters
  *  @param json_query <JSON>: [{sec_id:xxx, price_date: yyyy-mm-dd}]
+ *  
+ *  @return <TABLE>: [COLUMNS] sec_id, price_date, cumulative_nav, nav
  * 
  *  Example: get_nav_for_return_calculation('MI', 'm', '[{"sec_id": "IN00000008","price_date": "2004-12-31"}, {"sec_id": "IN00000077","price_date": "2003-12-31"}]', 1)
+ *           get_nav_for_return_calculation('MF', 'd', '[{"sec_id": "MF00003PW1","price_date": "2025-01-11"}]', 1);
  *  
  */
 def get_nav_for_return_calculation(entity_type, freq, json_query, pre_nav_incld=2) {

modules/returnCalculator.dos

@@ -163,10 +163,160 @@ def get_trailing_return(table_last_nav, table_nav, duration, return_column_name)
     return tb;
 }
 
+	
+/*
+ *  批量计算区间收益
+ *  TODO: mySQL version 向前取4天，向后不做限制。这里的逻辑是向前取4个交易日, 遇到大长假后可能取不到数据
+ *
+ *  
+ *
+ */
+def get_trailing_return2(entity_type, tb_last_nav, duration, return_column_name) {
+
+    // 在往前减duration代表的区间后往后加1天才能取前值
+	s_json = (SELECT sec_id, price_date.temporalAdd(duration).temporalAdd(1d) AS price_date FROM tb_last_nav).toStdJson();
+	tb_pre_nav = get_nav_for_return_calculation(entity_type, 'd', s_json, pre_nav_incld=1);
+
+    tb = SELECT n.sec_id AS entity_id, n.price_date, n.cumulative_nav \ p.cumulative_nav - 1 AS ret
+         FROM tb_last_nav n
+         INNER JOIN tb_pre_nav p ON n.sec_id = p.sec_id
+         WHERE n.cumulative_nav > 0
+           AND p.cumulative_nav > 0
+           AND n.price_date.temporalAdd(duration) <= p.price_date.temporalAdd(4B);
+
+    tb.rename!("ret", return_column_name);
+
+    return tb;
+}
+
+/*
+ *  批量计算区间最大回撤
+ *  
+ *  NOTE: 简单起见，和计算收益不同，这里并不往前找可用净值
+ *
+ */
+def get_trailing_max_drawdown(entity_type, tb_last_nav, duration, drawdown_column_name) {
+
+    // 取区间净值
+	s_json = (SELECT sec_id, price_date.temporalAdd(duration) AS price_date FROM tb_last_nav).toStdJson();
+	tb_pre_nav = get_nav_for_return_calculation(entity_type, 'd', s_json, pre_nav_incld=0);
+
+    tb = SELECT n.sec_id, cumulative_nav.maxDrawdown() AS max_drawdown
+         FROM tb_last_nav n
+         GROUP BY n.sec_id;
+
+    tb.rename!(['sec_id', 'max_drawdown'], ['entity_id', drawdown_column_name]);
+
+    return tb;
+}
+
 /*
  *  批量计算最新收益
  *
-   【老程序，可优化】
+ *  @param entity_info <TABLE>: [COLUMNS] entity_id, price_date, inception_date, benchmark_id, ini_value 
+ *
+ *  NOTE: 取消了最大回撤和卡玛
+ *
+ */
+def cal_latest_performance2(entity_type, entity_info, isFromMySQL) {
+
+    // 取最新净值
+    s_json = (SELECT entity_id AS sec_id, today() AS price_date FROM entity_info).toStdJson();
+    tb_last_nav = get_nav_for_return_calculation(entity_type, 'd', s_json, pre_nav_incld=1);
+
+    if(tb_last_nav.isVoid() || tb_last_nav.size() == 0) return;
+    
+    // 近1期收益，对应mySQL fund_latest_nav_performance 中的 net_value_change
+    s_json = (SELECT sec_id, price_date FROM tb_last_nav).toStdJson();
+    tb_pre_nav = get_nav_for_return_calculation(entity_type, 'd', s_json, pre_nav_incld=1);
+
+    tb_last_return = SELECT n.sec_id AS entity_id, n.price_date, p.price_date AS pre_price_date, n.cumulative_nav, n.nav, n.cumulative_nav \ p.cumulative_nav - 1 AS net_value_change
+                     FROM tb_last_nav n
+                     INNER JOIN tb_pre_nav p ON n.sec_id = p.sec_id
+                     WHERE p.cumulative_nav > 0
+                       AND n.cumulative_nav > 0;
+    
+    // 近1交易日收益
+    tb_1d = SELECT entity_id, price_date, net_value_change AS ret_1d
+            FROM tb_last_return
+            WHERE price_date = pre_price_date.temporalAdd(1d).businessDay();
+
+    
+    // 近1周、1/3/6月、1/2/3/4/5/10年收益
+    tb_1w = get_trailing_return2(entity_type, tb_last_nav, -7d, "ret_1w");
+    tb_1m = get_trailing_return2(entity_type, tb_last_nav, -1M, "ret_1m");
+    tb_3m = get_trailing_return2(entity_type, tb_last_nav, -3M, "ret_3m");
+    tb_6m = get_trailing_return2(entity_type, tb_last_nav, -6M, "ret_6m");
+    tb_1y = get_trailing_return2(entity_type, tb_last_nav, -1y, "ret_1y");
+    tb_2y = get_trailing_return2(entity_type, tb_last_nav, -2y, "ret_2y");
+    tb_3y = get_trailing_return2(entity_type, tb_last_nav, -3y, "ret_3y");
+    tb_4y = get_trailing_return2(entity_type, tb_last_nav, -4y, "ret_4y");
+    tb_5y = get_trailing_return2(entity_type, tb_last_nav, -5y, "ret_5y");
+    tb_10y = get_trailing_return2(entity_type, tb_last_nav, -10y, "ret_10y");
+
+    // ytd return
+    s_json = (SELECT sec_id, price_date.yearBegin() AS price_date FROM tb_last_nav).toStdJson();
+    tb_pre_nav = get_nav_for_return_calculation(entity_type, 'd', s_json, pre_nav_incld=1);
+    tb_ytd = SELECT n.sec_id AS entity_id, n.price_date, n.cumulative_nav \ p.cumulative_nav - 1 AS ret_ytd
+             FROM tb_last_nav n
+             INNER JOIN tb_pre_nav p ON n.sec_id = p.sec_id
+             WHERE p.cumulative_nav > 0
+               AND n.cumulative_nav > 0
+               AND n.price_date.yearBegin() <= p.price_date.temporalAdd(4B);
+    
+    // since inception return
+    tb_incep = SELECT a.sec_id AS entity_id, a.price_date, -1 + cumulative_nav \ ini_value AS ret_incep, fi.inception_date
+               FROM tb_last_nav a
+               INNER JOIN entity_info fi ON a.sec_id = fi.entity_id
+    
+    // annulized since reception return following GIPS rule
+    UPDATE tb_incep SET ret_incep_a = iif((price_date-inception_date)<=365, ret_incep, (1 + ret_incep).pow(365.25\(price_date-inception_date)) - 1)
+    UPDATE tb_incep SET ret_incep_a_all = ret_incep_a,
+                        ret_incep_a_gips = ret_incep_a
+/*
+    // 最大回撤， max_drawdown_incep 太耗流量会导致 out-of-memory 错误
+    tb_drawdown_1m = get_trailing_max_drawdown(entity_type, tb_last_nav, -1M, "drawdown_1m");
+    tb_drawdown_3m = get_trailing_max_drawdown(entity_type, tb_last_nav, -3M, "drawdown_3m");
+    tb_drawdown_1y = get_trailing_max_drawdown(entity_type, tb_last_nav, -1y, "drawdown_1y");
+*/
+
+    tb_rets = SELECT a.entity_id, a.price_date.datetimeFormat("yyyy-MM") AS end_date, a.price_date, a.pre_price_date, a.nav, a.cumulative_nav,
+                     a.net_value_change, d1.ret_1d, w1.ret_1w, m1.ret_1m, m3.ret_3m, m6.ret_6m,
+                     y1.ret_1y, y2.ret_2y, y3.ret_3y, y4.ret_4y, y5.ret_5y, y10.ret_10y,
+                     ytd.ret_ytd, incep.ret_incep, incep.ret_incep_a, incep.ret_incep_a_all, incep.ret_incep_a_gips,
+                     double(NULL) AS maxdrawdown_1m, double(NULL) AS maxdrawdown_3m, double(NULL) AS maxdrawdown_1y,
+                     double(NULL) AS maxdrawdown_incep,
+                     double(NULL) AS calmarratio_incep
+              FROM tb_last_return a
+              LEFT JOIN tb_1d d1 ON a.entity_id = d1.entity_id
+              LEFT JOIN tb_1w w1 ON a.entity_id = w1.entity_id
+              LEFT JOIN tb_1m m1 ON a.entity_id = m1.entity_id
+              LEFT JOIN tb_3m m3 ON a.entity_id = m3.entity_id
+              LEFT JOIN tb_6m m6 ON a.entity_id = m6.entity_id
+              LEFT JOIN tb_1y y1 ON a.entity_id = y1.entity_id
+              LEFT JOIN tb_2y y2 ON a.entity_id = y2.entity_id
+              LEFT JOIN tb_3y y3 ON a.entity_id = y3.entity_id
+              LEFT JOIN tb_4y y4 ON a.entity_id = y4.entity_id
+              LEFT JOIN tb_5y y5 ON a.entity_id = y5.entity_id
+              LEFT JOIN tb_10y y10 ON a.entity_id = y10.entity_id
+              LEFT JOIN tb_ytd ytd ON a.entity_id = ytd.entity_id
+              LEFT JOIN tb_incep incep ON a.entity_id = incep.entity_id
+              ORDER BY a.entity_id
+
+    // 忽略掉非GIPS标准的所有年化收益字段（包括ytd_a）
+    UPDATE tb_rets SET ret_1y_a = ret_1y, ret_2y_a = (1 + ret_2y).pow(1\2) - 1, ret_3y_a = (1 + ret_3y).pow(1\3) - 1,
+                       ret_4y_a = (1 + ret_4y).pow(1\4) - 1, ret_5y_a = (1 + ret_5y).pow(1\5) - 1, ret_10y_a = (1 + ret_10y).pow(1\10) - 1
+
+
+    return tb_rets
+
+}
+
+
+/*
+ *  批量计算最新收益
+ *
+   @param entity_info <TABLE>: [COLUMNS] entity_id, price_date, inception_date, benchmark_id, ini_value 
  *
  */
 def cal_latest_performance(entity_type, entity_info, isFromMySQL) {
@@ -284,4 +434,3 @@ def cal_latest_performance(entity_type, entity_info, isFromMySQL) {
     return tb_rets
 
 }
-

modules/task_monthlyPerformance.dos

@@ -19,12 +19,12 @@ use fundit::navCalculator;
  *   
  *   NOTE: 与 MySQL 不同，这里的 rank 最小值是0，不是1
  * 
- *   Example: CalEntityRankingTask('MF', 2024.09M, true);
- *   		  CalEntityRankingTask('CO', 2024.10M, true);
+ *   Example: CalEntityRankingTask('PL', 2024.12M, true);
+ *   		  CalEntityRankingTask('CO', 2024.12M, true);
  */
 def CalEntityRankingTask(entityType, endDate, isFromMySQL=true) {
 //entityType='MF'
-//endDate = 2024.11M
+//endDate = 2024.12M
 //isFromMySQL = true
 	if(!(entityType in ['MF', 'HF', 'PL', 'CO'])) return NULL;
 	
@@ -44,8 +44,8 @@ def CalEntityRankingTask(entityType, endDate, isFromMySQL=true) {
  *   @param isFromMySQL <BOOL>: false 时读取dolphin本地的收益及指标表，用于初始化数据
  *   
  * 
- *   Example: CalEntityBfiRankingTask('MF', 2024.09M, true);
- *            CalEntityBfiRankingTask('PL', 2024.10M, true);
+ *   Example: CalEntityBfiRankingTask('MF', 2024.12M, true);
+ *            CalEntityBfiRankingTask('PL', 2024.12M, true);
  */
 def CalEntityBfiRankingTask(entityType, endDate, isFromMySQL=true) {
 
@@ -86,7 +86,7 @@ def cal_and_save_relative_ranking(entity_type, benchmark_ranking, entity_ranking
  *   TODO: customer fund
  *   TODO: 计算单个组合时总耗时1.5min, 大部分时间用来获取Mysql数据
  * 
- *   Example: CalRelativeRankingTask('PF', NULL, 2024.09M, true);
+ *   Example: CalRelativeRankingTask('PF', NULL, 2024.12M, true);
  *            CalRelativeRankingTask('PF', 143109, 2024.09M, true); 
  */
 def CalRelativeRankingTask(entity_type, entity_ids, end_date, isFromMySQL=true) {
@@ -348,7 +348,7 @@ def cal_and_save_mc_indicator(entity_type, entity_date, monthly_returns, indicat
 /*
  *  [定时任务]: 基金经理/公司月净值计算
  * 
- *   Example: CalMCMonthlyNavTask('PL', 2024.12.01);
+ *   Example: CalMCMonthlyNavTask('CO', 2024.12.01);
  */
 def CalMCMonthlyNavTask(entity_type, updatetime) {
 //updatetime = 2024.12.01;
@@ -474,7 +474,7 @@ def CalManagerBfiIndicatorTask(updatetime) {
  * 
  *  [定时任务]: 基金经理的BFI MATCHING
  * 
- *   it takes 9.5 hours
+ *   it takes 16 min
  */
 def MatchManagerBFITask(updatetime) {
 

modules/task_portfolioPerformance.dos

@@ -100,9 +100,6 @@ def cal_and_save_portfolio_nav(cal_portfolio_info, is_save_local) {
 
     rt = '';
 
-    // 准备类似MySQL结构的数据表
-    tb_portfolio_nav = create_entity_nav(true);
-
     // 分批跑
     i = 0;
     batch_size = 1000;
@@ -111,6 +108,9 @@ def cal_and_save_portfolio_nav(cal_portfolio_info, is_save_local) {
 
     do { // 先把净值算出来存入数据库，落袋为安
 
+	    // 准备类似MySQL结构的数据表
+	    tb_portfolio_nav = create_entity_nav(true);
+
         portfolio_info = SELECT * FROM cal_portfolio_info
                          WHERE portfolio_id IN all_portfolio_id[i : min(all_portfolio_id.size(), i+batch_size)];
 
@@ -121,31 +121,31 @@ def cal_and_save_portfolio_nav(cal_portfolio_info, is_save_local) {
 
         INSERT INTO tb_portfolio_nav SELECT entity_id, price_date, nav FROM tb_ret;
 
+	    if(! tb_portfolio_nav.isVoid() && tb_portfolio_nav.size() > 0) {
+	
+	        // save data to MySQL  (12 sec)
+	        try {
+	
+	            tb_portfolio_nav.rename!('entity_id', 'portfolio_id');
+	            save_and_sync(tb_portfolio_nav, 'raw_db.pf_portfolio_nav', 'raw_db.pf_portfolio_nav');
+	
+	            // 数据初始化时将指标存入本地
+	            if(is_save_local == true) {
+	            	save_table(tb_portfolio_nav, 'pfdb.pf_portfolio_nav', false);
+	            }
+	
+	        } catch(ex) {
+	
+	            //TODO: Log errors
+	            rt = ex;
+	        }
+	    }
+
         i += batch_size;
 
     } while (i <= cal_portfolio_info.size());
 
 
-    if(! tb_portfolio_nav.isVoid() && tb_portfolio_nav.size() > 0) {
-
-        // save data to MySQL  (12 sec)
-        try {
-
-            tb_portfolio_nav.rename!('entity_id', 'portfolio_id');
-            save_and_sync(tb_portfolio_nav, 'raw_db.pf_portfolio_nav', 'raw_db.pf_portfolio_nav');
-
-            // 数据初始化时将指标存入本地
-            if(is_save_local == true) {
-            	save_table(tb_portfolio_nav, 'pfdb.pf_portfolio_nav', false);
-            }
-
-        } catch(ex) {
-
-            //TODO: Log errors
-            rt = ex;
-        }
-    }
-
     return rt;
 }
 
@@ -166,18 +166,6 @@ def cal_and_save_entity_indicators(entity_type, cal_entity_info, is_save_local)
     rt = '';
 
 	is_id_interger = iif(entity_type == 'PF', true, false);
-    
-    // 准备类似MySQL结构的数据表
-    tb_entity_performance = create_entity_performance(is_id_interger);
-
-    tb_entity_indicator = create_entity_indicator(is_id_interger);
-    tb_entity_risk_stats = create_entity_risk_stats(is_id_interger);
-    tb_entity_riskadjret_stats = create_entity_riskadjret_stats(is_id_interger);
-    tb_entity_style_stats = create_entity_style_stats(is_id_interger);
-
-    tb_entity_performance_weekly = create_entity_performance_weekly(is_id_interger);
-    tb_entity_latest_performance = create_entity_latest_performance(is_id_interger);
-
 
     // 分批跑
     i = 0;
@@ -187,6 +175,17 @@ def cal_and_save_entity_indicators(entity_type, cal_entity_info, is_save_local)
 
     do { 
 
+	    // 准备类似MySQL结构的数据表
+	    tb_entity_performance = create_entity_performance(is_id_interger);
+	
+	    tb_entity_indicator = create_entity_indicator(is_id_interger);
+	    tb_entity_risk_stats = create_entity_risk_stats(is_id_interger);
+	    tb_entity_riskadjret_stats = create_entity_riskadjret_stats(is_id_interger);
+	    tb_entity_style_stats = create_entity_style_stats(is_id_interger);
+	
+	    tb_entity_performance_weekly = create_entity_performance_weekly(is_id_interger);
+	    tb_entity_latest_performance = create_entity_latest_performance(is_id_interger);
+
         cal_entity = SELECT * FROM cal_entity_info
                      WHERE entity_id IN all_entity_id[i : min(all_entity_id.size(), i+batch_size)];
 
@@ -217,7 +216,6 @@ def cal_and_save_entity_indicators(entity_type, cal_entity_info, is_save_local)
         generate_entity_style_stats(entity_info, indicators, true, tb_entity_style_stats);
 
 
-
         // 计算周收益 (49s)
         entity_info = SELECT * FROM ej(entity_info, get_entity_info(entity_type, all_entity_id[i : min(all_entity_id.size(), i+batch_size)]), 'entity_id')
         rets_w = cal_weekly_returns(entity_type, entity_info);
@@ -238,59 +236,59 @@ def cal_and_save_entity_indicators(entity_type, cal_entity_info, is_save_local)
             generate_entity_latest_performance(entity_info, perf_latest, true, tb_entity_latest_performance);
         }
 
-        i += batch_size;
-
-    } while (i <= cal_entity_info.size());
-
-
-    if(! tb_entity_performance.isVoid() && tb_entity_performance.size() > 0) {
-
-        // save data to MySQL  
-        try {
-
-			des = get_performance_table_description(entity_type)[0];
-            chg_columns_for_mysql(tb_entity_performance, des.sec_id_col);
-            tb_entity_performance.rename!('cumulative_nav', des.cumulative_nav_col);
-            save_and_sync(tb_entity_performance, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_performance, des.table_name, false);
-
-			des = get_indicator_table_description(entity_type)[0];
-            chg_columns_for_mysql(tb_entity_indicator, des.sec_id_col);
-            save_and_sync(tb_entity_indicator, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_indicator, des.table_name, false);
-
-			des = get_risk_stats_table_description(entity_type)[0];
-            chg_columns_for_mysql(tb_entity_risk_stats, des.sec_id_col);
-            save_and_sync(tb_entity_risk_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_risk_stats, des.table_name, false);
-
-			des = get_riskadjret_stats_table_description(entity_type)[0];
-            chg_columns_for_mysql(tb_entity_riskadjret_stats, des.sec_id_col);
-            save_and_sync(tb_entity_riskadjret_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_riskadjret_stats, des.table_name, false);
-
-			des = get_capture_style_table_description(entity_type)[0];
-            chg_columns_for_mysql(tb_entity_style_stats, des.sec_id_col);
-            save_and_sync(tb_entity_style_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_style_stats, des.table_name, false);
-
-			des = get_performance_weekly_table_description(entity_type)[0];
-			tb_entity_performance_weekly.rename!('cumulative_nav', des.cumulative_nav_col);
-            save_and_sync(tb_entity_performance_weekly, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_performance_weekly, des.table_name, false);
-
-			des = get_latest_performance_table_description(entity_type)[0];
-			tb_entity_latest_performance.rename!('cumulative_nav', des.cumulative_nav_col);
-            save_and_sync(tb_entity_latest_performance, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
-            if(is_save_local == true) save_table(tb_entity_latest_performance, des.table_name, false);
 
+	    if(! tb_entity_performance.isVoid() && tb_entity_performance.size() > 0) {
+	
+	        // save data to MySQL  
+	        try {
+	
+				des = get_performance_table_description(entity_type)[0];
+	            chg_columns_for_mysql(tb_entity_performance, des.sec_id_col);
+	            tb_entity_performance.rename!('cumulative_nav', des.cumulative_nav_col);
+	            save_and_sync(tb_entity_performance, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_performance, des.table_name, false);
+	
+				des = get_indicator_table_description(entity_type)[0];
+	            chg_columns_for_mysql(tb_entity_indicator, des.sec_id_col);
+	            save_and_sync(tb_entity_indicator, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_indicator, des.table_name, false);
+	
+				des = get_risk_stats_table_description(entity_type)[0];
+	            chg_columns_for_mysql(tb_entity_risk_stats, des.sec_id_col);
+	            save_and_sync(tb_entity_risk_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_risk_stats, des.table_name, false);
+	
+				des = get_riskadjret_stats_table_description(entity_type)[0];
+	            chg_columns_for_mysql(tb_entity_riskadjret_stats, des.sec_id_col);
+	            save_and_sync(tb_entity_riskadjret_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_riskadjret_stats, des.table_name, false);
+	
+				des = get_capture_style_table_description(entity_type)[0];
+	            chg_columns_for_mysql(tb_entity_style_stats, des.sec_id_col);
+	            save_and_sync(tb_entity_style_stats, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_style_stats, des.table_name, false);
+	
+				des = get_performance_weekly_table_description(entity_type)[0];
+				tb_entity_performance_weekly.rename!('cumulative_nav', des.cumulative_nav_col);
+	            save_and_sync(tb_entity_performance_weekly, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_performance_weekly, des.table_name, false);
+	
+				des = get_latest_performance_table_description(entity_type)[0];
+				tb_entity_latest_performance.rename!('cumulative_nav', des.cumulative_nav_col);
+	            save_and_sync(tb_entity_latest_performance, des.table_name.strReplace('pfdb', 'raw_db').strReplace('mfdb', 'raw_db'), );
+	            if(is_save_local == true) save_table(tb_entity_latest_performance, des.table_name, false);
+	
+	
+	        } catch(ex) {
+	
+	            //TODO: Log errors
+	            rt = ex;
+	        }
+	    }
 
-        } catch(ex) {
+        i += batch_size;
 
-            //TODO: Log errors
-            rt = ex;
-        }
-    }
+    } while (i <= cal_entity_info.size());
 
     return rt;
 }
@@ -314,7 +312,7 @@ def CalPortfolioPerformanceTask(updatetime) {
 
     is_save_local = iif(updatetime <= get_ini_data_const()['date'], true, false);
 
-    // 26 min
+    // 1 min
     rt = cal_and_save_portfolio_nav(tb_cal_ports, is_save_local);
     // 9 min
 	tb_cal_ports.rename!('portfolio_id', 'entity_id');
@@ -540,11 +538,12 @@ def CalCategoryAverageNavTask(updatetime) {
 
     if(date_hedge_fund.isNull() && date_mutual_fund.isNull()) return rt;
 
-    is_save_local = iif(updatetime <= get_ini_data_const()['date'], true, false);
+    //is_save_local = iif(updatetime <= get_ini_data_const()['date'], true, false);
+    is_save_local = true;
 
     // 
     for(category_type in v_category_type) {
-
+//category_type=v_category_type[2]
 		oldest_date = min([date_hedge_fund, date_mutual_fund]);
 
     	// it could take mysql a few minutes to get results
@@ -564,7 +563,9 @@ def CalCategoryAverageNavTask(updatetime) {
 			save_and_sync(t_index_value, 'raw_db.indexes_ty_index', );
 	
 			if(is_save_local == true) {
-				save_table(t_index_value, 'mfdb.indexes_ty_index', false);
+
+				t_local_table = load_table_from_local('fundit', 'mfdb.indexes_ty_index');
+				t_local_table.append!(SELECT index_id AS entity_id, price_date, index_value AS cumulative_nav, index_value AS nav FROM t_index_value);
 			}
 		}
     }