|
|
@@ -0,0 +1,417 @@
|
|
|
+module fundit::task_weeklyPerformance
|
|
|
+
|
|
|
+use fundit::sqlUtilities;
|
|
|
+use fundit::operationDataPuller;
|
|
|
+use fundit::performanceDataPuller;
|
|
|
+use fundit::indicatorCalculator;
|
|
|
+use fundit::rbsaCalculator;
|
|
|
+use fundit::navCalculator;
|
|
|
+use fundit::bfiMatcher;
|
|
|
+use fundit::dataSaver;
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+/*
|
|
|
+ * 根据收益更新日期计算 RBSA (1h for funds; for portfolios)
|
|
|
+ *
|
|
|
+ * @param entityType <STRING>: MF, HF, PF (MF和HF等效)
|
|
|
+ *
|
|
|
+ * TODO: 是否要 monthly 运行，并写入 history 表？
|
|
|
+ * portfolio 未测试
|
|
|
+ *
|
|
|
+ * Example: CalEntityRBSATask('MF', ['MF00003PW1'], 2024.10.14T10:00:00);
|
|
|
+ * CalEntityRBSATask('MF', NULL, 2025.01.10);
|
|
|
+ * CalEntityRBSATask('PF', NULL, 2025.01.10);
|
|
|
+ */
|
|
|
+def CalEntityRBSATask(entityType, entityIds, updateTime) {
|
|
|
+// entityType = 'PF'
|
|
|
+//entityIds = NULL
|
|
|
+//updateTime = 2025.01.10T10:00:00
|
|
|
+
|
|
|
+ t_cal = get_entity_list_by_latest_return_updatetime(entityType, entityIds, updateTime, true);
|
|
|
+
|
|
|
+ window = 48;
|
|
|
+ step = 13;
|
|
|
+
|
|
|
+ if(t_cal.isVoid() || t_cal.size() == 0) return;
|
|
|
+
|
|
|
+ d_rbsa = get_rbsa_index();
|
|
|
+
|
|
|
+ // 拿到所有指数ID
|
|
|
+ v_index = d_rbsa.values().flatten().distinct();
|
|
|
+
|
|
|
+ // 因为用来做基准指数的可能是指数、因子、基金等等任何时间序列数据，所以不用填 entity_type
|
|
|
+ t_index_ret = get_entity_return(NULL, v_index, 'w', t_cal.price_date.min().temporalAdd(-window, 'w'), today(), true);
|
|
|
+
|
|
|
+ i = 0;
|
|
|
+ batch_size = 200; // 1 hour total
|
|
|
+ total_cnt = t_cal.size();
|
|
|
+
|
|
|
+ do {
|
|
|
+
|
|
|
+ tb_result = table(1000:0,
|
|
|
+ ["entity_id", "asset_type_id", "index_id", "effective_date", "level", "alternative_id", "weighting"],
|
|
|
+ [iif(entityType=='PF', INT, STRING), STRING, STRING, STRING, INT, STRING, DOUBLE]);
|
|
|
+
|
|
|
+ t = t_cal[i: min(total_cnt, i+batch_size)];
|
|
|
+
|
|
|
+ // 起始日期是最早更新的净值日期再向前推一个时间窗口+10(防止数据缺失的冗余)
|
|
|
+ s_json = (SELECT entity_id, price_date.temporalAdd(-window-10, 'w') AS price_date FROM t).toStdJson();
|
|
|
+ t_entity_ret = get_entity_ret_by_date(entityType, s_json, 'w', true);
|
|
|
+ if(entityType == 'PF') {
|
|
|
+ v_entity_id = t_entity_ret.entity_id$INT;
|
|
|
+ t_entity_ret.replaceColumn!('entity_id', v_entity_id);
|
|
|
+ }
|
|
|
+
|
|
|
+ for(entity in t) {
|
|
|
+
|
|
|
+ entity_ret = SELECT * FROM t_entity_ret WHERE entity_id = entity.entity_id;
|
|
|
+
|
|
|
+ for(asset_type in d_rbsa.keys()) {
|
|
|
+//asset_type=d_rbsa.keys()[0]
|
|
|
+ index_ret = SELECT entity_id, price_date, ret FROM t_index_ret WHERE entity_id IN d_rbsa[asset_type] AND price_date IS NOT NULL;
|
|
|
+
|
|
|
+ // 起始日期是最早更新日期再向前推一个时间窗口
|
|
|
+ res = cal_entity_RBSA(entityType, entity_ret, index_ret, 'w',
|
|
|
+ entity.price_date.temporalAdd(-window-10, 'w')[0], today(), true, window, step);
|
|
|
+
|
|
|
+ if(res.isVoid() || res.size() == 0) continue;
|
|
|
+
|
|
|
+ // 每日任务只负责更新最新的rbsa结果
|
|
|
+ latest_date = (EXEC price_date.max() AS price_date FROM res)[0];
|
|
|
+
|
|
|
+ tb_result.tableInsert(SELECT entity_id, asset_type, index_id, price_date, level, alternative_id, weights
|
|
|
+ FROM res WHERE price_date = latest_date);
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if(entityType IN ['MF', 'HF'])
|
|
|
+ save_and_sync(tb_result, 'raw_db.pf_fund_rbsa_breakdown', 'raw_db.pf_fund_rbsa_breakdown');
|
|
|
+ else
|
|
|
+ save_and_sync(tb_result, 'raw_db.pf_portfolio_rbsa_breakdown', 'raw_db.pf_portfolio_rbsa_breakdown');
|
|
|
+
|
|
|
+ i += batch_size;
|
|
|
+
|
|
|
+ } while (i < total_cnt);
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+/*
|
|
|
+ * [定时任务] 计算基金和组合的BFI
|
|
|
+ *
|
|
|
+ * @param entityType <STRING>: MF, HF, PF
|
|
|
+ *
|
|
|
+ *
|
|
|
+ * TODO: max_r2 表在哪里被用到了？应该和基金推荐有关系
|
|
|
+ *
|
|
|
+ * Example: MatchEntityBFITask('PF', 2025.01.01);
|
|
|
+ */
|
|
|
+def MatchEntityBFITask(entityType, date) {
|
|
|
+//entityType = 'MF'
|
|
|
+//date = 2024.12.01
|
|
|
+
|
|
|
+ rt = '';
|
|
|
+
|
|
|
+ if(find(['HF', 'MF', 'PF'], entityType) < 0) return null;
|
|
|
+
|
|
|
+ // 取有最新周收益变动的基金列表 (1s)
|
|
|
+ tb_cal_entity = get_entity_list_by_weekly_return_updatetime(entityType, NULL, date, true);
|
|
|
+
|
|
|
+ if(tb_cal_entity.isVoid() || tb_cal_entity.size() == 0 ) return;
|
|
|
+
|
|
|
+ i = 0;
|
|
|
+ size = tb_cal_entity.size();
|
|
|
+ batch_size = 100;
|
|
|
+
|
|
|
+ do {
|
|
|
+
|
|
|
+ t_tmp_entity = tb_cal_entity[i : min(size, i+batch_size)];
|
|
|
+
|
|
|
+ i = i + batch_size;
|
|
|
+
|
|
|
+ // 4 min per 1000 funds or 2 min per 1000 portfolios
|
|
|
+ coe = cal_entity_index_coe(entityType, t_tmp_entity);
|
|
|
+
|
|
|
+ if(coe.isVoid() || coe.size() == 0) continue;
|
|
|
+
|
|
|
+ entity_info = get_entity_info(entityType, t_tmp_entity.entity_id);
|
|
|
+
|
|
|
+ bfi_raw = match_entity_bfi(entityType, entity_info, coe);
|
|
|
+
|
|
|
+ // 先存到数据库，落袋为安
|
|
|
+ try {
|
|
|
+
|
|
|
+ // 筛掉 correlation 绝对值不够阈值的记录
|
|
|
+ t_coe = SELECT entity_id, end_date, index_id,
|
|
|
+ iif(coe_1y.abs() < get_min_threshold('correlation'), double(NULL), coe_1y) AS coe_1y,
|
|
|
+ iif(coe_3y.abs() < get_min_threshold('correlation'), double(NULL), coe_3y) AS coe_3y,
|
|
|
+ iif(coe_5y.abs() < get_min_threshold('correlation'), double(NULL), coe_5y) AS coe_5y,
|
|
|
+ t_value_1y, t_value_3y, t_value_5y, beta_1y, beta_3y, beta_5y
|
|
|
+ FROM coe;
|
|
|
+
|
|
|
+ DELETE FROM t_coe WHERE coe_1y IS NULL AND coe_3y IS NULL AND coe_5y IS NULL;
|
|
|
+
|
|
|
+ // 候选因子
|
|
|
+ t_bfi_candidates = SELECT entity_id, end_date, index_id AS factor_id, coe_1y AS coe, coe_1y.square() AS r2, 'w' AS performance_flag, t_value_1y, beta_1y
|
|
|
+ FROM t_coe WHERE index_id LIKE 'FA%';
|
|
|
+
|
|
|
+ chg_columns_for_mysql(t_coe, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
|
|
|
+
|
|
|
+ // 只有基金需要存 index_coe 表
|
|
|
+ if(entityType IN ['MF', 'HF']) save_and_sync(t_coe, 'raw_db.pf_fund_index_coe', );
|
|
|
+
|
|
|
+ // 所有的 factors 存到 xxx_factor_bfi 表；NOTE: Java 把所有 factor 的数据都存起来，这里只存 correlation 达标的记录 （反正这个表没啥用？）
|
|
|
+ chg_columns_for_mysql(t_bfi_candidates, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
|
|
|
+ save_and_sync(t_bfi_candidates, iif(entityType == 'PF', 'raw_db.pf_portfolio_factor_bfi', 'raw_db.cm_fund_factor_bfi'), );
|
|
|
+
|
|
|
+ if(bfi_raw.isVoid() || bfi_raw.size() == 0) continue;
|
|
|
+
|
|
|
+ // 有效因子
|
|
|
+ t_bfi = SELECT entity_id, end_date, factor_id, coe_1y AS coe, r2, performance_flag, t_value_1y, beta_1y
|
|
|
+ FROM bfi_raw ORDER BY entity_id, end_date, r2 DESC;
|
|
|
+
|
|
|
+ // 最大R2因子及所有有效因子标签
|
|
|
+ t_max_r2 = SELECT entity_id, factor_id.first() AS factor_id, end_date,
|
|
|
+ string(NULL) AS performance_flag, coe.first() AS coe, r2.first() AS r2, concat(factor_name, ",") AS rz_portrait
|
|
|
+ FROM ej(t_bfi, get_bfi_index_list(), 'factor_id')
|
|
|
+ GROUP BY entity_id, end_date;
|
|
|
+
|
|
|
+ // 有效 factors 存到 xxx_factor_bfi_by_category_group 表
|
|
|
+ chg_columns_for_mysql(t_bfi, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
|
|
|
+ save_and_sync(t_bfi, iif(entityType == 'PF', 'raw_db.pf_portfolio_factor_bfi_by_category_group', 'raw_db.pf_fund_factor_bfi_by_category_group'), );
|
|
|
+
|
|
|
+ // 有效因子中 R2 最大的因子存 xxx_max_r2
|
|
|
+ chg_columns_for_mysql(t_max_r2, iif(entityType == 'PF', 'portfolio_id', 'fund_id'));
|
|
|
+ save_and_sync(t_max_r2, iif(entityType == 'PF', 'raw_db.pf_portfolio_factor_bfi_max_r2', 'raw_db.pf_fund_factor_bfi_by_category_group_max_r2'), );
|
|
|
+
|
|
|
+
|
|
|
+ } catch (ex) {
|
|
|
+
|
|
|
+ //TODO: Log errors
|
|
|
+ rt += ex;
|
|
|
+ }
|
|
|
+
|
|
|
+ } while (i<size)
|
|
|
+
|
|
|
+
|
|
|
+ return rt;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * [定时任务] 计算BFI指标并存入数据库
|
|
|
+ *
|
|
|
+ * @param entityType <STRING>: 'MF', 'HF', 'PF' (MF和HF等效)
|
|
|
+ * @param date <DATETIME>: BFI更新时间, 为空时缺省为当前时间的前1天；为1989.01.01或更早日期时代表初始化，指标会被存入本地数据库
|
|
|
+ *
|
|
|
+ *
|
|
|
+ * Example: calEntityBfiIndicatorTask('MF', 2024.10.28);
|
|
|
+ * calEntityBfiIndicatorTask('PF', 2024.10.28);
|
|
|
+ */
|
|
|
+def calEntityBfiIndicatorTask(entityType, date) {
|
|
|
+
|
|
|
+// entityType = 'MF'
|
|
|
+// date = 2024.10.01
|
|
|
+
|
|
|
+ rt = '';
|
|
|
+
|
|
|
+ if(!(entityType IN ['MF', 'HF', 'PF'])) return null;
|
|
|
+
|
|
|
+ very_old_day = 1900.01.01;
|
|
|
+
|
|
|
+ if(date.isNothing() || date.isNull())
|
|
|
+ end_day = temporalAdd(now(), -1d);
|
|
|
+ else
|
|
|
+ end_day = date;
|
|
|
+
|
|
|
+ // 1989.01.01及以前的日期被认为从本地读数据
|
|
|
+ isFromMySQL = iif(end_day <= 1989.01.01, false, true);
|
|
|
+
|
|
|
+ // 取有最新bfi变动的基金列表 (1s)
|
|
|
+ tb_cal_entities = get_entity_bfi_factors(entityType, NULL, very_old_day.month(), today().month(), end_day);
|
|
|
+
|
|
|
+ if(tb_cal_entities.isVoid() || tb_cal_entities.size() == 0 ) return;
|
|
|
+
|
|
|
+ v_uniq_entity_id = EXEC DISTINCT entity_id FROM tb_cal_entities;
|
|
|
+
|
|
|
+ // 按照 MySQL 建好各表
|
|
|
+ tb_bfi_indicator = create_entity_bfi_indicator(iif(entityType=='PF', true, false));
|
|
|
+
|
|
|
+ // 分批跑
|
|
|
+ i = 0;
|
|
|
+ batch_size = 100;
|
|
|
+
|
|
|
+
|
|
|
+ do {
|
|
|
+
|
|
|
+ entities = SELECT * FROM tb_cal_entities WHERE entity_id IN v_uniq_entity_id[i : min(v_uniq_entity_id.size(), i+batch_size)];
|
|
|
+
|
|
|
+ if(entities.isVoid() || entities.size() == 0) break;
|
|
|
+
|
|
|
+ // 200ms
|
|
|
+ entity_info = SELECT entity_id, end_date.temporalParse('yyyy-MM') AS end_date, inception_date, factor_id AS benchmark_id, ini_value
|
|
|
+ FROM ej(entities, get_entity_info(entityType, entities.entity_id), 'entity_id');
|
|
|
+
|
|
|
+ // 取月收益 (12s)
|
|
|
+ rets = get_monthly_ret(entityType, entity_info.entity_id, very_old_day, entity_info.end_date.max().temporalFormat('yyyy-MM-dd').temporalParse('yyyy-MM-dd').monthEnd(), isFromMySQL);
|
|
|
+
|
|
|
+ // 把 yyyy-MM 格式的 end_date 改成 dolphin 的 MONTH
|
|
|
+ v_end_date = rets.end_date.temporalParse('yyyy-MM');
|
|
|
+ rets.replaceColumn!('end_date', v_end_date);
|
|
|
+
|
|
|
+ if(!rets.isVoid() && rets.size() > 0) {
|
|
|
+
|
|
|
+ // 计算月度指标 (5s)
|
|
|
+ indicators = cal_monthly_indicators(entityType, 'BFI', rets);
|
|
|
+
|
|
|
+ // 仿照MySQL的表结构准备好记录 （1s)
|
|
|
+ generate_entity_bfi_indicator(entity_info, indicators, true, tb_bfi_indicator);
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ i += batch_size;
|
|
|
+
|
|
|
+ } while (i <= v_uniq_entity_id.size());
|
|
|
+
|
|
|
+
|
|
|
+ if(! tb_bfi_indicator.isVoid() && tb_bfi_indicator.size() > 0) {
|
|
|
+
|
|
|
+ // save data to MySQL
|
|
|
+ try {
|
|
|
+
|
|
|
+ t_desc = get_bfi_indicator_table_description(entityType);
|
|
|
+
|
|
|
+ chg_columns_for_mysql(tb_bfi_indicator, t_desc.sec_id_col[0]);
|
|
|
+ db_name = t_desc.table_name[0].split('.')[0];
|
|
|
+ save_and_sync(tb_bfi_indicator, t_desc.table_name[0].strReplace(db_name, 'raw_db'), t_desc.table_name[0].strReplace(db_name, 'raw_db'));
|
|
|
+
|
|
|
+ // 数据初始化时将指标存入本地，做排名之用
|
|
|
+ if(end_day <= get_ini_data_const()['date'])
|
|
|
+ save_table(tb_bfi_indicator, t_desc.table_name[0], false);
|
|
|
+
|
|
|
+ } catch(ex) {
|
|
|
+
|
|
|
+ //TODO: Log errors
|
|
|
+ rt = ex;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return rt;
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+/*
|
|
|
+ * 计算并存储基金经理和公司月度净值
|
|
|
+ *
|
|
|
+ * @return <TABLE>: [COLUMNS] entity_id, curve_type, strategy, end_date, price_date, ret, nav
|
|
|
+ *
|
|
|
+ * NOTE: 基本上和 cal_and_save_mc_monthly_nav 一样
|
|
|
+ *
|
|
|
+ */
|
|
|
+def cal_and_save_mc_weekly_nav(entity_type, entity_date, is_save_local) {
|
|
|
+
|
|
|
+ rt = '';
|
|
|
+
|
|
|
+ if( !(entity_type in ['PL', 'CO']) ) return rt;
|
|
|
+
|
|
|
+ if(entity_date.isVoid() || entity_date.size() == 0) return rt;
|
|
|
+
|
|
|
+ // 准备类似MySQL结构的数据表
|
|
|
+ tb_entity_nav = create_mc_nav();
|
|
|
+
|
|
|
+ // 暂时与 MySQL 保持一致，只计算公募，私募，公私募综合三条时间序列。未来可细化至公、私募+主策略
|
|
|
+ d_curve_type = dict(INT, INT);
|
|
|
+ d_curve_type[1] = 1; // 私募
|
|
|
+ d_curve_type[4] = 2; // 公募
|
|
|
+ d_curve_type[7] = -99; // 公私募综合
|
|
|
+
|
|
|
+ // 分批跑
|
|
|
+ i = 0;
|
|
|
+ batch_size = 1000;
|
|
|
+
|
|
|
+ all_entity_id = entity_date.entity_id.distinct();
|
|
|
+
|
|
|
+ do { // 14 sec
|
|
|
+
|
|
|
+ tb_entity = SELECT entity_id, effective_date FROM entity_date
|
|
|
+ WHERE entity_id IN all_entity_id[i : min(all_entity_id.size(), i+batch_size)];
|
|
|
+
|
|
|
+ if(tb_entity.isVoid() || tb_entity.size() == 0) break;
|
|
|
+
|
|
|
+ s_json = tb_entity.toStdJson();
|
|
|
+ t_ret = get_mc_average_return(entity_type, 'w', s_json, 0, 1, true);
|
|
|
+
|
|
|
+ for(cur in d_curve_type.keys()) {
|
|
|
+//cur=7
|
|
|
+ tmp = SELECT entity_id, cur AS curve_type, 0 AS strategy, effective_date, price_date, ret, incl_cal_cnt
|
|
|
+ FROM t_ret WHERE raise_type = d_curve_type[cur] AND strategy = -99; // 目前只需要全策略
|
|
|
+
|
|
|
+ // 计算周收益
|
|
|
+ tb_nav = cal_mc_nav_by_return(entity_type, tmp, 'w');
|
|
|
+
|
|
|
+ INSERT INTO tb_entity_nav
|
|
|
+ SELECT entity_id, curve_type, strategy, effective_date AS year_week, price_date, nav, ret, incl_cal_cnt
|
|
|
+ FROM ej(tb_nav, tmp, ['entity_id', 'curve_type', 'strategy', 'effective_date']);
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ i += batch_size;
|
|
|
+
|
|
|
+ } while (i <= all_entity_id.size());
|
|
|
+
|
|
|
+
|
|
|
+ if(! tb_entity_nav.isVoid() && tb_entity_nav.size() > 0) {
|
|
|
+
|
|
|
+ // save data to MySQL (12 sec)
|
|
|
+ try {
|
|
|
+
|
|
|
+ tb_entity_nav.rename!('entity_id', iif(entity_type == 'PL', 'manager_id', 'company_id'));
|
|
|
+ save_and_sync(tb_entity_nav, iif(entity_type == 'PL', 'raw_db.manager_nav', 'raw_db.company_nav'), );
|
|
|
+
|
|
|
+ // 数据初始化时将指标存入本地
|
|
|
+ if(is_save_local == true) {
|
|
|
+ save_table(tb_entity_nav, iif(entity_type == 'PL', 'mfdb.manager_nav', 'mfdb.company_nav'), false);
|
|
|
+ }
|
|
|
+
|
|
|
+ } catch(ex) {
|
|
|
+
|
|
|
+ //TODO: Log errors
|
|
|
+ rt += ex;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return rt;
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+/*
|
|
|
+ * [定时任务]: 基金经理/公司周净值计算
|
|
|
+ *
|
|
|
+ * Example: CalMCWeeklyNavTask('CO', 2024.11.04);
|
|
|
+ */
|
|
|
+def CalMCWeeklyNavTask(entity_type, updatetime) {
|
|
|
+//updatetime = 2024.11.05;
|
|
|
+//entity_type = 'PL';
|
|
|
+ if(!(entity_type IN ['PL', 'CO'])) return;
|
|
|
+
|
|
|
+ is_save_local = iif(updatetime <= get_ini_data_const()['updatetime'], true, false);
|
|
|
+
|
|
|
+ // 60 sec 简化起见，不区分curve_type, strategy； TODO: 性能能否优化？
|
|
|
+ if(entity_type == 'PL') {
|
|
|
+ entity_date = get_manager_list_by_fund_updatetime(updatetime, 'w');
|
|
|
+ entity_date.rename!('manager_id', 'entity_id');
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ entity_date = get_company_list_by_fund_updatetime(updatetime, 'w');
|
|
|
+ entity_date.rename!('company_id', 'entity_id');
|
|
|
+ }
|
|
|
+
|
|
|
+ // 15 sec
|
|
|
+ cal_and_save_mc_weekly_nav(entity_type, entity_date, is_save_local);
|
|
|
+
|
|
|
+ entity_date = null;
|
|
|
+
|
|
|
+}
|
