RKiding/Awesome-finance-skills / skills/alphaear-predictor
alphaear-predictor
UnverifiedMarket prediction skill using Kronos. Use when user needs finance market time-series forecasting or news-aware finance market adjustments.
scripts/utils/predictor/evaluation.py
View on GitHubimport os
import sys
import torch
import pandas as pd
import numpy as np
import glob
from loguru import logger
from datetime import datetime, timedelta
# Setup paths
KRONOS_DIR = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.dirname(os.path.dirname(KRONOS_DIR))
if SRC_DIR not in sys.path:
sys.path.insert(0, SRC_DIR)
from ..kronos.auto_synthesis_training import AutoSynthesisTrainer
from ..kronos.model import KronosPredictor
from ..visualizer import VisualizerTools
from ..schema.models import ForecastResult, KLinePoint
class NewsModelEvaluator:
def __init__(self, model_path=None):
self.trainer = AutoSynthesisTrainer()
self.device = self.trainer.device
if model_path is None:
# Try to find the latest model in exports/models
model_files = glob.glob(os.path.join(SRC_DIR, "exports/models/*.pt"))
if not model_files:
logger.warning("β οΈ No trained models found in exports/models/. Using base model (zero-init proj).")
else:
model_path = max(model_files, key=os.path.getctime)
if model_path:
self.load_weights(model_path)
def load_weights(self, path):
logger.info(f"π Loading model weights from {path}...")
checkpoint = torch.load(path, map_location=self.device)
self.trainer.model.news_proj.load_state_dict(checkpoint['news_proj_state_dict'])
logger.success("β
News projection layer loaded.")
def evaluate_range(self, start_idx=100, end_idx=200, pred_len=5):
# 1. Fetch Tickers
res = self.trainer.db.execute_query("SELECT code FROM stock_list")
all_tickers = [row['code'] for row in res]
test_tickers = all_tickers[start_idx:end_idx]
if not test_tickers:
logger.error(f"No tickers found in range {start_idx}-{end_idx}")
return
logger.info(f"π Evaluating News Model on stocks {start_idx} to {end_idx}...")
# 2. Discover Shocks
shocks = self.trainer.discover_shocks(test_tickers, pred_len=pred_len)
# 3. Associate News & Predict
self.trainer.model.eval()
predictor = KronosPredictor(self.trainer.model, self.trainer.tokenizer, device=self.device)
save_dir = os.path.join(SRC_DIR, "exports/evaluation_results")
os.makedirs(save_dir, exist_ok=True)
count = 0
for shock in shocks:
summary = self.trainer.find_reason_and_verify(shock)
if not summary:
continue
logger.info(f"π Testing shock: {shock['ticker']} on {shock['date']}")
# Embedding news
news_emb = self.trainer.embedder.encode(summary)
# Prediction
h = shock['history']
t = shock['target']
actuals = t['close'].values[:pred_len]
x_ts = pd.to_datetime(h['date'])
future_dates = pd.date_range(start=x_ts.iloc[-1] + timedelta(days=1), periods=pred_len, freq='B')
y_ts = pd.Series(future_dates)
# A. Base Prediction (No news)
p_base = predictor.predict(h, x_ts, y_ts, pred_len=pred_len, news_emb=None, verbose=False)
# B. News-Aware Prediction
p_news = predictor.predict(h, x_ts, y_ts, pred_len=pred_len, news_emb=news_emb, verbose=False)
# Calculate Improvement
b_preds = p_base['close'].values[:len(actuals)]
n_preds = p_news['close'].values[:len(actuals)]
b_mae = np.mean(np.abs(b_preds - actuals))
n_mae = np.mean(np.abs(n_preds - actuals))
improvement = (b_mae - n_mae) / (b_mae + 1e-6) * 100
# C. Visualize
try:
def to_kp_list(preds_df):
points = []
for idx, row in preds_df.iterrows():
points.append(KLinePoint(
date=str(idx)[:10], open=row['open'], high=row['high'],
low=row['low'], close=row['close'], volume=row.get('volume', 0)
))
return points
forecast_obj = ForecastResult(
ticker=shock['ticker'],
base_forecast=to_kp_list(p_base),
adjusted_forecast=to_kp_list(p_news),
rationale=summary
)
chart = VisualizerTools.generate_stock_chart(
df=h, ticker=shock['ticker'],
title=f"Test Eval: {shock['ticker']} ({shock['date']}) Imp: {improvement:.1f}%",
forecast=forecast_obj,
ground_truth=t[['date', 'open', 'high', 'low', 'close', 'volume']]
)
safe_date = shock['date'].replace("-", "")
filename = f"test_{shock['ticker']}_{safe_date}.html"
VisualizerTools.render_chart_to_file(chart, os.path.join(save_dir, filename))
logger.success(f"π Result for {shock['ticker']} saved. Base MAE: {b_mae:.4f}, News MAE: {n_mae:.4f}")
count += 1
except Exception as e:
logger.error(f"Visualization failed: {e}")
logger.info(f"π Finished evaluation. {count} cases visualized in {save_dir}")
if __name__ == "__main__":
# If you have a specific model, pass the path here. Otherwise it picks the latest.
evaluator = NewsModelEvaluator()
evaluator.evaluate_range(start_idx=100, end_idx=200, pred_len=1)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why β repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" β or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself β your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |