#!/usr/bin/env python3 """Where are the losses coming from? Reads Sailar-x's journal (and, with --mt5, MetaTrader's own deal history). python loss_report.py --config config.mt5.json what the journal says python loss_report.py --config config.mt5.json --mt5 ALSO asks MT5 who really closed each trade (needs MT5 open) It only reads. It never changes anything. """ import argparse import math import sys import time from datetime import datetime, timedelta import pandas as pd from sailar import config from sailar.journal import Journal def wilson(k, n, z=1.96): if n == 0: return (0.0, 1.0) p, d = k / n, 1 + z * z / n c, m = p + z * z / (2 * n), z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) return ((c - m) / d, (c + m) / d) def r_of(t): return t["pnl"] / t["risk_amount"] if t.get("risk_amount") else 0.0 def group(trades, keyfn): out = {} for t in trades: out.setdefault(keyfn(t), []).append(t) return out def line(label, ts): n = len(ts) if not n: return f" {label:<14} none" w = sum(1 for t in ts if t["pnl"] > 0) return f" {label:<14} {n:3} trades win {w / n:4.0%} net {sum(t['pnl'] for t in ts):+10.2f} avg {sum(r_of(t) for t in ts) / n:+.2f}R" def head(t): print("\n" + t + "\n" + "-" * len(t)) def journal_report(j, version, notes): trades = j.trades(version, "closed") head(f"1. THE NUMBERS (strategy v{version})") if not trades: print(" No closed trades yet.") return n = len(trades) wins = [t for t in trades if t["pnl"] > 0] gp, gl = sum(t["pnl"] for t in wins), -sum(t["pnl"] for t in trades if t["pnl"] <= 0) lo, hi = wilson(len(wins), n) print(f" {n} trades, net {sum(t['pnl'] for t in trades):+.2f}, win rate {len(wins) / n:.0%} (real value is likely between {lo:.0%} and {hi:.0%})," f" profit factor {gp / gl if gl else 999:.2f}, average {sum(r_of(t) for t in trades) / n:+.2f}R per trade") if n < 30: print(f" !! Only {n} trades. That is far too few to say the system is good OR bad. Judge it after 100+.") notes.append(f"Only {n} trades so far: wait for 100+ before drawing conclusions.") head("2. WHO ENDED EACH TRADE") by = group(trades, lambda t: t.get("exit_reason") or "?") for k, ts in sorted(by.items(), key=lambda kv: sum(t["pnl"] for t in kv[1])): print(line(k, ts)) ext = by.get("CLOSED", []) + by.get("EXTERNAL", []) if ext: print(f"\n !! {len(ext)} of {n} trades were closed by something OTHER than Sailar-x's stop, target or exit rules" f" (net {sum(t['pnl'] for t in ext):+.2f}).") print(" Sailar-x labels its own exits SL / TP / TREND_FLIP / TIME. 'CLOSED' means someone or something else closed them:") print(" you by hand in MT5, or another program. Run again with --mt5 to see exactly who.") notes.append(f"{len(ext)} trades were closed outside Sailar-x. Find out who (run with --mt5) and stop it, or the statistics are not Sailar-x's.") head("3. VERY SHORT TRADES (closed within 1 candle)") quick = [t for t in trades if (t.get("bars_held") or 0) <= 1] print(line("<= 1 candle", quick)) qlosers = [t for t in quick if t["pnl"] < 0] if len(quick) >= 3 and len(qlosers) >= 2: notes.append(f"{len(quick)} trades lasted one candle or less ({len(qlosers)} lost). Trades that short are almost always a " "whipsaw or someone closing by hand. The spread alone costs more than they can make.") head("4. STOP SIZE vs SPREAD (the silent loss)") stops, pcts = [], [] for t in trades: if t.get("entry") and t.get("sl"): d = abs(t["entry"] - t["sl"]) stops.append(d / (0.01 if t["entry"] > 20 else 0.0001)) pcts.append(d / t["entry"] * 100) if stops: crypto_like = sum(pcts) / len(pcts) > 0.12 and (sum(t["entry"] for t in trades) / len(trades)) > 500 if crypto_like: avg = sum(pcts) / len(pcts) print(f" Average stop distance: {avg:.2f}% of the price.") for sp in (0.01, 0.03, 0.06): print(f" With a spread of {sp}% of the price, the spread alone is {sp / avg:.0%} of your risk on every trade.") else: avg = sum(stops) / len(stops) print(f" Average stop distance: {avg:.1f} pips.") for sp in (0.8, 1.0, 1.6): print(f" With a {sp} pip spread, the spread alone is {sp / avg:.0%} of your risk on every trade (it is paid before the trade can win).") if avg < 10: notes.append(f"Stops average only {avg:.1f} pips. A normal 1 pip spread then eats {1 / avg:.0%} of every trade's risk. " "Use a bigger stop (atr_stop_mult up) or a slower timeframe (30m or 1h). The guard also skips trades when the spread is too big for the stop.") head("5. SIDE, CONFIDENCE, HOUR (look for buckets that lose)") for k, ts in sorted(group(trades, lambda t: t["side"]).items()): print(line(k, ts)) for k, ts in sorted(group(trades, lambda t: (t.get("confidence") or 0) // 10 * 10).items()): print(line(f"conf {k}-{k + 9}%", ts)) hours = group(trades, lambda t: pd.Timestamp(t["entry_time"]).hour) for k, ts in sorted(hours.items()): print(line(f"{k:02d}:00 UTC", ts)) for k, ts in hours.items(): if len(ts) >= 4 and sum(r_of(t) for t in ts) / len(ts) < -0.3: notes.append(f"Trades opened around {k:02d}:00 UTC lose on average ({len(ts)} trades). Consider not trading that hour.") head("6. STREAKS") cur = mx = 0 for t in trades: cur = cur + 1 if t["pnl"] < 0 else 0 mx = max(mx, cur) print(f" Longest losing streak: {mx} trades in a row.") days = max(1, (pd.Timestamp(trades[-1]['exit_time']) - pd.Timestamp(trades[0]['entry_time'])).days + 1) print(f" Trades per day: {n / days:.1f}") head("7. CRASHES AND WARNINGS (last 7 days)") since = str(pd.Timestamp.now(tz="UTC") - pd.Timedelta(days=7)) crashes = [e for e in j.events(500, "CRASH") if e["ts"] >= since] print(f" {len(crashes)} crashes.") for e in crashes[:3]: last = [l for l in e["msg"].splitlines() if l.strip()][-1] print(f" {e['ts'][:19]} {last[:120]}") if crashes: notes.append(f"{len(crashes)} crashes in 7 days. Each one counts against the go-live gate, and a crash while a trade is open is dangerous.") NAMES = {0: "you, by hand in the terminal", 1: "you, by hand on mobile", 2: "you, by hand on web", 3: "a program (API/EA)", 4: "stop loss", 5: "take profit", 6: "stop-out (margin)", 7: "rollover", 8: "variation margin", 9: "split"} def mt5_report(days, notes): head(f"8. WHAT METATRADER ITSELF SAYS (last {days} days)") from sailar.mt5conn import get_mt5 m = get_mt5() deals = m.history_deals_get(datetime.now() - timedelta(days=days), datetime.now() + timedelta(days=1)) or [] outs = [d for d in deals if d.entry in (m.DEAL_ENTRY_OUT, getattr(m, "DEAL_ENTRY_OUT_BY", 3))] if not outs: print(" No closed deals found.") classes = {} for d in outs: r = NAMES.get(d.reason, f"reason {d.reason}") sailar = 20260000 <= d.magic <= 20269999 who = r if d.reason in (4, 5) else (("Sailar-x (its own exit rule)" if sailar else "ANOTHER program") if d.reason == 3 else r) if d.reason in (4, 5) and not sailar: who += " on a trade that is not Sailar-x's" classes.setdefault(who, []).append(d) print(f" {datetime.fromtimestamp(d.time):%m-%d %H:%M} {d.symbol:8} {d.volume:6.2f} lots {d.profit:+9.2f} closed by: {who:<34} magic {d.magic} '{d.comment}'") print("\n Summary:") for who, ds in sorted(classes.items(), key=lambda kv: sum(d.profit for d in kv[1])): print(f" {who:<44} {len(ds):3} closes {sum(d.profit for d in ds):+10.2f}") hand = [d for who, ds in classes.items() if who.startswith("you") for d in ds] other = [d for who, ds in classes.items() if who.startswith("ANOTHER") for d in ds] if hand: notes.append(f"{len(hand)} closes were done BY HAND in MT5 (net {sum(d.profit for d in hand):+.2f}). Closing trades manually breaks the " "stop/target logic and makes the statistics meaningless. Let Sailar-x manage its own trades.") if other: notes.append(f"{len(other)} closes were done by ANOTHER program (not Sailar-x). Check for a second Sailar-x window or a test script left running.") head("9. OPEN POSITIONS THAT ARE NOT SAILAR-X'S") mine = 0 for p in m.positions_get() or []: if 20260000 <= p.magic <= 20269999: mine += 1 continue print(f" {p.symbol} {'BUY' if p.type == 0 else 'SELL'} {p.volume} lots, opened {datetime.fromtimestamp(p.time):%m-%d %H:%M}," f" stop loss {p.sl or 'NONE'}, floating {p.profit:+.2f}, magic {p.magic}") notes.append(f"A {p.volume} lot {p.symbol} position (magic {p.magic}) is open and is not Sailar-x's" + (" and has NO stop loss." if not p.sl else ".")) print(f" (Sailar-x's own open positions: {mine})") print(" Note: if a manual trade shares the account, MT5's balance/equity includes it, so the account curve is not purely Sailar-x.") def main(): ap = argparse.ArgumentParser() ap.add_argument("--config") ap.add_argument("--mt5", action="store_true") ap.add_argument("--days", type=int, default=7) a = ap.parse_args() cfg = config.load(a.config) j = Journal(config.STATE / "journal.db") rows = j.all_versions() if not rows: sys.exit("No journal yet. Run the agent first.") version = (j.active_version() or j.version(rows[-1]["version"]))["version"] notes = [] journal_report(j, version, notes) if a.mt5: try: mt5_report(a.days, notes) except Exception as e: print(f"\n(Could not read MT5: {e})") head("WHAT TO DO ABOUT IT") for i, n in enumerate(notes or ["Nothing alarming stands out in this data."], 1): print(f" {i}. {n}") print("\n Remember: some losing trades are normal and expected. The goal is to remove AVOIDABLE losses (costs, whipsaws,\n" " outside interference, crashes), not to make every trade win.") if __name__ == "__main__": main()