@kentcdodds/invisible-software

import pandas as pd, numpy as np, matplotlib
matplotlib.use("Agg"); import matplotlib.pyplot as plt, matplotlib.dates as mdates
E="epoch/"
B={ # name: (file, score col)
 "SWE-bench Verified (Epoch runs)":("swe_bench_verified.csv","Best score (across scorers)"),
 "Aider Polyglot":("aider_polyglot_external.csv","Percent correct"),
 "Terminal-Bench":("terminalbench_external.csv","Accuracy mean"),
 "OSWorld":("os_world_external.csv","Score"),
 "Cybench":("cybench_external.csv","Unguided % Solved"),
 "TheAgentCompany":("the_agent_company_external.csv","% Score"),
}
rows=[]
for n,(f,c) in B.items():
    d=pd.read_csv(E+f); d=d[["Model version","Release date",c]].dropna()
    d["date"]=pd.to_datetime(d["Release date"],errors="coerce"); d=d.dropna(subset=["date"])
    s=d[c].astype(float); 
    if s.max()<=1.0: s=s*100
    d["score"]=s; d["benchmark"]=n
    rows.append(d[["benchmark","Model version","date","score"]])
raw=pd.concat(rows); raw.to_csv("benchmark-raw.csv",index=False)
months=pd.date_range("2023-01-01","2026-10-01",freq="MS")
fr={}
for n,g in raw.groupby("benchmark"):
    g=g.sort_values("date"); ser=[]
    for m in months:
        h=g[g.date<m+pd.offsets.MonthEnd(0)]
        ser.append(h.score.max() if len(h) else np.nan)
    fr[n]=ser
F=pd.DataFrame(fr,index=months)
last=raw.date.max(); F=F[F.index<=last]
F["median"]=F.median(axis=1,skipna=True); F["n"]=F.drop(columns="median").notna().sum(axis=1)
F.loc[F.n<3,"median"]=np.nan
F["smoothed"]=F["median"].rolling(5,center=True,min_periods=2).mean()
F.to_csv("benchmark-trend-data.csv",index_label="month")
plt.rcParams.update({"font.size":22,"font.family":"DejaVu Sans"})
fig,ax=plt.subplots(figsize=(19.2,10.8),dpi=100)
cols=plt.cm.tab10.colors
for i,n in enumerate(B):
    s=F[n].dropna(); ax.step(s.index,s,where="post",color=cols[i],alpha=.45,lw=2.5)
    ax.text(s.index[-1]+pd.Timedelta(days=12),s.iloc[-1]+(-1.5 if "Terminal" in n else (-4 if "SWE" in n else (1.5 if "Aider" in n else 0))),n.replace(" (Epoch runs)",""),color=cols[i],fontsize=17,va="center",alpha=.9)
m=F["smoothed"].dropna(); ax.plot(m.index,m,color="black",lw=7,label="Median across benchmarks (smoothed)")
ax.set_title("AI agents keep getting better at engineering work",fontsize=40,weight="bold",pad=25)
ax.set_ylabel("State of the art (% of max)",fontsize=26); ax.set_ylim(0,100)
ax.set_xlim(pd.Timestamp("2024-01-01"),last+pd.Timedelta(days=150))
ax.xaxis.set_major_locator(mdates.MonthLocator(bymonth=[1,7])); ax.xaxis.set_major_formatter(mdates.DateFormatter("%b %Y"))
for sp in ["top","right"]: ax.spines[sp].set_visible(False)
ax.grid(axis="y",alpha=.3); ax.legend(loc="upper left",frameon=False,fontsize=22)
fig.text(.01,.015,"Best score to date, by model release date. Source: Epoch AI Benchmarking Hub (epoch.ai/benchmarks).\nBenchmarks: SWE-bench Verified, Aider Polyglot, Terminal-Bench, OSWorld, Cybench, TheAgentCompany (retrieved Oct 2026).",fontsize=13,color="#555")
fig.tight_layout(rect=(0,.03,1,1)); fig.savefig("benchmark-trend.png",facecolor="white")
# METR
d=pd.read_csv(E+"metr_time_horizons_external.csv"); d["date"]=pd.to_datetime(d["Release date"],errors="coerce")
d=d.dropna(subset=["date","Time horizon"]); d=d[d["METR version"].astype(str).str.contains("1.1")] if d["METR version"].astype(str).str.contains("1.1").sum()>10 else d
d=d.sort_values("date"); d["sota"]=d["Time horizon"].cummax(); d.to_csv("metr-data.csv",index=False)
fig,ax=plt.subplots(figsize=(19.2,10.8),dpi=100)
ax.scatter(d.date,d["Time horizon"],s=90,color="#1f77b4",alpha=.4,label="Each model")
ax.step(d.date,d.sota,where="post",color="black",lw=6,label="Best to date")
ax.set_yscale("log"); ticks=[1,4,15,60,240,960]; ax.set_ylim(1,1500); ax.set_yticks(ticks); ax.set_yticklabels(["1 min","4 min","15 min","1 hr","4 hr","16 hr"])
ax.set_title("AI agents can finish ever-longer software tasks",fontsize=40,weight="bold",pad=25)
ax.xaxis.set_major_locator(mdates.MonthLocator(bymonth=[1,7])); ax.xaxis.set_major_formatter(mdates.DateFormatter("%b %Y"))
ax.set_ylabel("Task length at 50% success\n(human expert time, log scale)",fontsize=24)
for sp in ["top","right"]: ax.spines[sp].set_visible(False)
ax.grid(alpha=.3); ax.legend(loc="upper left",frameon=False)
fig.text(.01,.015,"Data: METR Time Horizon (metr.org/time-horizons) via Epoch AI Benchmarking Hub. METR notes measurements above 16 hrs are unreliable.",fontsize=13,color="#555")
fig.tight_layout(rect=(0,.03,1,1)); fig.savefig("metr-time-horizon.png",facecolor="white")
print(raw.groupby("benchmark").agg(n=("score","size"),start=("date","min"),end=("date","max"),max=("score","max")))
print(d["METR version"].unique(), d.date.min(), d.date.max(), d.sota.max())
print(F[["median","smoothed","n"]].dropna().iloc[[0,-1]])