feat: model prices and latency in browser, Waypoint-style settings tabs
- Model browser: per-million input/output price on every row; time to first token (p50) and tokens/s for the busiest OpenRouter provider, fetched lazily for visible rows and cached 10 minutes. - Fix: Free badge and free-first sorting (prices are strings), and the list was empty until typing. - Settings: text-only tabs styled like Waypoint's; drop the heading that repeated the tab name.
This commit is contained in:
1 parent
b4d962047b
commit
564e3ea8b3
4 files changed
+187
-83
No files matched your search
+150
-23
@@ -4,9 +4,69 @@ interface OpenRouterModel {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
/** USD per token, as decimal strings (e.g. "0.0000001"); "-1" means variable. */
|
||||
pricing?: {
|
||||
prompt: number;
|
||||
completion: number;
|
||||
prompt: string;
|
||||
completion: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface Percentiles {
|
||||
p50: number;
|
||||
}
|
||||
|
||||
interface OpenRouterEndpoint {
|
||||
provider_name: string;
|
||||
latency_last_30m: Percentiles | null;
|
||||
throughput_last_30m: Percentiles | null;
|
||||
perf_last_30m_by_workload?: {
|
||||
text_generation?: { request_count?: number };
|
||||
};
|
||||
}
|
||||
|
||||
interface ModelPerf {
|
||||
provider: string;
|
||||
/** Time to first token, p50, in ms. */
|
||||
latencyMs: number;
|
||||
/** Output tokens per second, p50. */
|
||||
tokensPerSecond: number | null;
|
||||
}
|
||||
|
||||
/** OpenRouter's stats cover the last 30 minutes; reuse them for a while across modal openings. */
|
||||
const PERF_TTL_MS = 10 * 60 * 1000;
|
||||
const PERF_DEBOUNCE_MS = 300;
|
||||
const PERF_CONCURRENCY = 6;
|
||||
const perfCache = new Map<string, { at: number; perf: ModelPerf | null }>();
|
||||
|
||||
/** "$0.10" for a per-token price string, scaled to per million tokens. */
|
||||
function formatPerMillion(perToken: string): string {
|
||||
const perMillion = parseFloat(perToken) * 1_000_000;
|
||||
if (perMillion >= 1) return `$${perMillion.toFixed(2)}`;
|
||||
return `$${perMillion.toPrecision(2)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks the endpoint that served the most requests, the one OpenRouter
|
||||
* most likely routes to, among those reporting latency.
|
||||
*/
|
||||
async function fetchPerf(modelId: string, apiKey: string): Promise<ModelPerf | null> {
|
||||
const resp = await fetch(`https://openrouter.ai/api/v1/models/${modelId}/endpoints`, {
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
});
|
||||
if (!resp.ok) return null;
|
||||
const endpoints: OpenRouterEndpoint[] = (await resp.json()).data?.endpoints ?? [];
|
||||
const busiest = endpoints
|
||||
.filter((e) => e.latency_last_30m)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(b.perf_last_30m_by_workload?.text_generation?.request_count ?? 0) -
|
||||
(a.perf_last_30m_by_workload?.text_generation?.request_count ?? 0)
|
||||
)[0];
|
||||
if (!busiest?.latency_last_30m) return null;
|
||||
return {
|
||||
provider: busiest.provider_name,
|
||||
latencyMs: busiest.latency_last_30m.p50,
|
||||
tokensPerSecond: busiest.throughput_last_30m?.p50 ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,9 +74,15 @@ export class ModelSuggestModal extends SuggestModal<OpenRouterModel> {
|
||||
private allModels: OpenRouterModel[] = [];
|
||||
private cached = false;
|
||||
private onChoose: (modelId: string) => void;
|
||||
/** Latency needs an OpenRouter key; without one, rows show price only. */
|
||||
private apiKey: string;
|
||||
/** Perf elements of the latest render pass, filled in once stats arrive. */
|
||||
private perfEls = new Map<string, HTMLElement>();
|
||||
private perfTimer: number | null = null;
|
||||
|
||||
constructor(app: App, onChoose: (modelId: string) => void) {
|
||||
constructor(app: App, apiKey: string, onChoose: (modelId: string) => void) {
|
||||
super(app);
|
||||
this.apiKey = apiKey;
|
||||
this.onChoose = onChoose;
|
||||
this.setPlaceholder("Search OpenRouter models...");
|
||||
this.setInstructions([
|
||||
@@ -35,18 +101,34 @@ export class ModelSuggestModal extends SuggestModal<OpenRouterModel> {
|
||||
this.allModels = (data.data || [])
|
||||
.filter((m: OpenRouterModel) => m.id && !m.id.includes(":"))
|
||||
.sort((a: OpenRouterModel, b: OpenRouterModel) => {
|
||||
const aFree = a.pricing?.prompt === 0 ? 0 : 1;
|
||||
const bFree = b.pricing?.prompt === 0 ? 0 : 1;
|
||||
const aFree = this.isFree(a) ? 0 : 1;
|
||||
const bFree = this.isFree(b) ? 0 : 1;
|
||||
if (aFree !== bFree) return aFree - bFree;
|
||||
return (a.name || a.id).localeCompare(b.name || b.id);
|
||||
});
|
||||
this.cached = true;
|
||||
// The modal rendered its (empty) list before the fetch finished.
|
||||
this.inputEl.dispatchEvent(new Event("input"));
|
||||
} catch (e) {
|
||||
// Silently fail — the text input still works for manual entry
|
||||
}
|
||||
}
|
||||
|
||||
onClose() {
|
||||
if (this.perfTimer !== null) window.clearTimeout(this.perfTimer);
|
||||
this.perfEls.clear();
|
||||
}
|
||||
|
||||
private isFree(model: OpenRouterModel): boolean {
|
||||
return (
|
||||
parseFloat(model.pricing?.prompt ?? "") === 0 &&
|
||||
parseFloat(model.pricing?.completion ?? "") === 0
|
||||
);
|
||||
}
|
||||
|
||||
getSuggestions(query: string): OpenRouterModel[] {
|
||||
// A new render pass starts; stats for rows no longer shown aren't needed.
|
||||
this.perfEls.clear();
|
||||
const q = query.toLowerCase().trim();
|
||||
if (!q) return this.allModels.slice(0, 50);
|
||||
return this.allModels
|
||||
@@ -64,31 +146,76 @@ export class ModelSuggestModal extends SuggestModal<OpenRouterModel> {
|
||||
const nameEl = container.createDiv({ cls: "model-suggestion-name" });
|
||||
nameEl.setText(model.name || model.id);
|
||||
|
||||
const idEl = container.createDiv({ cls: "model-suggestion-id" });
|
||||
idEl.setText(model.id);
|
||||
idEl.style.fontSize = "0.8em";
|
||||
idEl.style.opacity = "0.6";
|
||||
if (this.isFree(model)) {
|
||||
nameEl.createSpan({ cls: "model-suggestion-badge", text: "Free" });
|
||||
}
|
||||
|
||||
if (model.pricing?.prompt === 0 && model.pricing?.completion === 0) {
|
||||
const badge = container.createSpan({ cls: "model-suggestion-badge" });
|
||||
badge.setText("Free");
|
||||
badge.style.marginLeft = "8px";
|
||||
badge.style.padding = "0 6px";
|
||||
badge.style.borderRadius = "4px";
|
||||
badge.style.backgroundColor = "var(--color-green)";
|
||||
badge.style.color = "var(--text-on-accent)";
|
||||
badge.style.fontSize = "0.75em";
|
||||
container.createDiv({ cls: "model-suggestion-id", text: model.id });
|
||||
|
||||
const stats = container.createDiv({ cls: "model-suggestion-stats" });
|
||||
const price = model.pricing;
|
||||
if (price && !this.isFree(model)) {
|
||||
stats.createSpan({
|
||||
text:
|
||||
parseFloat(price.prompt) < 0
|
||||
? "Variable price"
|
||||
: `${formatPerMillion(price.prompt)} in · ${formatPerMillion(price.completion)} out per M tokens`,
|
||||
});
|
||||
}
|
||||
if (this.apiKey) {
|
||||
const perfEl = stats.createSpan({ cls: "model-suggestion-perf" });
|
||||
const hit = perfCache.get(model.id);
|
||||
if (hit && Date.now() - hit.at < PERF_TTL_MS) {
|
||||
this.showPerf(perfEl, hit.perf);
|
||||
} else {
|
||||
perfEl.setText("latency…");
|
||||
this.perfEls.set(model.id, perfEl);
|
||||
this.schedulePerfFetch();
|
||||
}
|
||||
}
|
||||
|
||||
if (model.description) {
|
||||
const descEl = container.createDiv({ cls: "model-suggestion-desc" });
|
||||
descEl.setText(model.description.slice(0, 120));
|
||||
descEl.style.fontSize = "0.75em";
|
||||
descEl.style.opacity = "0.5";
|
||||
container.createDiv({
|
||||
cls: "model-suggestion-desc",
|
||||
text: model.description.slice(0, 120),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private showPerf(el: HTMLElement, perf: ModelPerf | null) {
|
||||
if (!perf) {
|
||||
el.setText("no latency data");
|
||||
return;
|
||||
}
|
||||
const firstToken = `first token ${(perf.latencyMs / 1000).toFixed(1)} s`;
|
||||
const speed = perf.tokensPerSecond ? ` · ${Math.round(perf.tokensPerSecond)} tok/s` : "";
|
||||
el.setText(`${firstToken}${speed} (${perf.provider})`);
|
||||
}
|
||||
|
||||
/** Waits for typing to pause so each keystroke doesn't fire ~50 requests. */
|
||||
private schedulePerfFetch() {
|
||||
if (this.perfTimer !== null) window.clearTimeout(this.perfTimer);
|
||||
this.perfTimer = window.setTimeout(() => {
|
||||
this.perfTimer = null;
|
||||
void this.fetchVisiblePerf();
|
||||
}, PERF_DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
private async fetchVisiblePerf() {
|
||||
const queue = [...this.perfEls.keys()];
|
||||
const worker = async () => {
|
||||
for (let id = queue.shift(); id !== undefined; id = queue.shift()) {
|
||||
if (!this.perfEls.has(id)) continue;
|
||||
const perf = await fetchPerf(id, this.apiKey).catch(() => null);
|
||||
perfCache.set(id, { at: Date.now(), perf });
|
||||
const el = this.perfEls.get(id);
|
||||
if (el?.isConnected) this.showPerf(el, perf);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: PERF_CONCURRENCY }, worker));
|
||||
}
|
||||
|
||||
onChooseSuggestion(model: OpenRouterModel) {
|
||||
this.onChoose(model.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
+11
-22
@@ -1,5 +1,5 @@
|
||||
import NibbleAI from "main";
|
||||
import { App, PluginSettingTab, SecretComponent, Setting, setIcon } from "obsidian";
|
||||
import { App, PluginSettingTab, SecretComponent, Setting } from "obsidian";
|
||||
import {
|
||||
SettingsManager,
|
||||
PostProcessingProvider,
|
||||
@@ -21,17 +21,16 @@ type TabId =
|
||||
|
||||
interface TabInfo {
|
||||
id: TabId;
|
||||
icon: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
const TABS: TabInfo[] = [
|
||||
{ id: "api-keys", icon: "key", label: "API Keys" },
|
||||
{ id: "transcription", icon: "mic", label: "Transcription" },
|
||||
{ id: "proofreading", icon: "wand", label: "Proofreading" },
|
||||
{ id: "title-generation", icon: "heading", label: "Title Generation" },
|
||||
{ id: "property-filler", icon: "pencil", label: "Property Filler" },
|
||||
{ id: "advanced", icon: "settings", label: "Advanced" },
|
||||
{ id: "api-keys", label: "API Keys" },
|
||||
{ id: "transcription", label: "Transcription" },
|
||||
{ id: "proofreading", label: "Proofreading" },
|
||||
{ id: "title-generation", label: "Title Generation" },
|
||||
{ id: "property-filler", label: "Property Filler" },
|
||||
{ id: "advanced", label: "Advanced" },
|
||||
];
|
||||
|
||||
export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
@@ -61,12 +60,9 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
|
||||
for (const tab of TABS) {
|
||||
const tabBtn = tabBar.createEl("button", {
|
||||
cls: `nibbleai-tab ${this.currentTab === tab.id ? "nibbleai-tab-active" : ""}`,
|
||||
attr: { "data-tab": tab.id },
|
||||
cls: `nibbleai-tab${this.currentTab === tab.id ? " is-active" : ""}`,
|
||||
text: tab.label,
|
||||
});
|
||||
const iconEl = tabBtn.createSpan({ cls: "nibbleai-tab-icon" });
|
||||
setIcon(iconEl, tab.icon);
|
||||
tabBtn.createSpan({ cls: "nibbleai-tab-label", text: tab.label });
|
||||
tabBtn.addEventListener("click", () => {
|
||||
this.currentTab = tab.id;
|
||||
this.display();
|
||||
@@ -102,7 +98,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
// ── API Keys Tab ──
|
||||
|
||||
private renderApiKeysTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "API Keys" });
|
||||
containerEl.createEl("p", {
|
||||
text: "Pick a secret from Obsidian's keychain (Settings → Keychain) for each provider. NibbleAI stores only the secret's name; the key itself stays in the keychain.",
|
||||
cls: "setting-item-description",
|
||||
@@ -172,8 +167,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
// ── Transcription Tab ──
|
||||
|
||||
private renderTranscriptionTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "Transcription" });
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName("Enable transcription")
|
||||
.setDesc("General settings for the audio-to-text feature")
|
||||
@@ -431,7 +424,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
}
|
||||
|
||||
private renderProofreadingTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "Proofreading" });
|
||||
containerEl.createEl("p", {
|
||||
text: "Proofread selected text with AI. Right-click any selection or use the command palette.",
|
||||
cls: "setting-item-description",
|
||||
@@ -553,7 +545,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
// ── Title Generation Tab ──
|
||||
|
||||
private renderTitleGenTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "Title Generation" });
|
||||
containerEl.createEl("p", {
|
||||
text: "Generate a title for the current note \u2014 either rename the file or set a frontmatter property.",
|
||||
cls: "setting-item-description",
|
||||
@@ -630,7 +621,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
// ── Property Filler Tab ──
|
||||
|
||||
private renderPropertyFillerTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "Property Filler" });
|
||||
containerEl.createEl("p", {
|
||||
text: "Fill any frontmatter property with AI-generated content.",
|
||||
cls: "setting-item-description",
|
||||
@@ -703,8 +693,6 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
// ── Advanced Tab ──
|
||||
|
||||
private renderAdvancedTab(containerEl: HTMLElement): void {
|
||||
containerEl.createEl("h2", { text: "Advanced" });
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName("Debug mode")
|
||||
.setDesc("Increase verbosity for troubleshooting")
|
||||
@@ -884,7 +872,8 @@ export class NibbleAISettingsTab extends PluginSettingTab {
|
||||
.setButtonText("Browse")
|
||||
.setIcon("search")
|
||||
.onClick(() => {
|
||||
new ModelSuggestModal(this.app, (modelId: string) => {
|
||||
const openRouterKey = getApiKey(this.app, this.plugin.settings.openRouterSecret);
|
||||
new ModelSuggestModal(this.app, openRouterKey, (modelId: string) => {
|
||||
onChange(modelId);
|
||||
textComponent.setValue(modelId);
|
||||
}).open();
|
||||
|
||||
Reference in new issue
Block a user