{
  "title": "Talooner — PR review activity",
  "uid": "talooner-pr-review",
  "tags": [
    "talooner",
    "code-review"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "editable": true,
  "time": {
    "from": "now-7d",
    "to": "now"
  },
  "refresh": "1m",
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "Prometheus",
        "type": "datasource",
        "query": "prometheus",
        "current": {},
        "hide": 0
      },
      {
        "name": "repo",
        "label": "Repository",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": "label_values(talooner_pr_evaluations_total, repo)",
        "includeAll": true,
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "title": "Auto-approval rate",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 8,
        "x": 0,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.3
              },
              {
                "color": "green",
                "value": 0.6
              }
            ]
          }
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\", verdict=\"approve\"}[$__rate_interval])) / clamp_min(sum(rate(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval])), 1e-9)"
        }
      ]
    },
    {
      "id": 2,
      "title": "Needs a human (block) rate",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 8,
        "x": 8,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.3
              },
              {
                "color": "red",
                "value": 0.6
              }
            ]
          }
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\", verdict=\"block\"}[$__rate_interval])) / clamp_min(sum(rate(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval])), 1e-9)"
        }
      ]
    },
    {
      "id": 3,
      "title": "Evaluation error rate",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 8,
        "x": 16,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.01
              },
              {
                "color": "red",
                "value": 0.05
              }
            ]
          }
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(talooner_action_requests_total{repo=~\"$repo\", result=\"error\"}[$__rate_interval])) / clamp_min(sum(rate(talooner_action_requests_total{repo=~\"$repo\"}[$__rate_interval])), 1e-9)"
        }
      ]
    },
    {
      "id": 4,
      "title": "Verdict mix over time (execute)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 40,
            "stacking": {
              "mode": "normal"
            }
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (verdict) (rate(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval]))",
          "legendFormat": "{{verdict}}"
        }
      ]
    },
    {
      "id": 5,
      "title": "Evaluation latency (p50 / p95 / p99)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 5
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.50, sum by (le) (rate(talooner_pr_evaluation_duration_seconds_bucket{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval])))",
          "legendFormat": "p50"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(talooner_pr_evaluation_duration_seconds_bucket{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval])))",
          "legendFormat": "p95"
        },
        {
          "refId": "C",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.99, sum by (le) (rate(talooner_pr_evaluation_duration_seconds_bucket{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval])))",
          "legendFormat": "p99"
        }
      ]
    },
    {
      "id": 6,
      "title": "Top repositories",
      "type": "table",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 13
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "options": {
        "showHeader": true
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk(10, sum by (repo) (increase(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__range])))",
          "instant": true,
          "format": "table"
        }
      ]
    },
    {
      "id": 7,
      "title": "Action mix (execute)",
      "type": "piechart",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 13
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right"
        },
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (verb) (increase(talooner_pr_actions_total{repo=~\"$repo\", mode=\"execute\"}[$__range]))",
          "legendFormat": "{{verb}}",
          "instant": true
        }
      ]
    },
    {
      "id": 8,
      "title": "LLM reviews by result",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 21
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 40,
            "stacking": {
              "mode": "normal"
            }
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (result) (rate(talooner_llm_reviews_total{repo=~\"$repo\"}[$__rate_interval]))",
          "legendFormat": "{{result}}"
        }
      ]
    },
    {
      "id": 9,
      "title": "Warnings by code (ruleset health)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 21
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 20
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (code) (rate(talooner_pr_warnings_total{repo=~\"$repo\"}[$__rate_interval]))",
          "legendFormat": "{{code}}"
        }
      ]
    },
    {
      "id": 10,
      "title": "Rules fired per evaluation (execute)",
      "type": "heatmap",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 29
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "options": {
        "calculate": false,
        "cellGap": 1,
        "yAxis": {
          "unit": "short"
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (le) (rate(talooner_pr_rules_fired_bucket{repo=~\"$repo\", mode=\"execute\"}[$__rate_interval]))",
          "legendFormat": "{{le}}",
          "format": "heatmap"
        }
      ]
    },
    {
      "id": 11,
      "type": "row",
      "title": "LLM cost — needs a model price (TALOONER_MODEL_COST_INPUT / _OUTPUT)",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 37
      },
      "panels": []
    },
    {
      "id": 12,
      "title": "LLM cost in range",
      "description": "What Talooner's LLM calls cost in the selected time range, priced with the model's configured cost.",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 0,
        "y": 38
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "decimals": 2
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(increase(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\"}[$__range]))",
          "instant": true
        }
      ]
    },
    {
      "id": 13,
      "title": "Cost per review",
      "description": "LLM cost divided by the number of PR reviews in the range.",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 6,
        "y": 38
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "decimals": 4
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(increase(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\"}[$__range])) / clamp_min(sum(increase(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__range])), 1)",
          "instant": true
        }
      ]
    },
    {
      "id": 14,
      "title": "LLM tokens in range",
      "description": "Input + output tokens of Talooner's LLM calls.",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 12,
        "y": 38
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          }
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(increase(opentalon_plugin_llm_tokens_total{repo=~\"$repo\"}[$__range]))",
          "instant": true
        }
      ]
    },
    {
      "id": 15,
      "title": "Reviews in range",
      "description": "PR reviews (execute mode) in the range.",
      "type": "stat",
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 18,
        "y": 38
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          }
        }
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(increase(talooner_pr_evaluations_total{repo=~\"$repo\", mode=\"execute\"}[$__range]))",
          "instant": true
        }
      ]
    },
    {
      "id": 16,
      "title": "Cost by plugin and action",
      "description": "Where the money goes: LLM cost per plugin action, e.g. talooner · evaluate_pr.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 43
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "stacking": {
              "mode": "normal"
            }
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (plugin, action) (increase(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\"}[$__interval]))",
          "legendFormat": "{{plugin}} · {{action}}"
        }
      ]
    },
    {
      "id": 17,
      "title": "Cost by model",
      "description": "LLM cost per model.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 43
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "stacking": {
              "mode": "normal"
            }
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (model) (increase(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\"}[$__interval]))",
          "legendFormat": "{{model}}"
        }
      ]
    },
    {
      "id": 18,
      "title": "Top repositories by cost",
      "type": "table",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 51
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "decimals": 2
        }
      },
      "options": {
        "showHeader": true
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk(10, sum by (repo) (increase(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\"}[$__range])))",
          "instant": true,
          "format": "table"
        }
      ]
    },
    {
      "id": 19,
      "title": "Tokens by plugin (input / output)",
      "description": "Token throughput per plugin, split into input and output.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 51
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (plugin, direction) (rate(opentalon_plugin_llm_tokens_total{repo=~\"$repo\"}[$__rate_interval]))",
          "legendFormat": "{{plugin}} · {{direction}}"
        }
      ]
    },
    {
      "id": 20,
      "title": "Top pull requests by LLM cost",
      "type": "table",
      "description": "Needs TALOONER_METRICS_PR_LABEL=true. Each PR's total so far — LLM cost, tokens and reviews across all its runs (since the gateway started) — for PRs active in the selected range. Click a PR to open it.",
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 59
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "instant": true,
          "format": "table",
          "expr": "topk(15, sum by (repo, pr) (max_over_time(opentalon_plugin_llm_cost_usd_total{repo=~\"$repo\", pr!=\"\"}[$__range]))) * on (repo, pr) group_left(title, url) clamp(topk by (repo, pr) (1, max by (repo, pr, title, url) (talooner_pr_info{repo=~\"$repo\"})), 1, 1)"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "instant": true,
          "format": "table",
          "expr": "sum by (repo, pr) (max_over_time(opentalon_plugin_llm_tokens_total{repo=~\"$repo\", pr!=\"\"}[$__range]))"
        },
        {
          "refId": "C",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "instant": true,
          "format": "table",
          "expr": "sum by (repo, pr) (max_over_time(talooner_pr_evaluations_total{repo=~\"$repo\", pr!=\"\", mode=\"execute\"}[$__range]))"
        }
      ],
      "transformations": [
        {
          "id": "merge",
          "options": {}
        },
        {
          "id": "filterByValue",
          "options": {
            "type": "exclude",
            "match": "any",
            "filters": [
              {
                "fieldName": "Value #A",
                "config": {
                  "id": "isNull",
                  "options": {}
                }
              }
            ]
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "url": true
            },
            "indexByName": {
              "repo": 0,
              "pr": 1,
              "title": 2,
              "Value #A": 3,
              "Value #B": 4,
              "Value #C": 5
            },
            "renameByName": {
              "Value #A": "LLM cost",
              "Value #B": "Tokens",
              "Value #C": "Reviews",
              "repo": "Repository",
              "pr": "PR",
              "title": "Title"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "sort": [
              {
                "field": "LLM cost",
                "desc": true
              }
            ]
          }
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "LLM cost"
            },
            "properties": [
              {
                "id": "unit",
                "value": "currencyUSD"
              },
              {
                "id": "decimals",
                "value": 3
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Tokens"
            },
            "properties": [
              {
                "id": "unit",
                "value": "short"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Reviews"
            },
            "properties": [
              {
                "id": "decimals",
                "value": 0
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "PR"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "title": "Open ${__value.text}",
                    "url": "${__data.fields.url}",
                    "targetBlank": true
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Title"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "title": "Open pull request",
                    "url": "${__data.fields.url}",
                    "targetBlank": true
                  }
                ]
              }
            ]
          }
        ]
      },
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "LLM cost",
            "desc": true
          }
        ]
      }
    }
  ],
  "description": "Talooner pipeline-step metrics pushed to a Prometheus aggregation gateway. https://talooner.dev/install/#pipeline"
}