Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Stop grepping, start reasoning

Stop grepping, start reasoning

Skill-based Agentic SRE for Production incidents

SRE Extension is here: https://github.com/gemini-cli-extensions/sre

Avatar for Riccardo Carlesso

Riccardo Carlesso

November 06, 2026

More Decks by Riccardo Carlesso

Other Decks in Programming

Transcript

  1. ? SRE Anthem v0.13.1 Stage III ? Main Track ??

    Stop Grepping,Start Reasoning Skill-Based Agentic SRE for Production Incidents ?? Milan, Italy ? October 28, 2026 Riccardo Carlesso AI & SRE Developer Advocate Google Cloud ..made with Antigravity (function() { if (window._marpUniversalSpaceHandlerInstalled) return; window._marpUniversalSpaceHandlerInstalled = true; document.addEventListener('focusin', function(e) { var tag = e.target && e.target.tagName; if (tag === 'AUDIO' || tag === 'VIDEO' || tag === 'BUTTON') { e.target.blur(); } }, true); function getActiveSlide() { var active = document.querySelector('svg.bespoke-marp-slide.bespoke-marp-active, .bespoke-marp-active'); if (!active) { active = document.querySelector('section.bespoke-marp-active'); } return active; } function resetInactiveSlides() { var active = getActiveSlide(); document.querySelectorAll('video, audio').forEach(function(media) { if (!active || !active.contains(media)) { media.removeAttribute('data-played-by-space'); media.removeAttribute('data-sound-autoplayed'); if (!media.paused) { media.pause(); } } }); autoplayActiveMediaWithSound(active); scheduleDelayedReveals(active); } // Elements with class delayed-reveal-Ns get .revealed N seconds after their slide becomes active. function scheduleDelayedReveals(active) { document.querySelectorAll('[class*="delayed-reveal-"]').forEach(function(el) { var m = /delayed-reveal-(\d+)s/.exec(el.className); if (!m) return; var inActive = active && active.contains(el); if (!inActive) { if (el._revealTimer) { clearTimeout(el._revealTimer); el._revealTimer = null; } if (el.classList.contains('revealed')) { el.classList.remove('revealed'); el.style.opacity = ''; el.style.transform = ''; } return; } if (el.classList.contains('revealed') || el._revealTimer) return; el._revealTimer = setTimeout(function() { el._revealTimer = null; el.classList.add('revealed'); el.style.opacity = '1'; el.style.transform = 'translate(0, 0) scale(1)'; }, parseInt(m[1], 10) * 1000); }); } // Videos marked data-autoplay-sound start automatically WITH AUDIO when their slide becomes active. // (Browsers allow unmuted autoplay once the user has interacted with the page, e.g. pressed a key.) // If the browser still refuses, we fall back to muted autoplay so the demo never stalls. function autoplayActiveMediaWithSound(active) { if (!active) return; active.querySelectorAll('video[data-autoplay-sound], audio[data-autoplay-sound]').forEach(function(media) { if (media.hasAttribute('data-sound-autoplayed')) return; media.setAttribute('data-sound-autoplayed', 'true'); media.setAttribute('data-played-by-space', 'true'); media.muted = false; media.volume = 1.0; try { media.currentTime = 0; } catch (err) {} var p = media.play(); if (p !== undefined) { p.catch(function(err) { console.warn('[PRESENTER HANDLER] Unmuted autoplay blocked, falling back to muted:', err); media.muted = true; media.play().catch(function(){}); }); } }); } function findUnplayedMedia() { var active = getActiveSlide(); if (!active) return null; var mediaList = active.querySelectorAll('video, audio'); for (var i = 0; i < mediaList.length; i++) { var m = mediaList[i]; if (m.hasAttribute('data-no-space-autoplay')) continue; if (m.paused && !m.ended && !m.hasAttribute('data-played-by-space')) { return m; } } return null; } function advanceNextSlide() { resetInactiveSlides(); var nextBtn = document.querySelector('[data-bespoke-marp-osc="next"]'); if (nextBtn) { nextBtn.click(); setTimeout(resetInactiveSlides, 50); return; } document.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39, which: 39, bubbles: true })); setTimeout(resetInactiveSlides, 50); } var _lastAdvanceTime = 0; function handleForwardStep(e) { var now = Date.now(); if (now _lastAdvanceTime < 180) { if (e) { e.preventDefault(); e.stopPropagation(); e.stopImmediatePropagation(); } return; } _lastAdvanceTime = now; var tag = (document.activeElement && document.activeElement.tagName) || ''; if (tag === 'INPUT' || tag === 'TEXTAREA' || (document.activeElement && document.activeElement.isContentEditable)) return; if (tag === 'AUDIO' || tag === 'VIDEO') { document.activeElement.blur(); } var unplayed = findUnplayedMedia(); if (unplayed) { if (e) { e.preventDefault(); e.stopPropagation(); e.stopImmediatePropagation(); } unplayed.setAttribute('data-played-by-space', 'true'); var p = unplayed.play(); if (p !== undefined) { p.catch(function(err) { console.warn('[PRESENTER HANDLER] Play error:', err); unplayed.muted = true; unplayed.play().catch(function(){}); }); } return; } setTimeout(resetInactiveSlides, 30); setTimeout(resetInactiveSlides, 120); } window.addEventListener('keydown', function(e) { if (e && e.isTrusted === false) return; var isSpace = (e.code === 'Space' || e.key === ' ' || e.key === 'Spacebar' || e.keyCode === 32 || e.which === 32); var forwardKeyList = ['ArrowRight', 'PageDown', 'ArrowDown', 'KeyN', 'Right', 'Down', 'n', 'N']; var isForwardKey = isSpace || forwardKeyList.includes(e.code) || forwardKeyList.includes(e.key) || e.keyCode === 34 || e.keyCode === 39 || e.keyCode === 40; if (isForwardKey) { handleForwardStep(e); setTimeout(resetInactiveSlides, 30); setTimeout(resetInactiveSlides, 120); return; } var isBackwardKey = ['ArrowLeft', 'PageUp', 'ArrowUp', 'KeyP', 'Backspace', 'Left', 'Up', 'p', 'P'].includes(e.code) || ['ArrowLeft', 'PageUp', 'ArrowUp', 'KeyP', 'Backspace', 'Left', 'Up', 'p', 'P'].includes(e.key) || e.keyCode === 33 || e.keyCode === 37 || e.keyCode === 38; if (isBackwardKey) { setTimeout(resetInactiveSlides, 30); setTimeout(resetInactiveSlides, 100); return; } }, true); window.addEventListener('hashchange', function() { resetInactiveSlides(); setTimeout(resetInactiveSlides, 50); }); window.addEventListener('load', function() { setTimeout(resetInactiveSlides, 100); setTimeout(resetInactiveSlides, 300); }); setInterval(resetInactiveSlides, 300); window.addEventListener('storage', function() { setTimeout(resetInactiveSlides, 30); }); window.addEventListener('popstate', function() { resetInactiveSlides(); setTimeout(resetInactiveSlides, 50); }); document.addEventListener('click', function() { setTimeout(resetInactiveSlides, 50); }); })();
  2. 01' 3AM: PagerDuty Screams Rule #1 of Italian On-Call "Never

    debug production on an empty stomach." The 4-Minute Moka Window While the espresso brews, your agent investigates.
  3. 02' Have You Ever Been There? ? AI Agent: "I

    scaled the pods to 0 and all errors stopped! Was I a good boy? :)" ???? Show of Hands #1 Ever been paged at 3AM? ???? Show of Hands #2 Ever used AI for an outage investigation? ???? Show of Hands #3 Ever given AI root access to Production?
  4. 04' Enter sre-extension: 16 Portable SRE Runbooks Bring Your Own

    Stack Plain Markdown SKILL.md + Python scripts ? Built by Google SREs & DevRel Riccardo Ramón Szymon Madhavi ? github.com/gemini-cli-extensions/sre (Apache-2.0)
  5. 05' 02 Insidesre-extension Origin: Zurich Espresso to Cloud Next? safe-sre-investigator:

    Read-Only IAM Gate? 16 Skills: 11 Universal Brain + 5 Cloud Adapters? 1 Repo Across 5 CLI Harnesses? Live Chaos Arena & 12-Token Sparklines? Deterministic Python Graphs & 30s Postmortems?
  6. 06' Origin: From a Zurich Espresso to Cloud Next ?

    August 2025 ? Google Zurich Cafeteria Coffee with Ramon Llamas: open-sourcing Google ProdAgent ? February 2026 ? Expanding the Core Team Szymon Stawski & Madhavi Karra join the team!
  7. ? 15 Apr 2026 07'30" Cloud Next Las Vegas: sre-extension

    Is Launched! April 15, 2026 ? Google Cloud Next '26 (Las Vegas) From internal ProdAgent to 100% Apache-2.0 Open Source ? "Ops, I Did It Again: The SRE Extension Is Out!" github.com/gemini-cli-extensions/sre 16 Battle-Tested Skills ? OneMCP + Romin Irani ? 5 Harnesses ? How Do You Trigger It at 3 AM? It all starts with /investigation entrypoint ?
  8. 08' safe-sre-investigator: Read-Only IAM & Risk Gate ? Step 0:

    Triggered by the Orchestrator It all starts with /investigation entrypoint Layer 1: Hard IAM Boundary + safe_kubectl Read-Only Service Account (Mutations return HTTP 403) Layer 2: Mandatory Risk Badge on Every Command ? NONE ? ? LOW ? ? MEDIUM ? ? HIGH
  9. 10' 16 Skills in 2 Layers: Generic SRE vs. GCP

    Cloud ? Generic SRE Brain (9 Cloud-Agnostic Skills) investigation-entrypoint/ safe-sre-investigator/ generic-mitigations/ anomaly-detection/ data-ingestion/ monitoring-graphs/ postmortem-generator/ postmortem-aggregator/ pre-publish-checker/ ? GCP Cloud Adapters (7 Skills ? Forkable to AWS/PromQL in 4 Lines) gcp-setup/ gcp-mcp-setup/ gcp-playbooks/ gcp-slo-management/ gcp-architecture-discovery/ cloud-logging/ cloud-monitoring/
  10. 11' 1 Plugin -> 5 CLI Harnesses (Zero Lock-In) Antigravity

    (agy) .agents/ Claude Code .claude-plugin/ OpenAI Codex .codex-plugin/ GitHub Copilot .copilot-plugin/ Gemini CLI deprecated
  11. 13' Chaos Arena: Breaking Online Boutique on GKE 3 Fault

    Injections ? Built with Madhavi Karra Scenario 1: Wrong Pod Selector (checkoutservice) Scenario 2: Broken Canary (frontend-canary 50% 500s) Scenario 3: Firewall Route (VPC DENY blackhole) Madhavi ? Self-Paced Google Codelab Curious? Try it yourself: bit.ly/sre-codelab
  12. 14' Live Diagnosis: Two Simultaneous Faults in 180 Seconds Full

    6-Minute Terminal Recording youtu.be/bPCznsjW8BU (Zero Mocks, Live GKE) Why It Beats Single-Root-Cause Bias Isolates both frontend-canary (50% 500s) & checkoutservice NetworkPolicy drop
  13. 15' gcp-architecture-discovery: AI Needs Context What It Does (Async Background

    Subagent) Crawls your blast radius (GKE, Cloud Run, VPCs, Firewalls & Cloud SQL) in Read-Only SAFE_MODE What It Builds (./discover/<project>/) ? discover.json + Graphviz discover.png ? Topological wiki.<category>.md mental model
  14. 16' generic-mitigations: Stop the Bleeding Before Root Cause ? 1.

    Rollback Revert bad canary / config ? 2. Drain / Shift Shift traffic away from bad zone ? 3. Degrade / Flags Disable heavy feature flags ? 4. Upsize & Shed Scale replicas & shed load
  15. 17' cloud-monitoring: Sparklines for Context Optimization Context Diet ? Before:

    38 MB Raw Dump 967K JSON lines + 53K CSV rows -> Context OOM ? Context Diet ? After: 10-Token Sparkline |???? ?? ? ? ? ?| -> Instant glance in 10 tokens!
  16. 17' postmortem-generator: Investigation to Google Doc Automated 5-Step Postmortem Pipeline

    1. ? Investigation (investigation-entrypoint) 2. ?? Detailed RUNNING_DOC.md with UTC timestamps 3. ? RCA Found & correlated across layers 4. ? Blameless POMO.md with Action Items & Owners 5. ? Google Doc Created & ready for review!
  17. 18' monitoring-graphs: Data Don't Lie ? LLMs Do! 1. Augments

    Your Postmortem (matplotlib) Never ask an LLM to draw pixels! Deterministic Python plots exact metrics + Start / Detected / Mitigated / End markers. 2. ? Forces RCA Re-Evaluation Loop! A well-done graph can disprove your first theory and force a new RCA:
  18. 19'30" What People Say About Us ?? Peer SRE Validation

    (Zurich) "Overall the extension performed on par as a human SRE, coming up to the same conclusions as our SRE." Antoine Mercier-Pronchery Lead SRE, Lufthansa-Swiss (Zurich)
  19. 20' 03 What's Next:Labs & Evals 1. Project Attila: Docker

    Capsules & the Barbarian Horde? 2. Jev for Ops Decisions (70ms Gate)? 3. Cloud Context Discovery & Midnight Evals on Borg? 4. Public Links, Codelab & Thank You?
  20. 22'30" Attila: Enforcing /safe-sre-investigator in Docker ?? Workspace-Level Enforcement (palladius/attila)

    Remember /safe-sre-investigator (Slide 08)? Attila enforces it at the container level for every spapparo! Recall Slide 08: ? Read-Only GCP SA mounted in Docker ? Shared /memory across the horde ? Future Idea Migrate capsules & ATIF-v1.8 trajectory evals onto Harbor
  21. 24' Jev for Ops Decisions ? Glossary: What is Jev?

    Jev is a 70ms System-1 model by TypeSafe AI { "safe": true, "confidence_pct": 92.7 } typesafe.ai ? The Dilemma Auto-approve in 70ms with > 90% confidence? ? Approve a sudo reboot / stateless pod restart? ? What about a stateful Production DB migration?
  22. 25'30" Memory & Discovery ? Ideas 1. The Need ?

    Institutional SRE Memory Give AI your Playbooks, Runbooks & Past Postmortems ? how? 2. Meet Your Team in YOUR Favorite Wiki GitHub, Confluence, Google Drive, MS Teams, Notion, or Obsidian (LLM Wiki) 3. Multi-Org Discovery + Open Knowledge Format (OKF) ? Crawl N projects in 3+ Cloud Orgs (palladius/gcp-okf) ? Standardize on Official Google OKF Official Google OKF
  23. 27' Coming Soon: Evals ? Unit Tests for Agentic Flows

    Evals Are the Unit Tests for Agents We are currently building & integrating automated skill Evals (Harbor) The Hard Part: Breaking Real GKE Clusters at 00:00 Inside Borg, ephemeral GKE clusters are broken every midnight to test our skills!
  24. 27'15" ?? HolmesGPT: The CNCF 24/7 SRE Agent 1. What

    It Does (CNCF Sandbox ? Robusta + Microsoft) Alert-driven 24/7 Operator inside your cluster (helm install): Prometheus / PagerDuty fire ? ReAct RCA loop ? Slack / Jira 2. Stack, Models & Alternatives kubectl, Loki, Tempo, Grafana, Datadog ? Gemini, Claude, OpenAI Alternatives: ?? Hyground (hyground.ai) ? governed in-cluster & air-gapped Discovered 2 Days Ago ? Instant Fan! github.com/HolmesGPT/holmesgpt holmesgpt.dev ?
  25. 27'30" HolmesGPT + sre-extension: Better Together 1. ? Night vs.

    ? Morning (Complementary Roles) 03:00 AM (Operator): HolmesGPT triages the alert ? Slack RCA 03:04 AM (Your CLI): /investigation verifies & mitigates safely 2. ?? Safe Guardrails + Deterministic Postmortem /safe-sre-investigator (RO IAM + ????) ? /generic-mitigations /postmortem-generator + /monitoring-graphs (Python Charts) 3. ? One Shared Brain (Markdown Runbooks) Holmes Runbooks ? our SKILL.md ? write once, feed both!
  26. 27'45" Future Project: Attila + HolmesGPT (attila#8) ?? + ??

    Stateless Brain Meets Stateful Cage ?? A.TT.I.L.A. (The Cage) RO IAM ? safe_gcloud Sticky GCS /app/memory ?? HolmesGPT (The Brain) K8s / PromQL RCA Loop Stateless CLI by default ? Roadmap Spec ? palladius/attila#8 --harness holmes 1. ? /memory/in/ (RO) rules/ ? playbooks/ ? DISCOVERY.md 2. ? attila_memory.yaml safe_gcloud + context readers 3. ?? /memory/out/ (RW) report.md + summary.md + INDEX.md 4. ? run_evals.py holmes vs. geminicli vs. adk github.com/palladius/attila/issues/8 Mode A ? GCS Sync ? A come Atrocità ? TT come Tracing & Telemetry I come /investigation ? L come Logging ? A come Alerting!
  27. 28' I'm Not Selling You Anything. 100% Apache-2.0 Open Source

    Zero paywall. Zero lock-in. Fork the SKILL.md files today. github.com/gemini-cli-extensions/sre Just ? star the repo.PLEASE. ??? ? Run Anywhere Antigravity, Claude, Codex, Copilot, Gemini Spark, or OpenClaw
  28. 28'30" Key Takeaways: Pack Your SRE Backpack 1. ?? Mitigate

    First ? Safe by Default Read-Only IAM /safe-sre-investigator Badges ???? ? /generic-mitigations 2. ? Graphs Don't Lie (LLMs Do!) 98% Context Diet ? /postmortem-generator Python Charts ? /monitoring-graphs 3. ? Portable SKILL.md Runbooks 16 Skills ? 5 Harnesses (Zero Lock-In) A.TT.I.L.A. + Jev ? 24/7 with HolmesGPT
  29. 29' Snap One Photo: All Public Links, Repos & Codelab

    1. Open-Source GitHub Repositories & Tools (Apache-2.0) 1. SRE Extension (16 Skills, 5 CLIs): github.com/gemini-cli-extensions/sre ? 13 PostMortems: palladius/about-sre-extension 2. GKE Chaos Arena (3 Faults): palladius/sre-testing-suite ? Project Benjamin (IMAG ADK): palladius/adk-sre-benjamin 3. Attila & Harbor: palladius/attila ? harbor-framework/harbor ? Jev: typesafe.ai ? OKF: GoogleCloudPlatform/knowledge-catalog 2. External Articles, Hands-On Codelab & YouTube Videos 4. Google Cloud Official Blog: How Google SREs use Gemini CLI to solve real-world outages 5. Articles & Theory: ricc.rocks ? Medium ? Romin Irani Part 17 ? Generic Mitigations (O'Reilly) 6. Hands-On Codelab & Map: GKE Outage & PostMortem Codelab ? Public Mind Map 7. YouTube Videos: YT 01: 3m Install ? YT 02: 6m Outage 3. HyperFrames Video-as-Code, Hub & Google SRE Extension Anthem (MP3) ?? Grazie Milano! ? ? HyperFrames Promo (YT) ? Storagify Web Deck ? Interactive Hub ? ricc.rocks Public Links Deck Public Google Slides Hyperframe & Slides Live Slides ? Hub
  30. 29'30" One More Thing: The Google SRE Book (2nd Edition)!

    1. ? 10th Anniversary (2016 ? 2026) ? Completely Rewritten Site Reliability Engineering, 2nd Edition (O'Reilly) Beyer, Jones, Leng, Huska, Petoff & Murphy 2. ? New Dual Focus: AI for SRE & SRE for AI ? GenAI Force Multiplier: real-time triage, NL telemetry & Postmortems ? SRE for AI: GPU/TPU clusters, OpenTelemetry & dynamic burn-rate SLOs 3. ? Hot Off the Press ? Read the Official Announcement cloud.google.com/blog/.../sre-book-second-edition O'Reilly ISBN 979-8-341-60767-5 ?
  31. Backup ? Notes 1/2 Post-Milan Notes (1/2): A.TT.I.L.A. -> "Olmo"

    ? 1. Spappari -> Attila Telemetry & Tracing Report Trajectories via Pub/Sub or GCS ? Have spappari report back to Attila asynchronously via Pub/Sub or GCS ? Invisible background tracing mode: spapparo --tracing=true|false ? 2. Harbor & ATIF-v1.8 Trajectory Evals Migrate Capsules & Evals onto Harbor ? Pair invisible trajectory reporting with migrating Attila capsules & ATIF-v1.8 trajectory evals onto Harbor ? 3. Memory Consolidation => "Dreaming" Memory Build / Update / Consolidation ? Post-incident offline "dreaming" cycle that digests raw episodes in /memory/out/ and promotes distilled runbooks into read-only /memory/in/ ? Tracked Upstream => GitHub Issue Opened! ? github.com/palladius/attila/issues/9 feat(spapparo,memory,evals): Trajectory reporting, Harbor & Dreaming
  32. Backup ? Notes 2/2 Post-Milan Notes (2/2): HolmesGPT Field Tests

    ? Test #1 ? CI/CD Deployment Failures GitHub Actions (GHA) Errors in Deploys ? Point HolmesGPT ("Olmo") at real failing GitHub Actions deployment pipelines ? Verify automated log correlation across build steps, container pushes & Cloud Run / GKE rollouts ? Test #2 ? Cloud Monitoring -> Pub/Sub -> Olmo Google Cloud SLO Violations via Pub/Sub ? Wire Google Cloud SLO burn-rate alerts into HolmesGPT via Cloud Pub/Sub ("do some sort of pubs into it ? needs to work!") ? Validate end-to-end autonomous triage from GCP SLO alert to RCA report ? Verbatim Raw Notes (Riccardo + AIs @ Milan) Feedback slides Milano ? Idee HOlmesGPT -> Olmo FR Attila: Spapari to report to Attila via PubSub or GCS or other. Report back trajectories in invisible mode (spaparo --tracing=true/false)? Insieme a Migrate capsules & ATIF-v1.8 trajectory evals onto Harbor. Memory build.update/consolidation => dreaming. => GHI HolmesGPT: Riccardo test it with: GHA errors in deploys Google cloud SLO violations -> do some sort of pubs into it. Needs to work