913 lines
71 KiB
TeX
913 lines
71 KiB
TeX
% !TEX encoding = UTF-8 Unicode
|
|
% ============================================================================
|
|
% AISE502 -- AI in Software Engineering II
|
|
% Lecture 12 slides, typeset with the official FHGR beamer theme
|
|
% (beamerthemeFHGR.sty, University of Applied Sciences of the Grisons).
|
|
% Slide content is unchanged; only the presentation layer is the FHGR template.
|
|
% ============================================================================
|
|
\documentclass[aspectratio=169]{beamer}
|
|
|
|
\usetheme[showsection, titlebg=pics/theme_pics/titlepage.png]{FHGR}
|
|
|
|
% ============================================
|
|
% PACKAGES
|
|
% The theme already loads tikz, graphicx, xcolor, tabularx, colortbl,
|
|
% listings, hyperref, environ and xparse -- only the extras are needed here.
|
|
% ============================================
|
|
\usepackage[british]{babel}
|
|
\usepackage{booktabs}
|
|
\usepackage{amsmath}
|
|
\usepackage{amssymb}
|
|
\usepackage{tcolorbox}
|
|
\usetikzlibrary{shapes.geometric, arrows.meta, positioning, fit, backgrounds, calc}
|
|
|
|
% ============================================
|
|
% SEMANTIC COLOURS, MAPPED ONTO THE FHGR PALETTE
|
|
% The names used throughout the slides are kept, so no slide text changes;
|
|
% they now resolve to the FHGR brand colours defined by the theme.
|
|
% ============================================
|
|
\colorlet{bankblue}{blue} % FHGR blue (4B92A4)
|
|
\colorlet{bankgreen}{green} % FHGR green (817E65)
|
|
\colorlet{bankred}{red} % FHGR red (C60219)
|
|
\colorlet{codegray}{gray} % FHGR gray (595959)
|
|
\colorlet{backcolour}{linen} % FHGR linen (E1D3B5)
|
|
\definecolor{aiviolet}{HTML}{6B4E71} % muted plum, kept distinct for the AI lens
|
|
|
|
% Attribution labels in English (theme default is German)
|
|
\renewcommand{\source}[1]{\par\hfill {\tiny\color{FHGRDeco} Source:\,\itshape #1}}
|
|
\renewcommand{\imagesource}[1]{\par\hfill {\tiny\color{FHGRDeco} Image source:\,\itshape #1}}
|
|
|
|
% ============================================
|
|
% CUSTOM TCOLORBOXES (same semantics as the script, FHGR colours)
|
|
% ============================================
|
|
\newtcolorbox{keypoint}{
|
|
colback=bankblue!7!white,
|
|
colframe=bankblue,
|
|
title=Key Concept,
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{examplebox}[1][]{
|
|
colback=bankgreen!10!white,
|
|
colframe=bankgreen,
|
|
title={Example: #1},
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{definitionbox}[1][]{
|
|
colback=linen!40!white,
|
|
colframe=camel!85!black,
|
|
title={Definition: #1},
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{thinkbox}{
|
|
colback=lightGray!35!white,
|
|
colframe=darkGray,
|
|
title=Discussion,
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{hinweisbox}{
|
|
colback=bankred!5!white,
|
|
colframe=bankred,
|
|
title=Important Note,
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{ailinse}[1][]{
|
|
colback=aiviolet!7!white,
|
|
colframe=aiviolet,
|
|
title={AI Lens: #1},
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
\newtcolorbox{projektbox}{
|
|
colback=bankblue!4!white,
|
|
colframe=bankblue!70!black,
|
|
title=Project Link: Portfolio Intelligence Platform,
|
|
fonttitle=\bfseries\small,
|
|
boxrule=0.8pt,
|
|
arc=2pt,
|
|
top=2pt, bottom=2pt, left=4pt, right=4pt
|
|
}
|
|
|
|
% ============================================
|
|
% TITLE METADATA
|
|
% ============================================
|
|
\title[AI in Software Engineering II]{AISE502: AI in Software Engineering II}
|
|
\subtitle{Lecture 12: The AI Dimension I -- Axis A Evidence, Axis B Foundations\\[0.4ex]{\small Script: Part V, Sections 40--41, 42.1--42.5}}
|
|
\author{Dr.\ Florian Herzog}
|
|
\shortname{AISE502}
|
|
\fullname{Fachhochschule Graub\"unden, Chur -- Autumn Semester 2026}
|
|
|
|
\begin{document}
|
|
|
|
% ============================================
|
|
% TITLE SLIDE
|
|
% ============================================
|
|
\FHGRTitlePage
|
|
|
|
% ============================================
|
|
% AGENDA
|
|
% ============================================
|
|
\begin{frame}{Agenda}
|
|
\small
|
|
\begin{enumerate}\setlength\itemsep{1pt}
|
|
\item \textbf{Two axes, one method} -- Assumption A6 falls due
|
|
\item \textbf{Axis A}: two contradictory RCTs and the empirical record
|
|
\item Reconciling the divergence; the verification bottleneck (\textbf{Maxim 7}); Axis A compact
|
|
\item \textbf{Axis B}: the news-sentiment call, wired the obvious way
|
|
\item The three component types; why containment: the SE4AI classics
|
|
\item The reference architecture: \textbf{LLM gateway, queue, ontology guard}
|
|
\item The eval harness as an engineering artefact
|
|
\item This week's exercise: \textbf{AdvisorAgent $+$ sub-agents behind the gateway}
|
|
\end{enumerate}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% RECAP
|
|
% ============================================
|
|
\section{Recap}
|
|
|
|
\begin{frame}{Recap: where we are}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Parts I--IV complete; Lecture 11 closed Part IV}: fitness functions -- three families (dependency gates, budgets, chaos experiments); the four-layer cascade and the eight-row C10 reference contract; cost of change flat \emph{within} / steep \emph{across} architecture boundaries; Conway -- the fit is three-way; six limits; \textbf{Maxim 9} -- Part IV closed
|
|
\item \textbf{AI Lens threads so far} -- deck 1: the two AI axes and Assumption A6; deck 3: an LLM component stresses D3, D10, D12; an agent drafts the ADR, a human owns the decision; \textbf{ADR-011}: all LLM calls through one gateway port
|
|
\item \textbf{Deck 6}: C10 profile -- D12 $=$ H, evals as the operative meaning of testability, cost per request; outlook: agent orchestration reuses the catalogue's topologies, workflows before agents ($15\times$ token finding); \textbf{deck 11}: fitness functions are the operating licence for agents (A); eval pass rate and token budget are fitness functions with old mechanics (B)
|
|
\item \textbf{Today}: Part V redeems A6 systematically -- the two axes as one method; the Axis A evidence and its resolution; the Axis B foundations up to the eval harness
|
|
\item \textbf{Project}: M4 delivered (deterministic core fully tested and resilient); \textbf{M5 begins} -- AdvisorAgent $+$ 2--3 sub-agents behind the gateway, ontology guard active; the reference architecture today is just-in-time
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% TWO AXES, ONE METHOD
|
|
% ============================================
|
|
\section{Two Axes, One Method}
|
|
|
|
\begin{frame}{Part V opens: the promissory note falls due}
|
|
\emph{\textcolor{bankblue}{Four parts built a complete decision theory without ever making artificial intelligence its subject -- does the construction survive the technology that defines its decade?}}
|
|
|
|
\vspace{0.2cm}
|
|
\small
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item Not a rhetorical flourish but a \textbf{promissory note falling due}: Part I issued it as \textbf{Assumption A6} -- \emph{AI components extend the quality attribute space but do not change the method}. The bet in two sentences: everything AI does to software engineering can be absorbed by the apparatus you now own
|
|
\item If AI-bearing systems required a genuinely different method, the bet would be lost -- this part is where the claim must \textbf{survive contact with the evidence}
|
|
\item Roadmap -- \textbf{cases first, generalisation after}: two contradictory randomised experiments open Axis A (\S 41); one concrete LLM call, wired wrongly and then rightly, opens Axis B (\S 42); the matrix reading (\S 43) and the emergent pattern (\S 44) follow next week
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The two axes of the AI dimension (1/2): the definition}
|
|
\small The module analyses AI along \textbf{two strictly separated axes}:
|
|
|
|
\vspace{0.1cm}
|
|
\begin{definitionbox}[The two axes of the AI dimension]
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Axis A -- AI as a tool in the SDLC.} Code assistants, agentic coding tools, review bots participate in \emph{building} the software: they generate code, tests, documentation, draft design artefacts. The software that ships may contain no AI at all. Unit of analysis: the \emph{development process} and its economics
|
|
\item \textbf{Axis B -- AI as a runtime component.} LLM services, trained ML models, optimisation solvers are \emph{part of the delivered system} and execute in production. Unit of analysis: the \emph{running system} and its quality attributes
|
|
\item \textbf{The axes are independent}: a classical payroll system built with heavy agent support (A without B); a hand-crafted AI-native advisory platform (B without A). In practice, and in the course project, both apply simultaneously -- which is precisely why they must be kept conceptually apart
|
|
\end{itemize}
|
|
\end{definitionbox}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The two axes of the AI dimension (2/2): attack points, one method}
|
|
\begin{center}
|
|
\resizebox{0.62\textwidth}{!}{%
|
|
\begin{tikzpicture}[
|
|
sysbox/.style={rectangle, draw, rounded corners=4pt, minimum width=3.6cm, minimum height=1.0cm, align=center, font=\small\sffamily, line width=0.8pt},
|
|
ax/.style={sysbox, fill=aiviolet!15, draw=aiviolet},
|
|
proc/.style={sysbox, fill=bankgreen!15, draw=bankgreen},
|
|
core/.style={sysbox, fill=bankblue!20, draw=bankblue, font=\small\sffamily\bfseries},
|
|
arr/.style={-{Stealth[length=2.5mm]}, thick, gray!60!black}
|
|
]
|
|
\node[ax] (axisa) at (0,2.4) {\textbf{Axis A}\\AI as tool: agents, assistants};
|
|
\node[ax] (axisb) at (5.6,2.4) {\textbf{Axis B}\\AI as component: LLM, ML, solver};
|
|
\node[proc] (sdlc) at (0,0) {Development process\\(specify, build, verify, operate)};
|
|
\node[core] (system) at (5.6,0) {Delivered system\\(structure, quality attributes)};
|
|
\draw[arr] (axisa) -- node[right, font=\scriptsize\sffamily, align=left]{shifts SDLC\\economics} (sdlc);
|
|
\draw[arr] (axisb) -- node[right, font=\scriptsize\sffamily, align=left]{stretches quality\\attribute space} (system);
|
|
\draw[arr] (sdlc) -- node[above, font=\scriptsize\sffamily]{produces} (system);
|
|
\end{tikzpicture}%
|
|
}
|
|
\end{center}
|
|
|
|
\vspace{-0.05cm}
|
|
\footnotesize Axis A changes \emph{how} systems are built; Axis B \emph{what} the built system contains -- both absorbed by the same method: scenarios, tactics, profiles, ADRs, fitness functions.
|
|
|
|
\vspace{0.1cm}
|
|
\begin{keypoint}
|
|
\footnotesize \textbf{Two axes, one method}: Axis A changes \emph{how} systems are built, Axis B \emph{what} they contain. The axes are independent and must be kept apart -- and both are analysed with the apparatus of Parts I--IV, \emph{nothing new}.
|
|
\end{keypoint}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% AXIS A -- EVIDENCE AND RESOLUTION
|
|
% ============================================
|
|
\section{Axis A -- Evidence and Resolution}
|
|
|
|
\begin{frame}{Case 1 -- the Copilot RCT: $+55.8\,\%$ on a greenfield task}
|
|
\emph{\textcolor{bankblue}{AI makes developers 55.8\,\% faster -- or 19\,\% slower. Which study is wrong?}}
|
|
|
|
\vspace{0.1cm}
|
|
\small Both numbers come from randomised controlled trials, both methodologically sound; few topics in software engineering carry a larger gap between headline and evidence.
|
|
|
|
\vspace{0.15cm}
|
|
\small
|
|
\begin{itemize}\setlength\itemsep{2pt}
|
|
\item \textbf{Case 1 (published 2023)}: 95 professional developers randomly split into two groups; same task -- implement an HTTP server in JavaScript; one group with GitHub Copilot, one without; the clock measured time to completion
|
|
\item The treatment group finished \textbf{55.8\,\% faster}
|
|
\item \textbf{Qualification 1}: the confidence interval (\textbf{21--89\,\%}) is very wide -- the headline number is a point estimate, not a natural constant
|
|
\item \textbf{Qualification 2}: a bounded, well-defined \emph{greenfield} exercise -- no legacy context, no architectural constraints, no review process
|
|
\item \textbf{Qualification 3}: speed was measured, not quality; completion rates did not differ significantly
|
|
\item Within those bounds the result is real -- and it is the origin of the ``AI doubles productivity'' headline genre
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Case 2 -- the METR RCT: $19\,\%$ slower in your own mature codebase}
|
|
\emph{\textcolor{bankblue}{What happens when the same technology meets experts on their own terrain?}}
|
|
|
|
\vspace{0.15cm}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{16 experienced open-source maintainers}; 246 real issues in repositories they had maintained for years -- large, mature codebases (over a million lines) with high implicit quality standards; each issue randomly assigned to an AI-allowed condition (predominantly Cursor with frontier models of early 2025) or an AI-forbidden condition
|
|
\item With AI, the developers took \textbf{19\,\% longer}
|
|
\item \textbf{The perception data are the didactic core}: forecast before the study \textbf{$+24\,\%$} speed-up; measured \textbf{$-19\,\%$}; post-hoc estimate \textbf{$+20\,\%$} faster -- even experts cannot validly introspect their own AI-assisted productivity
|
|
\item METR's own explanation \emph{maps boundary conditions} rather than refuting Case 1: deep repository familiarity left little for AI-supplied context to add; codebases large and conventionally dense; substantial time spent checking, repairing, discarding AI proposals
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The full empirical record, 2023--2025}
|
|
\scriptsize The two cases are the extreme corners of a seven-strand record, 2023--2025 -- read every row \emph{setting first, finding second}:
|
|
|
|
\vspace{-0.15cm}
|
|
\renewcommand{\arraystretch}{0.9}%
|
|
\begin{center}
|
|
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{2.4cm}>{\raggedright\arraybackslash}p{3.4cm}>{\raggedright\arraybackslash}p{7.3cm}@{}}
|
|
\toprule
|
|
\textbf{Evidence} & \textbf{Setting} & \textbf{Finding} \\
|
|
\midrule
|
|
Copilot RCT & 95 developers; greenfield HTTP server (JavaScript) & \textbf{$+55.8\,\%$} task speed (95\,\% CI 21--89\,\%); completion rate not significantly different \\
|
|
Three field experiments & 4{,}867 developers; Microsoft, Accenture, Fortune-100 firm & \textbf{$+26.1\,\%$} completed tasks (s.e.\ 10.3\,\%); less experienced developers gain most \\
|
|
METR RCT & 16 expert OSS maintainers; 246 real issues, own mature repositories & \textbf{19\,\% slower} with AI -- while estimating afterwards that AI had made them 20\,\% faster \\
|
|
DORA 2024 & $\sim$3{,}000 respondents; organisational delivery level & $+25\,\%$ AI adoption associated with \textbf{$-1.5\,\%$ throughput} and \textbf{$-7.2\,\%$ delivery stability} \\
|
|
DORA 2025 & $\sim$5{,}000 respondents & throughput association now positive; \textbf{instability persists}; AI acts as an \emph{amplifier} of existing strengths and dysfunctions \\
|
|
GitClear longitudinal & 211 million changed code lines, 2020--2024 & \textbf{$4\times$} growth in code duplication; moved-code share (the refactoring signature) collapsed from $\sim$25\,\% to below 10\,\% \\
|
|
Stack Overflow survey & $>$49{,}000 developers & 84\,\% use or plan to use AI; \textbf{46\,\% actively distrust} its output; top frustration: ``almost right'' code \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\end{center}
|
|
|
|
\vspace{-0.1cm}
|
|
\scriptsize \textcolor{codegray}{Caution (GitHub's telemetry-plus-survey study): the best predictor of \emph{perceived} productivity is the suggestion acceptance rate, not the persistence of accepted code; Case 2's perception gap is the controlled-trial demonstration of this fact.}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The system level: DORA 2024 and 2025}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item DORA measures neither task times nor perceptions but \textbf{delivery performance at the level of the organisation} -- throughput and stability -- exactly the level at which architecture acts
|
|
\item \textbf{2024} ($\sim$3{,}000 respondents; 75.9\,\% use AI for at least part of their work, roughly three quarters report productivity gains) -- a 25\,\% increase in AI adoption is associated with \textbf{gains} of $+7.5\,\%$ documentation quality, $+3.4\,\%$ code quality, $+3.1\,\%$ review speed -- and \textbf{losses} of $-1.5\,\%$ delivery throughput and $-7.2\,\%$ delivery stability. Proposed mechanism is classical: more code per change, and larger batch sizes have been a documented risk driver for years
|
|
\item \textbf{2025} ($\sim$5{,}000 respondents): adoption near saturation (90\,\%, median about two hours of daily use); more than 80\,\% report productivity gains; 30\,\% still express little or no trust in AI-generated code; the throughput association has \textbf{turned positive} as tools and practices matured -- the negative association with delivery stability \textbf{persists}
|
|
\item Central metaphor: \textbf{AI is an amplifier} -- it magnifies the strengths of well-run organisations and the dysfunctions of badly run ones
|
|
\item \emph{Individual acceleration and system-level performance are different quantities, and only the second one pays salaries}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Code structure and practitioner trust in the longitudinal record}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item \textbf{GitClear}, 211 million changed lines (2020--2024): duplicated code blocks (five or more lines) at \textbf{four times} their pre-AI level in 2024; \emph{moved} lines -- the fingerprint of refactoring and modularisation -- fell from roughly \textbf{25\,\% to under 10\,\%}: 2024 was the first year in which copy-paste exceeded code movement
|
|
\item \textbf{Churn} -- code reworked or discarded within two weeks of commit -- rose from a pre-AI baseline of roughly 3--4\,\% to \textbf{5.7\,\%} in 2024, trend continuing; two obligatory caveats: GitClear is a commercial analytics vendor, and the analysis is \emph{correlational} -- AI's causal share is plausible but not isolated
|
|
\item Converges with DORA's stability data: \textbf{more code, produced faster, structurally worse maintained} -- reuse by abstraction displaced by reuse by duplication, the opposite of what Parnas-style modularisation (Part II) works to achieve
|
|
\item \textbf{Stack Overflow 2025} ($>$49{,}000 developers): \textbf{84\,\%} use or plan to use AI tools; \textbf{46\,\%} actively distrust the accuracy of the output; most-cited frustration (45\,\%): ``almost right, but not quite'' -- \emph{adoption rises while trust falls}, consistent with METR and DORA: the effort has migrated from writing to verifying
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Reconciling the divergence: five moderator variables}
|
|
\footnotesize \textbf{So which study is wrong? Neither} -- resolving the contradiction \emph{is} the lesson: different populations (task novices vs.\ domain experts in their own code), different codebases (greenfield vs.\ mature), different tasks (bounded vs.\ real issues) -- the results never actually compete; the resolution requires reading \emph{study designs}, not abstracts. Case 1 and Case 2 sit at opposite corners of a five-dimensional design space -- and every other row of the record finds its place in the same coordinates.
|
|
|
|
\vspace{0.1cm}
|
|
\footnotesize
|
|
\renewcommand{\arraystretch}{0.9}%
|
|
\begin{center}
|
|
\begin{tabular}{@{}p{2.2cm}p{5.0cm}p{5.2cm}@{}}
|
|
\toprule
|
|
\textbf{Moderator} & \textbf{Gains high} & \textbf{Gains low or negative} \\
|
|
\midrule
|
|
Experience & juniors, task novices (Copilot RCT, field experiments) & domain experts in their own code (METR) \\
|
|
Codebase & greenfield, small, standard stack & mature, large, dense implicit conventions \\
|
|
Task & well-defined, bounded & under-specified, cross-cutting \\
|
|
Measurement & task time, perceived productivity & delivery stability, maintainability, churn (DORA 2024, GitClear) \\
|
|
Organisation & small batches, test automation, loose coupling & large batches, weak guardrails, tight coupling (DORA 2025) \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\end{center}
|
|
|
|
\vspace{0.05cm}
|
|
\footnotesize The same technology yields $+55.8\,\%$ and $-19\,\%$ because the two cases differ on \textbf{all five rows}.
|
|
\end{frame}
|
|
|
|
\begin{frame}{Discussion: which setting is yours?}
|
|
\begin{thinkbox}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item The same class of technology produced $+55.8\,\%$ in one randomised experiment and $-19\,\%$ in another. Walk through the five moderators: \textbf{on which rows do the two studies differ?}
|
|
\item Consider the systems you are likely to work on two years after graduation -- greenfield exercises, or mature codebases with implicit conventions? \textbf{Which study's setting is closer to that reality?}
|
|
\item What does the METR perception gap (forecast $+24\,\%$, measured $-19\,\%$, post-hoc estimate $+20\,\%$) imply about \textbf{relying on your own felt productivity as evidence}?
|
|
\end{itemize}
|
|
\end{thinkbox}
|
|
|
|
\vspace{0.2cm}
|
|
\scriptsize \textcolor{codegray}{\textbf{Project transfer:} which moderator row describes your repository in week 12?}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The verification bottleneck}
|
|
\small The structural conclusion underneath the moderator table, in one sentence:
|
|
|
|
\vspace{0.1cm}
|
|
\begin{center}
|
|
\textbf{Code generation became cheap; specification, verification, and architecture became the binding constraints.}
|
|
\end{center}
|
|
|
|
\vspace{0.1cm}
|
|
\small
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item When the marginal cost of producing plausible code approaches zero, the scarce resource is no longer typing but \textbf{everything that surrounds it}: understanding the requirement precisely enough to specify it, reviewing and testing what was generated, and accepting responsibility for shipping it
|
|
\item \textbf{The strands converge}: DORA -- individual acceleration coexists with delivery instability where control systems are weak; two thirds of surveyed developers spend more time on almost-right code; a substantial share of METR's slow-down is time spent checking, repairing, discarding AI proposals; industry analyses describe \emph{code review as the new bottleneck} -- more and larger pull requests meeting unchanged human review capacity
|
|
\item Economically put: \textbf{AI lowers the cost of \emph{producing} code, not the cost of \emph{taking responsibility} for code}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Three consequences bind Axis A into the fit theory -- Maxim 7}
|
|
\footnotesize
|
|
\begin{enumerate}\setlength\itemsep{2pt}
|
|
\item \textbf{Architecture quality gates AI gains} -- DORA 2025's core finding: teams in loosely coupled architectures with fast feedback loops convert AI adoption into throughput; tightly coupled systems with slow processes do not -- the AI-era echo of loosely coupled architectures and teams as the strongest predictor of continuous delivery performance \textcolor{codegray}{(the coupling finding of Lecture 11)}. In the theory's vocabulary: \textbf{D7} (evolvability) and \textbf{D9} (testability and deployability) gain weight in \emph{every} requirements profile -- architecture--application fit acquires a second reading: fit to a \emph{mode of work} in which change volume rises by an order of magnitude
|
|
\item \textbf{Architecture documentation becomes a control interface} -- ADRs, repository convention files, and machine-readable rules are no longer passive records; agents execute them on every run (\S 41.6)
|
|
\item \textbf{Fitness functions become the operating licence for agents} -- an agent iterating against a dense test suite and CI-enforced architecture rules is contained; without them, every agent change is unpriced risk (\S 41.7)
|
|
\end{enumerate}
|
|
|
|
\vspace{0.1cm}
|
|
\begin{keypoint}
|
|
\footnotesize \textbf{Maxim 7.} Good architecture was always the art of making change cheap and safe; AI raises the change rate by an order of magnitude -- and therefore \textbf{raises, not lowers, the value of architecture}.
|
|
\end{keypoint}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% AXIS A -- CONTROL INTERFACE AND GUARDRAILS
|
|
% ============================================
|
|
\section{Axis A -- Control Interface and Guardrails}
|
|
|
|
\begin{frame}{Architecture documentation as a control interface for agents (1/2)}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{5pt}
|
|
\item Agentic tools are \textbf{context-driven}: they produce architecture-conformant code only if the architecture is \emph{explicit, machine-readable, and in the repository} -- Part I's documentation artefacts, written for human readers, upgrade into a \textbf{control interface for machine collaborators}. \textbf{ADRs} (preferably MADR) serve agents twice: as \emph{input context} (why is the system structured this way? which options were rejected, and why?) and as \emph{output format} -- an agent drafts, a human decides and signs, per Assumption A1 \textcolor{codegray}{\scriptsize (deck 3)}
|
|
\item \textbf{Agent instruction files} -- project-local \texttt{CLAUDE.md} and the vendor-neutral \texttt{AGENTS.md} (published 2025, adopted within months by over 60{,}000 open-source repositories) -- carry stack, conventions, build and test commands, module boundaries, no-go zones; loaded at every session start: documentation once ``too expensive to maintain for human readers'' now amortises because it is \emph{executed} on every agent run. \textbf{Machine-checkable conventions} (dependency directions, naming, layering) are a failing test rather than a prose exhortation -- the fitness-function discipline of Part IV
|
|
\item \textbf{The corollary cuts both ways}: documentation debt is now reproduced at machine speed -- a stale convention file or ADR is executed by every agent session; DORA 2025: ``AI-accessible internal knowledge'' and healthy data ecosystems rank among the seven capabilities that amplify AI benefits
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Architecture documentation as a control interface for agents (2/2)}
|
|
\footnotesize Excerpt from the course project's agent instruction file (\texttt{AGENTS.md}):
|
|
|
|
\vspace{0.05cm}
|
|
\begin{tcolorbox}[colback=gray!4!white, colframe=gray!55!black, boxrule=0.6pt, arc=2pt, top=3pt, bottom=3pt, left=6pt, right=6pt]
|
|
\scriptsize\ttfamily
|
|
\# Portfolio Intelligence Platform -- agent instructions\\
|
|
\textbf{\#\# Architecture (binding; see docs/adr/)}\\
|
|
- Modular monolith, module boundaries enforced by CI\\
|
|
\hspace*{2\fontcharwd\font`0}(see fitness\_functions/boundaries\_test.py). Do not add\\
|
|
\hspace*{2\fontcharwd\font`0}cross-module imports; use the module's public API.\\
|
|
- All LLM access goes through gateway/ -- never call a\\
|
|
\hspace*{2\fontcharwd\font`0}provider SDK from domain code (ADR-011).\\
|
|
\textbf{\#\# Verification (run before proposing changes)}\\
|
|
- make test~~~~~~~~~~\# unit + module-boundary rules\\
|
|
- make evals~~~~~~~~~\# eval harness; required for any\\
|
|
\hspace*{21\fontcharwd\font`0}\# change under prompts/ or gateway/\\
|
|
\textbf{\#\# No-go zones}\\
|
|
- ledger/ : append-only audit journal. Propose changes\\
|
|
\hspace*{2\fontcharwd\font`0}as an ADR draft instead of editing code.
|
|
\end{tcolorbox}
|
|
|
|
\vspace{0.05cm}
|
|
\scriptsize \textcolor{codegray}{Every line is a control statement that an agent executes on each run -- and that therefore must be kept as current as code.}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Guardrails as the precondition for safe agent use}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{1pt}
|
|
\item If verification is the scarce resource, then everything that \emph{automates} verification multiplies the value of AI tooling -- and everything that leaves verification informal converts AI speed into instability
|
|
\item \textbf{Test suites are the operating licence}: against a dense, fast test suite an agent can iterate -- wrong code fails immediately and is repaired or discarded at machine speed; without that net every agent-generated change ships \emph{unpriced risk} (DORA's ``strong version control and test automation'' amplifier pair)
|
|
\item \textbf{Architectural fitness functions fence the structure}: an objective integrity assessment of an architectural characteristic is the machine-readable form of an architecture decision -- dependency rules, cycle checks, module-boundary verification as CI gates were good practice before AI; with agents in the loop they are the mechanism by which an architect constrains \emph{a collaborator who never attends design meetings}
|
|
\item \textbf{The delivery pipeline becomes a defence instrument}: static analysis, SAST, dependency and secret scanning, contract tests, progressive delivery move from hygiene to necessity -- the only controls that \emph{scale with generation volume}
|
|
\item \textbf{Continuity with Part IV}: nothing here is new machinery -- the measurement contract already demanded executable invariants; Axis A merely adds a new class of change producer whose volume makes the contract \emph{non-optional}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The tool landscape, soberly}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item Record the landscape \emph{as a geologist records a riverbed} -- evidence of forces, not a map that stays accurate. Generation 2021--2023 (autocomplete-style assistants) suggested lines; generation 2024/2025 onwards \textbf{plans, edits multiple files, runs builds and tests, iterates on failures} -- agentic loops with tool access
|
|
\item \textbf{Claude Code} (Anthropic): agentic CLI tool, research preview February 2025, GA May 2025; repository-level anchor \texttt{CLAUDE.md}
|
|
\item \textbf{Cursor} (Anysphere): AI-first IDE with an agent mode; the dominant tool among the METR study's experts
|
|
\item \textbf{GitHub Copilot}: Copilot Workspace retired May 2025; its concepts live on in the asynchronous \emph{Copilot coding agent} (issues to pull requests, in CI) and the synchronous IDE agent mode
|
|
\item \textbf{Devin} (Cognition): ``first AI software engineer'' (2024); its 13.86\,\% SWE-bench result in March 2024 helped trigger the agent wave; acquired Windsurf July 2025 -- rapid market consolidation
|
|
\end{itemize}
|
|
|
|
\vspace{0.1cm}
|
|
\small \textcolor{codegray}{Examinable are the \emph{patterns} (next frames), not the product names.}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Two open standards -- and the AI Lens on MCP}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Two open standards matter more than any product, because they are architectural}
|
|
\item \textbf{Model Context Protocol (MCP)} -- Anthropic, November 2024: JSON-RPC; servers expose tools, resources, prompts; adopted by OpenAI, Google DeepMind, Microsoft in 2025; December 2025 to the Agentic AI Foundation under the Linux Foundation; over 10{,}000 public servers
|
|
\item \textbf{\texttt{AGENTS.md}} for project-level instructions
|
|
\item Vendor SDKs extract the agent loop as a library -- \emph{the bridge to Axis B}: the same building blocks that run SDLC agents also run runtime agent workflows (\S 44, next week)
|
|
\end{itemize}
|
|
|
|
\vspace{0.15cm}
|
|
\begin{ailinse}[MCP is ports-and-adapters at ecosystem scale]
|
|
\footnotesize Strip the branding and MCP is a familiar shape: a technology-neutral \emph{port} (the protocol) with swappable \emph{adapters} (servers wrapping databases, ticket systems, browsers), letting any conforming client use any conforming tool -- the role JDBC/ODBC played for databases. \textbf{The hexagonal pattern of Part II did not become obsolete in the agent era; it became an ecosystem standard.}
|
|
\end{ailinse}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Benchmarks and their limits -- an expiry date on this section}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{SWE-bench}: 2{,}294 real GitHub issues from twelve Python projects -- given repository and issue text, produce a patch that passes hidden tests. Trajectory: \textbf{1.96\,\%} (best 2023 setup) $\to$ \textbf{13.86\,\%} (Devin, March 2024) $\to$ around \textbf{77--81\,\%} for frontier models by late 2025 on the human-validated 500-task \emph{SWE-bench Verified} subset
|
|
\item \textbf{Four qualifications keep the number honest}: (1) \emph{contamination} -- the repositories are in the training data; (2) \emph{scope} -- Python only, issues with tests only; (3) \emph{criterion} -- ``tests pass'' is not ``maintainable, architecture-conformant''; (4) \emph{saturation} -- on the contamination-resistant SWE-bench Pro, frontier models initially scored around 23\,\%. Near-80\,\% benchmark scores next to METR's measured slow-down: \textbf{the module's canonical exercise in benchmark literacy}
|
|
\end{itemize}
|
|
|
|
\vspace{0.1cm}
|
|
\begin{hinweisbox}
|
|
\footnotesize This section encodes the state of \textbf{early 2026}; product names carry an expiry date measured in months (Copilot Workspace lived roughly a year). Stable -- and \textbf{examinable} -- are the \emph{patterns}: the synchronous pair-agent versus the asynchronous task-agent as interaction modes, context files and ADRs as the control interface, fitness functions as the containment mechanism. Every concrete tool claim carries its own temporal fitness function: \emph{re-verify on every tool generation}.
|
|
\end{hinweisbox}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Risks and responsibility (1/2): security, automation bias}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item \textbf{Security} -- the evidence predates the agent wave and gains relevance with volume: roughly \textbf{40\,\%} of 1{,}689 Copilot-generated programs (89 security-relevant scenarios) contained CWE top-25 vulnerabilities; a user study: participants with an AI assistant wrote \textbf{less secure code on most tasks while believing it more secure}; package hallucination (``slopsquatting''): across roughly 576{,}000 generations, \textbf{about a fifth} of recommended package references did not exist -- names an attacker can register pre-emptively. Consequence: SAST, dependency and secret scanning, licence checks in CI are not optional; \emph{security review capacity must scale with generation volume}
|
|
\item \textbf{Automation bias}: over-trust in automated systems is a decades-old human-factors finding -- Perry et al.'s participants overestimated their security, METR's experts overestimated their speed
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Risks and responsibility (2/2): skill, accountability}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Skill formation}: a randomised study of engineers learning a new library -- AI assistance reduced comprehension-test scores by roughly \textbf{17\,\%}; the usage pattern is the decisive moderator (conceptual questions preserved learning, wholesale delegation destroyed it); \textbf{entry-level developer positions are measurably declining}. For this module: the role being trained is the \emph{specifier, verifier, and architect} -- rebuild the competence ladder deliberately, including AI-free practice of fundamentals
|
|
\item \textbf{Accountability}: legally and professionally, \textbf{the person who merges code answers for it}, regardless of what generated it; AI tools are not liability-bearing entities -- treat AI output as \emph{the contribution of an unknown third party}: mandatory review, provenance labelling, an explicit policy for permitted uses (DORA 2025: a clearly communicated AI policy \emph{first} among the seven amplifier capabilities)
|
|
\item AI may \emph{draft} an ADR; a nameable person decides, signs, and defends it \textcolor{codegray}{\scriptsize (deck 3)}
|
|
\end{itemize}
|
|
|
|
\vspace{0.05cm}
|
|
\begin{center}
|
|
\small\emph{\textcolor{bankblue}{\textbf{Architecture is an accountability performance, not a text-production performance.}}}
|
|
\end{center}
|
|
|
|
\vspace{0.05cm}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{IP risk open but manageable}: \emph{Doe v.\ GitHub} -- the DMCA claim dismissed in 2024, licence-related claims continue; response: provider duplication filters and indemnification, licence scanning in CI, a documented residual risk in the governance record
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Project link: Axis A governs how you build the platform}
|
|
\begin{projektbox}
|
|
\footnotesize Axis A governs \emph{how} you build the Portfolio Intelligence Platform; the project applies every mechanism of this section:
|
|
\begin{enumerate}\setlength\itemsep{2pt}
|
|
\item[(i)] the repository carries an \texttt{AGENTS.md}\,/\,\texttt{CLAUDE.md} in the spirit of the listing -- and you are expected to \textbf{keep it as current as code}
|
|
\item[(ii)] every architecture decision is an \textbf{ADR} -- agents may draft, but a named team member signs
|
|
\item[(iii)] agent-generated changes enter the main branch \textbf{only through the CI gate}: module-boundary fitness functions, the test suite, and -- for anything touching prompts or the gateway -- the \textbf{eval harness} of \S 42.5
|
|
\item[(iv)] your project handbook contains a \textbf{one-page AI policy}: permitted tools, provenance labelling, review rules
|
|
\end{enumerate}
|
|
\vspace{0.05cm}
|
|
\textbf{The graded artefact is not the generated code -- it is the control system around it.}
|
|
\end{projektbox}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% AXIS B -- COMPONENT TYPES
|
|
% ============================================
|
|
\section{Axis B -- Component Types}
|
|
|
|
\begin{frame}{Axis B opens: the news-sentiment call, wired the obvious way}
|
|
\emph{\textcolor{bankblue}{One of the platform's features is a single LLM call -- news in, sentiment out. Why not call it like any other function?}}
|
|
|
|
\vspace{0.05cm}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item Axis B moves AI \textbf{from the workshop into the product}; as always the case precedes the taxonomy: walk one concrete call end to end, watch what breaks, and name every break with a dimension you already own
|
|
\item \textbf{The feature}: when a user opens a portfolio, the platform fetches the latest news items for its positions and asks an LLM, per item -- \emph{is this news positive, negative, or neutral for this holding, and why?} One prompt, one structured answer: the simplest runtime AI component the course project owns
|
|
\item \textbf{Wired the obvious way}: a provider-SDK call inside the request handler, synchronously in the page-rendering path
|
|
\end{itemize}
|
|
|
|
\vspace{0.1cm}
|
|
\begin{center}
|
|
\begin{tikzpicture}[
|
|
sysbox/.style={rectangle, draw, rounded corners=4pt, align=center, font=\scriptsize\sffamily, line width=0.8pt, fill=gray!15, draw=gray!60!black, minimum width=2.6cm, minimum height=0.6cm},
|
|
arr/.style={-{Stealth[length=2.5mm]}, thick, gray!60!black}
|
|
]
|
|
\node[sysbox] (h) at (0,0) {request handler};
|
|
\node[sysbox] (sdk) at (4.2,0) {provider SDK call\\(synchronous)};
|
|
\node[sysbox] (r) at (8.4,0) {page render};
|
|
\draw[arr] (h) -- (sdk);
|
|
\draw[arr] (sdk) -- (r);
|
|
\end{tikzpicture}
|
|
\end{center}
|
|
|
|
\vspace{0.05cm}
|
|
\small \textbf{Five failures arrive on schedule} -- each landing on one of the twelve dimensions.
|
|
\end{frame}
|
|
|
|
\begin{frame}{Five failures on schedule (1/2): latency, cost, non-determinism}
|
|
\scriptsize \textcolor{codegray}{The deck-3 AI Lens (D3, D10, D12; queue, port, measurement point), now concrete.}
|
|
|
|
\vspace{0.1cm}
|
|
\footnotesize
|
|
\begin{enumerate}\setlength\itemsep{4pt}
|
|
\item \textbf{Latency (D3)}: the call takes \emph{seconds} -- one to sixty-plus, depending on model and load -- where every other call in the handler takes milliseconds: the page now blocks on the slowest and least controllable component in the stack
|
|
\item \textbf{Cost (D10)}: priced per token, so the feature bills per \emph{request} -- every portfolio open costs real money; a loop over twenty positions is a twenty-fold cost regression the way an $n{+}1$ query is a latency regression. No classical component in the platform has this property
|
|
\item \textbf{Non-determinism (D12)}: run the same article twice and the answers differ; sometimes an answer is garbage -- a score for a company not in the portfolio, a negative headline read as positive
|
|
\end{enumerate}
|
|
|
|
\vspace{0.1cm}
|
|
\footnotesize Wired synchronously, the component has \textbf{none of the three things Part I said such a component needs}: no \emph{queue} to absorb its latency and outages, no \emph{port} behind which a test can substitute a deterministic fake, no \emph{measurement point} where the cost and quality of every call are observable.
|
|
\end{frame}
|
|
|
|
\begin{frame}{Five failures on schedule (2/2): drift, injection, diagnosis}
|
|
\footnotesize
|
|
\begin{enumerate}\setlength\itemsep{4pt}
|
|
\setcounter{enumi}{3}
|
|
\item \textbf{Drift (D7)}: the provider ships a new model version or deprecates the old one -- GA models carry deprecation windows of the order of \emph{six months} -- and the component's behaviour changes \textbf{without any local action}: no commit, no deployment, no reviewable diff. The feature's behaviour is now co-owned by a third party
|
|
\item \textbf{Injection (D6)}: the news article is untrusted input read by a component that cannot reliably separate instructions from data -- a crafted ``article'' can carry instructions to the model; the feature has quietly opened an attack surface that no classical threat model in the platform covers \textcolor{codegray}{(the attack surface in depth: \S 42.6, next week)}
|
|
\end{enumerate}
|
|
|
|
\vspace{0.15cm}
|
|
\begin{center}
|
|
\textbf{Nothing on this list is a bug in the model, and nothing on it is fixed by a better prompt -- every failure is a property of the \emph{wiring}: a non-deterministic, fallible, latency-heavy, per-call-priced component was integrated as if it were deterministic, reliable, fast, and free.}
|
|
\end{center}
|
|
|
|
\vspace{0.1cm}
|
|
\footnotesize The rest of the section generalises: the component taxonomy $\to$ why containment, not mere integration (SE4AI classics) $\to$ the reference architecture that re-wires the call correctly $\to$ the test instrument for a component without exact assertions (eval harness).
|
|
\end{frame}
|
|
|
|
\begin{frame}{The three component types -- one species, three profiles}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{1pt}
|
|
\item The sentiment call is \textbf{one instance of a species}: for the first time, production systems contain building blocks that are non-deterministic, fallible, latency-heavy, priced per call, and capable of changing behaviour without any local action (model updates, data drift, provider deprecation)
|
|
\item \textbf{Thesis, prepared by Assumption A6}: such components change no principle of software engineering but \emph{shift the weights in the quality attribute space} -- and thereby the pattern choice; loose coupling, asynchronous integration, explicit contracts, and observability migrate from ``nice to have'' to \textbf{mandatory}
|
|
\item Industry speaks of \textbf{compound AI systems} \textcolor{codegray}{(deck 6, C10)} for this reason: state-of-the-art results come from systems of models, retrievers, validators, and deterministic services, not a single model call
|
|
\end{itemize}
|
|
\begin{definitionbox}[AI runtime component]
|
|
\footnotesize A component of the delivered system whose output is produced by a \emph{learned or search-based model} rather than by explicitly programmed logic. Three types with systematically different profiles: \textbf{(a)} LLM components for analysis, extraction, and generation over unstructured input; \textbf{(b)} classical ML components for classification and regression; \textbf{(c)} optimisation components (LP/MIP and constraint solvers, metaheuristics). The types differ exactly on the dimensions this theory measures -- determinism, latency, cost model, dominant risk, explainability -- and therefore demand \textbf{different integration forms}.
|
|
\end{definitionbox}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The three AI component types and their profiles}
|
|
\scriptsize
|
|
\renewcommand{\arraystretch}{0.95}%
|
|
\vspace{-0.2cm}
|
|
\begin{center}
|
|
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{2.0cm}>{\raggedright\arraybackslash}p{3.5cm}>{\raggedright\arraybackslash}p{3.5cm}>{\raggedright\arraybackslash}p{3.6cm}@{}}
|
|
\toprule
|
|
\textbf{Dimension} & \textbf{(a) LLM analysis / generation} & \textbf{(b) ML classification / regression} & \textbf{(c) Optimisation (LP/MIP/CP)} \\
|
|
\midrule
|
|
Determinism & non-deterministic (even at $T=0$ only ``mostly'') & deterministic after training & reproducible at fixed seed/threads/limit; variance in practice \\
|
|
Latency & seconds (1--60+) & milliseconds possible & seconds to hours; anytime behaviour \\
|
|
Cost model & per token/call (operating expenditure) & training expensive, inference cheap & compute $+$ solver licence \\
|
|
Dominant risk & hallucination, prompt injection, provider drift/deprecation & data/concept drift, training/serving skew & modelling errors, runtime explosion \\
|
|
Explainability & low (generated justifications are themselves model output) & medium (feature importance) & \textbf{high -- provable}: optimality gap, duals, IIS \\
|
|
Integration form & gateway $+$ async $+$ cache & serving endpoint $+$ MLOps pipeline & job queue / batch worker \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\end{center}
|
|
|
|
\vspace{0.1cm}
|
|
\footnotesize Each column implies a \textbf{different integration form} -- which is why ``add AI'' is never a single architectural decision.
|
|
\end{frame}
|
|
|
|
\begin{frame}{Type (a): LLM components -- RAG, prompts, structured outputs}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{1pt}
|
|
\item LLM components turn unstructured input -- documents, e-mails, reports -- into analyses, extractions, or generated text; \textbf{three engineering building blocks} define the type
|
|
\item \textbf{Retrieval-augmented generation (RAG)}: knowledge is moved out of the model weights into a swappable, versionable, inspectable \emph{data component} -- updated by re-indexing rather than retraining, with provenance through citable sources
|
|
\item RAG is an \emph{engineering} problem, not a model problem: case-study evidence documents seven recurring failure points (missing content, failed ranking, extraction and formatting errors, incomplete answers) -- with the sobering observation that RAG robustness \emph{evolves} in operation rather than being designed in
|
|
\item \textbf{Prompts are configuration artefacts}: version-controlled, regression-tested, behaviour-determining like code -- exactly the configuration-debt territory Sculley et al.\ mapped
|
|
\item \textbf{Structured outputs}: since 2024 provider APIs can enforce, via constrained decoding, that outputs conform to a developer-supplied JSON schema -- \emph{syntactic} correctness guaranteed; \emph{semantic} correctness remains to be verified (reference architecture, eval harness)
|
|
\item \textbf{Lifecycle risk is the provider}: GA models carry deprecation windows of the order of six months, shorter windows observed -- a hard-coded model name is a \emph{ticking dependency}: an architectural statement, not an operational one
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Type (b): classical ML components -- perishable models}
|
|
\small
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item \textbf{(b)} Self-trained models (scoring, churn, fraud, forecasting) bring the full \textbf{nine-stage workflow} -- model requirements and data collection through training, evaluation, deployment, monitoring -- with dense feedback loops; characteristic problems: \emph{training/serving skew} (divergent data preparation, one of the most frequent production failure sources) and \emph{data/concept drift} (sudden, gradual, incremental, recurring)
|
|
\item \textbf{(b)} \textbf{A deployed model is a perishable good} -- monitoring and retraining are operating requirements, not options; tooling: feature stores (consistent online/offline views), model registries (model, data, code, configuration versioned together), the MLOps discipline \textcolor{codegray}{(maturity ladder: \S 43, next week)}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Type (c): optimisation components -- heavy solvers}
|
|
\small
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item \textbf{(c)} Routinely overlooked in the SE4AI literature but belongs in every advisory platform: LP/MIP solvers, constraint programming (CP-SAT: recent MiniZinc Challenges dominated, gold-medal sweep 2024), stochastic metaheuristics -- the profile \emph{inverts} the LLM's: \textbf{deterministic but heavy}. Reproducible at fixed seed, thread count, time limit (run-to-run variability in practice); runtimes seconds to hours, often anytime behaviour $\to$ \textbf{asynchronous integration}: job queue, status polling, callback -- never a synchronous call in a web request path
|
|
\item \textbf{(c)} Compensating strength: \textbf{provable explainability} -- optimality gap, dual values and shadow prices, on infeasibility an irreducible infeasible subset (IIS) as the explanation: a minimal set of contradictory constraints. In regulated domains the load-bearing argument for the project's division of labour: \emph{hard, auditable decisions belong to the solver and the deterministic services, not to the LLM}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Key concept: adding AI is a per-component matching problem}
|
|
\footnotesize The three types differ exactly where the twelve dimensions measure: \textbf{determinism (D4)} $\cdot$ \textbf{latency (D3)} $\cdot$ \textbf{cost (D10)} $\cdot$ \textbf{auditability (D6)} $\cdot$ \textbf{testability (D9)}
|
|
|
|
\vspace{0.2cm}
|
|
\begin{keypoint}
|
|
\footnotesize ``We are adding AI'' is therefore \textbf{never one decision} -- it is a \emph{per-component matching problem}, answered with the same profile logic as everything else in this module. One rule spans all three types: \textbf{contain the component behind an explicit boundary; never scatter it through the domain.}
|
|
\end{keypoint}
|
|
|
|
\vspace{0.2cm}
|
|
\scriptsize \textcolor{codegray}{\textbf{Project transfer:} the sentiment call and the AdvisorAgent's insights are type (a); the Optimization service is type (c); type (b) has no instance in the project.}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% AXIS B -- CONTAINMENT
|
|
% ============================================
|
|
\section{Axis B -- Containment}
|
|
|
|
\begin{frame}{Why containment: the SE4AI classics -- hidden debt and CACE}
|
|
\footnotesize Two foundational results explain why AI components need architectural \textbf{containment} rather than mere integration.
|
|
\begin{itemize}\setlength\itemsep{2pt}
|
|
\item \textbf{Sculley et al.\ (2015)} transferred the technical-debt metaphor to ML systems. \textbf{Observation 1}: \emph{only a small fraction of a real-world ML system is ML code} -- the famous figure shows the model as a small black box amid large blocks of configuration, data collection, feature extraction, data verification, serving infrastructure, and monitoring. The system around the model is the actual engineering task -- precisely this module's perspective
|
|
\item \textbf{Observation 2}: ML components resist modularisation:
|
|
\end{itemize}
|
|
|
|
\vspace{0.05cm}
|
|
\begin{definitionbox}[CACE -- Changing Anything Changes Everything]
|
|
\footnotesize ML models \emph{entangle} their input signals -- no feature is ever truly independent, so a change to one feature distribution, hyperparameter, or upstream data source changes the behaviour of the whole model. Architectural consequence: \textbf{boundary erosion} -- the strong abstraction boundaries on which modular design relies are systematically undermined by ML components.
|
|
\end{definitionbox}
|
|
|
|
\vspace{0.05cm}
|
|
\footnotesize Alongside the paper's system anti-patterns: glue code, pipeline jungles, dead experimental code paths, configuration debt, hidden feedback loops, undeclared consumers of model outputs.
|
|
\end{frame}
|
|
|
|
\begin{frame}{Three differences, 28 tests -- AI Lens: Parnas meets CACE}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{1pt}
|
|
\item \textbf{Amershi et al.\ (Microsoft product teams)} -- three fundamental differences from classical development: (1) \emph{data} discovery, versioning, labelling, schema management: harder than and qualitatively different from code management, no Git-equivalent of comparable maturity; (2) model customisation and reuse demand combined SE and ML competence; (3) \textbf{AI components are harder to modularise than software modules} -- entangled (CACE), non-monotonic error behaviour, poorly predictable model interactions
|
|
\item Operational counterpart -- the \textbf{ML Test Score}: a rubric of \textbf{28 concrete tests} and monitoring requirements across data, model development, infrastructure, and monitoring, from Google production experience: production readiness made measurable -- a checklist for the course project
|
|
\end{itemize}
|
|
|
|
\vspace{0.05cm}
|
|
\begin{ailinse}[Parnas meets CACE]
|
|
\footnotesize \textbf{Maxim 4} (Part II): domain-oriented partitioning around anticipated change is the strongest single predictor of evolvability. CACE identifies a component class in which change anticipation fails \emph{inside} the component -- everything co-varies with everything. The resolution is not to abandon Parnas but to apply him one level up: if the component cannot be decomposed, the decomposition happens \emph{around} it -- \textbf{the module boundary goes where the entanglement stops, at the component's contract}. That is the entire intellectual content of the gateway pattern -- and it is sixty-year-old advice.
|
|
\end{ailinse}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The reference architecture -- the sentiment call, re-wired}
|
|
\scriptsize Not new machinery but old machinery applied more strictly -- the correct re-wiring of the sentiment call: behind a \textbf{typed port} into the gateway (curing drift, containing injection); non-interactive volume onto the \textbf{queue} (curing latency, buying batch pricing); every call across \textbf{one measurement point} (cost and quality observable).
|
|
|
|
\vspace{-0.05cm}
|
|
\begin{center}
|
|
\resizebox{0.95\textwidth}{!}{%
|
|
\begin{tikzpicture}[
|
|
sysbox/.style={rectangle, draw, rounded corners=4pt, minimum width=3.0cm, minimum height=0.9cm, align=center, font=\small\sffamily, line width=0.8pt},
|
|
core/.style={sysbox, fill=bankblue!20, draw=bankblue, font=\small\sffamily\bfseries, minimum height=2.4cm, minimum width=3.2cm},
|
|
gwpart/.style={sysbox, fill=aiviolet!15, draw=aiviolet, text width=3.3cm, minimum width=3.4cm, minimum height=0.75cm, font=\scriptsize\sffamily},
|
|
comp/.style={sysbox, fill=bankgreen!15, draw=bankgreen},
|
|
extern/.style={sysbox, fill=gray!15, draw=gray!60!black, minimum width=2.6cm},
|
|
guard/.style={sysbox, fill=bankred!10, draw=bankred},
|
|
evalb/.style={sysbox, fill=teal!10, draw=teal},
|
|
arr/.style={-{Stealth[length=2.5mm]}, thick, gray!60!black}
|
|
]
|
|
% Domain core (left)
|
|
\node[core] (core) at (0.4,1.7) {Deterministic\\domain core\\[2pt]{\scriptsize\mdseries decides and books;}\\{\scriptsize\mdseries no provider SDK imports}};
|
|
% Gateway internals (2+3 grid)
|
|
\node[gwpart] (assemble) at (7.0,2.7) {prompt assembly + schema validation};
|
|
\node[gwpart] (router) at (10.9,2.7) {model router (cheap $\rightarrow$ expensive cascade)};
|
|
\node[gwpart] (cache) at (7.0,1.7) {semantic cache};
|
|
\node[gwpart] (breaker) at (10.9,1.7) {timeouts, circuit breakers, fallback chains};
|
|
\node[gwpart, text width=5.6cm, minimum width=5.8cm] (cost) at (8.95,0.7) {cost telemetry per request / feature / tenant};
|
|
\begin{scope}[on background layer]
|
|
\node[draw=aiviolet, line width=1pt, rounded corners=5pt, fill=aiviolet!5,
|
|
fit=(assemble)(router)(cache)(breaker)(cost),
|
|
inner sep=8pt, label={[font=\small\sffamily\bfseries, aiviolet]above:{LLM gateway (anti-corruption layer)}}] (gw) {};
|
|
\end{scope}
|
|
% Providers (right)
|
|
\node[extern] (prov1) at (15.2,2.6) {Provider A\\{\scriptsize models $m_1, m_2$}};
|
|
\node[extern] (prov2) at (15.2,1.55) {Provider B\\{\scriptsize fallback}};
|
|
\node[extern] (local) at (15.2,0.5) {Local model\\{\scriptsize last-resort degradation}};
|
|
% Bottom row
|
|
\node[comp, minimum width=3.0cm] (queue) at (0.0,-1.7) {Async job queue\\{\scriptsize batching, backpressure, retries}};
|
|
\node[comp, minimum width=2.4cm] (workers) at (3.9,-1.7) {Worker pool\\{\scriptsize bounded concurrency}};
|
|
\node[guard, minimum width=5.2cm] (guard) at (8.95,-1.7) {Ontology / schema guard\\{\scriptsize entity resolution, domain axioms, citation check}};
|
|
\node[evalb, minimum width=3.4cm] (eval) at (14.6,-1.7) {Eval harness\\{\scriptsize CI gate: prompts, models, providers}};
|
|
% Arrows
|
|
\draw[arr] ([yshift=0.5cm]core.east) -- node[above, font=\scriptsize\sffamily]{typed port} ([yshift=0.5cm]core.east -| gw.west);
|
|
\draw[arr] (gw.east |- prov1.west) -- (prov1.west);
|
|
\draw[arr] (gw.east |- prov2.west) -- (prov2.west);
|
|
\draw[arr] (gw.east |- local.west) -- (local.west);
|
|
\draw[arr] ([xshift=-0.4cm]core.south) -- node[right, font=\scriptsize\sffamily]{non-interactive jobs} ([xshift=0.0cm]queue.north);
|
|
\draw[arr] (queue.east) -- (workers.west);
|
|
\draw[arr] (workers.north) |- ([yshift=-0.9cm]gw.west);
|
|
\draw[arr] (gw.south) -- node[right, font=\scriptsize\sffamily]{every output} (guard.north);
|
|
\draw[arr] (guard.south) -- (8.95,-2.75) -- (-2.3,-2.75) -- (-2.3,1.0) -- (core.west |- 0,1.0);
|
|
\node[font=\scriptsize\sffamily, anchor=north] at (3.3,-2.8) {validated result or rejection};
|
|
\draw[arr, dashed] (eval.west) -- (guard.east);
|
|
\draw[arr, dashed] (eval.north) |- ([yshift=-0.2cm]gw.south east);
|
|
\end{tikzpicture}%
|
|
}
|
|
\end{center}
|
|
|
|
\vspace{-0.05cm}
|
|
\scriptsize \textcolor{codegray}{This is the topology your AdvisorAgent and sub-agents are wired into this week -- ADR-011 (deck 3) made structural.}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The elements justified (1/4): gateway, deterministic core}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Anti-corruption layer / LLM gateway}: from domain-driven design -- a translation layer that prevents a foreign system's model from corrupting one's own. Applied to LLMs: \emph{no domain code speaks to a provider API}
|
|
\item A \textbf{facade} owns the provider SDKs, prompt construction, schema validation, retry logic, model selection, and cost telemetry; the domain sees only a typed interface: \texttt{analyse\_report(document) -> RiskAssessment}
|
|
\item Provider deprecation becomes an \emph{adapter task} instead of a crisis; the facade is mockable in every test; as an industry pattern the gateway has consolidated into its own infrastructure layer -- the AI counterpart of the API gateway. In hexagonal terms the LLM is an \textbf{adapter on a port} -- the strongest single reason HX gains weight in the AI era
|
|
\item \textbf{Deterministic core, probabilistic edge}: everything deterministically computable -- validation, aggregation, key-figure computation, authorisation, persistence, booking -- stays deterministic code; the LLM handles only what determinism cannot (language understanding, extraction from unstructured text, formulation). Keep the non-deterministic core \emph{as small as possible} and push it to the edge
|
|
\end{itemize}
|
|
|
|
\vspace{0.15cm}
|
|
\begin{center}
|
|
\emph{\textcolor{bankblue}{\textbf{LLM agents propose; deterministic services decide and book.}}}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The elements justified (2/4): queue, semantic cache}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{4pt}
|
|
\item \textbf{Asynchronous integration}: seconds-scale latency, rate limits, and outage risk put AI calls behind a queue wherever the domain allows -- the caller enqueues a job, a worker pool calls the model at a controlled degree of parallelism, results return by event or callback
|
|
\item The queue buys \textbf{backpressure} instead of overload, \textbf{retries} without blocking users, \textbf{smoothing} of rate limits, and natural \textbf{batching points}: provider batch APIs process non-urgent volume at roughly 50\,\% discount within processing windows up to 24 hours \textcolor{codegray}{(figure from deck 6 -- here placed where it lives: in the gateway)}
|
|
\item Axis B's direct coupling to \textbf{EDA and PF} (Part II) -- exactly the mechanisms their D12 rows priced at $++$
|
|
\item \textbf{Semantic caching}: instead of exact-match keys, requests are compared by embedding similarity, so semantically equivalent queries hit the cache
|
|
\item The engineering point not to miss: a false-positive cache hit is a \textbf{correctness risk}, not a performance blemish -- the similarity threshold is a quality/cost regulator and \emph{belongs in the eval harness}, not in a config file nobody reviews
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The elements justified (3/4): model routing -- AI Lens}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{2pt}
|
|
\item \textbf{Model routing}: model choice per request is one of the largest cost levers in the stack -- cascades that start with the cheapest model and escalate only on insufficient answer quality report up to \textbf{98\,\% cost reduction} at comparable quality; learned routers trained on human preference data cut cost by \textbf{more than a factor of two} without quality loss, generalising to unseen model pairs \textcolor{codegray}{(figures from deck 6 -- here placed where they live: in the gateway)}
|
|
\end{itemize}
|
|
|
|
\vspace{0.2cm}
|
|
\begin{ailinse}[Model routing is a classical tactic in new clothes]
|
|
\footnotesize Part I defined \textbf{tactics} as the atomic units of architectural design. Routing traffic across a cheap and an expensive resource depending on demand is the ancient \emph{resource-arbitration} tactic -- the FrugalGPT cascade is its token-economics incarnation. Note where it lives in the figure: \textbf{in the gateway, as infrastructure, invisible to domain logic}. A tactic that leaks into the domain layer stops being a tactic and starts being coupling.
|
|
\end{ailinse}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The elements justified (4/4): stability, ontology as contract}
|
|
\footnotesize
|
|
\begin{itemize}\setlength\itemsep{3pt}
|
|
\item \textbf{Stability patterns} transfer directly from the classical catalogue: \emph{timeouts} (an LLM call without one blocks a thread for minutes); \emph{retries with exponential backoff} -- only for idempotent calls and with cost awareness, since every retry burns tokens; \emph{circuit breakers} per provider and model; \emph{fallback chains} -- alternative model $\to$ alternative provider $\to$ cached or rule-based answer $\to$ honest degradation (``analysis currently unavailable''); \emph{bulkheads} separating interactive from batch quotas
|
|
\item Only the failure semantics are new: a \textbf{semantically unusable} answer -- schema violation, suspected hallucination -- must trigger the error path exactly like an HTTP 500
|
|
\item \textbf{Ontology and schema as contract} -- the most effective systematic hallucination defence is layered: (1) \textbf{structured outputs} enforce syntax; (2) every extracted \textbf{entity} (account number, ISIN, customer name, key figure) is resolved against the deterministic data store -- unresolvable references are \emph{rejected}, not passed on; (3) \textbf{domain axioms} hold as invariants -- sums add up, weights lie in $[0,1]$, cited passages exist in the source document; (4) \textbf{grounding} via RAG makes citations mandatory
|
|
\item The schema becomes a \emph{contract} in the design-by-contract sense, and the gateway is the contract checker -- this is the \textbf{ontology guard} your project activates on all insights this week
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}{The eval harness -- definition and course thesis}
|
|
\footnotesize Non-determinism breaks the classical test idiom: \texttt{assert expected == actual} presupposes that equal inputs produce equal outputs. When that falls, correctness must be redefined \emph{statistically} -- ``correct in at least 95\,\% of the evaluation cases'' -- carried by a test artefact of the first rank.
|
|
|
|
\vspace{0.05cm}
|
|
\begin{definitionbox}[Eval harness]
|
|
\footnotesize A \textbf{versioned suite of test cases, scoring logic, and statistical thresholds} for a non-deterministic component, executed in the CI/CD pipeline like a test suite. It gates every prompt change, model update, and provider migration. Its thresholds are the \emph{response measures} of the AI-related quality attribute scenarios (Assumption A4), and its pass rate is a \emph{fitness function} in the measurement contract of Part IV.
|
|
\end{definitionbox}
|
|
|
|
\vspace{0.05cm}
|
|
\begin{keypoint}
|
|
\footnotesize \textbf{The course thesis on testing AI.} The eval harness is to AI components what the test pyramid is to deterministic code -- the artefact that converts ``it seems to work'' into a \emph{falsifiable, continuously executed claim}. Without it, every model migration is a blind flight -- and with deprecation windows of months, migrations are not hypothetical. \textbf{Statistical acceptance replaces exact assertion; the thresholds are architecture decisions and belong in the measurement contract.}
|
|
\end{keypoint}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Four complementary evaluation strategies make a complete harness}
|
|
\footnotesize
|
|
\begin{enumerate}\setlength\itemsep{3pt}
|
|
\item \textbf{Regression against labelled references}: a curated \emph{golden set} of input/expectation pairs from the domain, scored with task-appropriate metrics (exact match or F1 on extracted fields, rubric scores for generated text); every prompt change, model update, and migration runs against this suite -- the direct counterpart of the regression test
|
|
\item \textbf{LLM-as-judge}: strong LLM judges agree with human preference judgements in over 80\,\% of cases -- the level of human--human agreement -- a scalable scoring instrument; biases to control for: position, verbosity, self-enhancement, weak reasoning grading. Conclusion: the judge is a measurement instrument that must itself be calibrated against human labels -- \emph{the judge needs its own eval}
|
|
\item \textbf{Domain axioms and property-based testing}: instead of exact expected values, the harness checks \emph{properties} that must hold for all valid outputs -- schema validity, referential integrity against the ontology, metamorphic relations (a paraphrased input must yield a semantically equivalent output), domain monotonicities; axioms catch failure classes that no finite golden set covers
|
|
\item \textbf{Online evaluation}: sampled human review, user feedback signals, drift monitoring of the eval metrics in production -- the LLM counterpart of model monitoring in the ML workflow
|
|
\end{enumerate}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Example: an eval harness for the portfolio platform}
|
|
\scriptsize \textbf{Thresholds $=$ response measures}: mean F1 $\geq 0.92$ on the golden set $\cdot$ zero axiom violations $\cdot$ judge--human agreement $\kappa \geq 0.70$ (Cohen's chance-corrected measure) -- changing any of them is an architecture decision requiring an ADR; note what is \emph{absent}: no assertion demands an exact output string.
|
|
|
|
\vspace{0.02cm}
|
|
\begin{tcolorbox}[colback=gray!4!white, colframe=gray!55!black, boxrule=0.6pt, arc=2pt, top=1pt, bottom=1pt, left=6pt, right=6pt]
|
|
\scriptsize\ttfamily
|
|
GOLDEN = load\_cases("evals/portfolio\_extraction\_v3.jsonl")\\
|
|
def test\_extraction\_regression(gateway):\\
|
|
\hspace*{4\fontcharwd\font`0}scores = [f1(gateway.extract(c.report), c.expected) for c in GOLDEN]\\
|
|
\hspace*{4\fontcharwd\font`0}assert mean(scores) >= 0.92~~~\# statistical threshold\\
|
|
def test\_domain\_axioms(gateway, ontology):\\
|
|
\hspace*{4\fontcharwd\font`0}answer = gateway.advise(sample\_portfolio())\\
|
|
\hspace*{4\fontcharwd\font`0}for pos in answer.positions:\\
|
|
\hspace*{8\fontcharwd\font`0}assert ontology.resolves(pos.isin), f"unknown: \{pos.isin\}"\\
|
|
\hspace*{4\fontcharwd\font`0}total = sum(p.weight for p in answer.positions)\\
|
|
\hspace*{4\fontcharwd\font`0}assert abs(total - 1.0) < 1e-6\\
|
|
\hspace*{4\fontcharwd\font`0}for cit in answer.citations:\\
|
|
\hspace*{8\fontcharwd\font`0}assert cit.passage in source\_text(cit.doc\_id)\\
|
|
def test\_judge\_is\_calibrated(judge, human\_labels):\\
|
|
\hspace*{4\fontcharwd\font`0}agreement = cohens\_kappa(judge.score(GOLDEN), human\_labels)\\
|
|
\hspace*{4\fontcharwd\font`0}assert agreement >= 0.70~~~\# the judge's own eval
|
|
\end{tcolorbox}
|
|
|
|
\vspace{0.02cm}
|
|
\scriptsize \textcolor{codegray}{Runs in CI on every change to prompts, models, or the gateway, alongside the deterministic test suite.}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% CLOSING
|
|
% ============================================
|
|
\section{Closing}
|
|
|
|
\begin{frame}{This week's exercise: AdvisorAgent $+$ sub-agents behind the gateway}
|
|
\begin{projektbox}
|
|
\footnotesize \textbf{Coaching slot (1 lesson). Milestone M5} -- Multi-Agent Orchestration, Evaluation, and Hardening (weeks 12--13):
|
|
\begin{enumerate}\setlength\itemsep{2pt}
|
|
\item \textbf{Mandatory this week}: the AdvisorAgent orchestrates 2--3 sub-agents \emph{through contracts} -- every LLM call through the gateway port (ADR-011); no provider SDK import in domain code
|
|
\item \textbf{Ontology guard active on all insights}: entity resolution against the deterministic store (valid ticker, sector, event type), domain axioms, citation check -- unresolvable references are \emph{rejected}, not passed on
|
|
\item \textbf{LLM agents propose; deterministic services decide and book} -- keep the deterministic core free of LLM calls \emph{(the line that is graded)}
|
|
\end{enumerate}
|
|
\vspace{0.05cm}
|
|
\textbf{Axis A discipline as on the project-link slide; week 13 turns the harness into a CI gate.}
|
|
\end{projektbox}
|
|
|
|
\vspace{0.1cm}
|
|
\scriptsize \textcolor{codegray}{The reference architecture of today is the topology you are wiring: typed port $\to$ gateway $\to$ queue $\to$ guard.}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Summary}
|
|
\footnotesize
|
|
\begin{enumerate}\setlength\itemsep{2pt}
|
|
\item \textbf{A6 falls due -- two axes, one method}: Axis A changes \emph{how} systems are built, Axis B \emph{what} they contain; independent, kept apart, analysed with the apparatus of Parts I--IV
|
|
\item \textbf{Copilot $+55.8\,\%$ vs.\ METR $-19\,\%$}: neither wrong -- five moderators reconcile the record; the perception gap ($+24$ / $-19$ / $+20$) is the didactic core
|
|
\item \textbf{System level}: DORA -- throughput positive, instability persists, AI is an amplifier; GitClear -- duplication $4\times$, refactoring signature collapsed; adoption up, trust down
|
|
\item \textbf{Verification bottleneck, Maxim 7}: generation cheap, specification/verification/architecture binding; D7 and D9 gain weight in every profile
|
|
\item \textbf{Axis A compact}: documentation as control interface, fitness functions as operating licence, who merges answers; patterns examinable, products expire; MCP $=$ ports-and-adapters
|
|
\item \textbf{Axis B}: the sentiment call fails on D3, D10, D12, D7, D6 -- properties of the wiring; three types $=$ a per-component matching problem; CACE $\to$ decompose \emph{around} the component
|
|
\item \textbf{Reference architecture}: typed port $\to$ gateway (routing, cache, stability, cost telemetry) $\to$ queue $\to$ ontology guard; eval harness $=$ statistical acceptance, thresholds in the measurement contract
|
|
\end{enumerate}
|
|
\end{frame}
|
|
|
|
\begin{frame}{Next week}
|
|
\begin{columns}[T]
|
|
\begin{column}{0.55\textwidth}
|
|
\textcolor{bankblue}{\textbf{Lecture 13 -- Part V closes: security and law, the matrix shift, the eighth pattern, synthesis}}
|
|
\begin{itemize}\small
|
|
\item OWASP LLM Top 10 and prompt injection; the EU AI Act as hard constraint
|
|
\item how AI shifts the matrix: the C10 row cell by cell, D12 across the seven patterns, which cells shift, MLOps maturity
|
|
\item agent orchestration as the emergent eighth pattern: topologies, the economics of autonomy, capability-profile sketch
|
|
\item synthesis: one theory, five parts; exam orientation
|
|
\end{itemize}
|
|
\end{column}
|
|
\begin{column}{0.42\textwidth}
|
|
\textcolor{bankblue}{\textbf{Reading}}
|
|
\begin{itemize}\small
|
|
\item this week: Part V, sections 40--41, 42.1--42.5
|
|
\item ahead: Part V, sections 42.6--42.7, 43--45
|
|
\end{itemize}
|
|
|
|
\vspace{0.2cm}
|
|
\textcolor{bankblue}{\textbf{Exercise / deliverable}}
|
|
\begin{itemize}\small
|
|
\item coaching; eval harness as CI gate; cost/latency observability; hardening; distinction work
|
|
\item \textbf{milestone: eval harness in CI $+$ guard $+$ cost observability}
|
|
\end{itemize}
|
|
\end{column}
|
|
\end{columns}
|
|
\end{frame}
|
|
|
|
% ============================================
|
|
% END
|
|
% ============================================
|
|
\FHGRClosingPage[][{\color{white}\parbox{0.9\paperwidth}{\centering Thank you!\\[3ex]
|
|
{\normalsize\mdseries Dr.\ Florian Herzog\\[0.9ex]
|
|
Fachhochschule Graub\"unden, Chur\\[2.4ex]
|
|
{\small AISE502 -- AI in Software Engineering II}}}}]
|
|
|
|
\end{document}
|