Alex Wang

\[ % Made for MathJax %mathbf letters for matrices/vectors \renewcommand{\vec}[1]{\mathbf{#1}} \newcommand{\avec}{\vec{a}} \newcommand{\bvec}{\vec{b}} \newcommand{\cvec}{\vec{c}} \newcommand{\dvec}{\vec{d}} \newcommand{\evec}{\vec{e}} \newcommand{\fvec}{\vec{f}} \newcommand{\gvec}{\vec{g}} \newcommand{\hvec}{\vec{h}} \newcommand{\ivec}{\vec{i}} \newcommand{\jvec}{\vec{j}} \newcommand{\kvec}{\vec{k}} \newcommand{\lvec}{\vec{l}} \newcommand{\mvec}{\vec{m}} \newcommand{\nvec}{\vec{n}} \newcommand{\ovec}{\vec{o}} \newcommand{\pvec}{\vec{p}} \newcommand{\qvec}{\vec{q}} \newcommand{\rvec}{\vec{r}} \newcommand{\svec}{\vec{s}} \newcommand{\tvec}{\vec{t}} \newcommand{\uvec}{\vec{u}} \newcommand{\vvec}{\vec{v}} \newcommand{\wvec}{\vec{w}} \newcommand{\xvec}{\vec{x}} \newcommand{\yvec}{\vec{y}} \newcommand{\zvec}{\vec{z}} \newcommand{\abf}{\mathbf{a}} \newcommand{\bbbf}{\mathbf{b}} \newcommand{\cbf}{\mathbf{c}} \newcommand{\dbf}{\mathbf{d}} \newcommand{\ebf}{\mathbf{e}} \newcommand{\fbf}{\mathbf{f}} \newcommand{\gbf}{\mathbf{g}} \newcommand{\hbf}{\mathbf{h}} \newcommand{\ibf}{\mathbf{i}} \newcommand{\jbf}{\mathbf{j}} \newcommand{\kbf}{\mathbf{k}} \newcommand{\lbf}{\mathbf{l}} \newcommand{\mbf}{\mathbf{m}} \newcommand{\nbf}{\mathbf{n}} \newcommand{\obf}{\mathbf{o}} \newcommand{\pbf}{\mathbf{p}} \newcommand{\qbf}{\mathbf{q}} \newcommand{\rbf}{\mathbf{r}} \newcommand{\sbf}{\mathbf{s}} \newcommand{\tbf}{\mathbf{t}} \newcommand{\ubf}{\mathbf{u}} \newcommand{\vbf}{\mathbf{v}} \newcommand{\wbf}{\mathbf{w}} \newcommand{\xbf}{\mathbf{x}} \newcommand{\ybf}{\mathbf{y}} \newcommand{\zbf}{\mathbf{z}} \newcommand{\alphabf}{\boldsymbol{\alpha}} \newcommand{\Betabf}{\boldsymbol{\Beta}} \newcommand{\betabf}{\boldsymbol{\beta}} \newcommand{\Gammabf}{\boldsymbol{\Gamma}} \newcommand{\gammabf}{\boldsymbol{\gamma}} \newcommand{\Deltabf}{\boldsymbol{\Delta}} \newcommand{\deltabf}{\boldsymbol{\delta}} \newcommand{\epsilontbf}{\boldsymbol{\epsilon}} \newcommand{\varepsilontbf}{\boldsymbol{\varepsilon}} \newcommand{\zetabf}{\boldsymbol{\zeta}} \newcommand{\etabf}{\boldsymbol{\eta}} \newcommand{\Thetabf}{\boldsymbol{\Theta}} \newcommand{\thetabf}{\boldsymbol{\theta}} \newcommand{\iotabf}{\boldsymbol{\iota}} \newcommand{\kappabf}{\boldsymbol{\kappa}} \newcommand{\Lambdabf}{\boldsymbol{\Lambda}} \newcommand{\lambdabf}{\boldsymbol{\lambda}} \newcommand{\mubf}{\boldsymbol{\mu}} \newcommand{\nubf}{\boldsymbol{\nu}} \newcommand{\Xibf}{\boldsymbol{\Xi}} \newcommand{\xibf}{\boldsymbol{\xi}} \newcommand{\Pibf}{\boldsymbol{\Pi}} \newcommand{\pibf}{\boldsymbol{\pi}} \newcommand{\rhobf}{\boldsymbol{\rho}} \newcommand{\Sigmabf}{\boldsymbol{\Sigma}} \newcommand{\sigmabf}{\boldsymbol{\sigma}} \newcommand{\taubf}{\boldsymbol{\tau}} \newcommand{\Upsilonbf}{\boldsymbol{\Upsilon}} \newcommand{\upsilonbf}{\boldsymbol{\upsilon}} \newcommand{\Phibf}{\boldsymbol{\Phi}} \newcommand{\phibf}{\boldsymbol{\phi}} \newcommand{\varphibf}{\boldsymbol{\varphi}} \newcommand{\chibf}{\boldsymbol{\chi}} \newcommand{\Psibf}{\boldsymbol{\Psi}} \newcommand{\psibf}{\boldsymbol{\psi}} \newcommand{\Omegabf}{\boldsymbol{\Omega}} \newcommand{\omegabf}{\boldsymbol{\omega}} \newcommand{\Abf}{\mathbf{A}} \newcommand{\Bbf}{\mathbf{B}} \newcommand{\Cbf}{\mathbf{C}} \newcommand{\Dbf}{\mathbf{D}} \newcommand{\Ebf}{\mathbf{E}} \newcommand{\Fbf}{\mathbf{F}} \newcommand{\Gbf}{\mathbf{G}} \newcommand{\Hbf}{\mathbf{H}} \newcommand{\Ibf}{\mathbf{I}} \newcommand{\Jbf}{\mathbf{J}} \newcommand{\Kbf}{\mathbf{K}} \newcommand{\Lbf}{\mathbf{L}} \newcommand{\Mbf}{\mathbf{M}} \newcommand{\Nbf}{\mathbf{N}} \newcommand{\Obf}{\mathbf{O}} \newcommand{\Pbf}{\mathbf{P}} \newcommand{\Qbf}{\mathbf{Q}} \newcommand{\Rbf}{\mathbf{R}} \newcommand{\Sbf}{\mathbf{S}} \newcommand{\Tbf}{\mathbf{T}} \newcommand{\Ubf}{\mathbf{U}} \newcommand{\Vbf}{\mathbf{V}} \newcommand{\Wbf}{\mathbf{W}} \newcommand{\Xbf}{\mathbf{X}} \newcommand{\Ybf}{\mathbf{Y}} \newcommand{\Zbf}{\mathbf{Z}} % mathsf letters \newcommand{\asf}{\mathsf{a}} \newcommand{\bsf}{\mathsf{b}} \newcommand{\csf}{\mathsf{c}} \newcommand{\dsf}{\mathsf{d}} \newcommand{\esf}{\mathsf{e}} \newcommand{\fsf}{\mathsf{f}} \newcommand{\gsf}{\mathsf{g}} \newcommand{\hsf}{\mathsf{h}} \newcommand{\isf}{\mathsf{i}} \newcommand{\jsf}{\mathsf{j}} \newcommand{\ksf}{\mathsf{k}} \newcommand{\lsf}{\mathsf{l}} \newcommand{\msf}{\mathsf{m}} \newcommand{\nsf}{\mathsf{n}} \newcommand{\osf}{\mathsf{o}} \newcommand{\psf}{\mathsf{p}} \newcommand{\qsf}{\mathsf{q}} \newcommand{\rsf}{\mathsf{r}} \newcommand{\ssf}{\mathsf{s}} \newcommand{\tsf}{\mathsf{t}} \newcommand{\usf}{\mathsf{u}} \newcommand{\vsf}{\mathsf{v}} \newcommand{\wsf}{\mathsf{w}} \newcommand{\xsf}{\mathsf{x}} \newcommand{\ysf}{\mathsf{y}} \newcommand{\zsf}{\mathsf{z}} \newcommand{\Asf}{\mathsf{A}} \newcommand{\Bsf}{\mathsf{B}} \newcommand{\Csf}{\mathsf{C}} \newcommand{\Dsf}{\mathsf{D}} \newcommand{\Esf}{\mathsf{E}} \newcommand{\Fsf}{\mathsf{F}} \newcommand{\Gsf}{\mathsf{G}} \newcommand{\Hsf}{\mathsf{H}} \newcommand{\Isf}{\mathsf{I}} \newcommand{\Jsf}{\mathsf{J}} \newcommand{\Ksf}{\mathsf{K}} \newcommand{\Lsf}{\mathsf{L}} \newcommand{\Msf}{\mathsf{M}} \newcommand{\Nsf}{\mathsf{N}} \newcommand{\Osf}{\mathsf{O}} \newcommand{\Psf}{\mathsf{P}} \newcommand{\Qsf}{\mathsf{Q}} \newcommand{\Rsf}{\mathsf{R}} \newcommand{\Ssf}{\mathsf{S}} \newcommand{\Tsf}{\mathsf{T}} \newcommand{\Usf}{\mathsf{U}} \newcommand{\Vsf}{\mathsf{V}} \newcommand{\Wsf}{\mathsf{W}} \newcommand{\Xsf}{\mathsf{X}} \newcommand{\Ysf}{\mathsf{Y}} \newcommand{\Zsf}{\mathsf{Z}} % mathbb letters \newcommand{\Abb}{\mathbb{A}} \newcommand{\Cbb}{\mathbb{C}} \newcommand{\Dbb}{\mathbb{D}} \newcommand{\Ebb}{\mathbb{E}} \newcommand{\Fbb}{\mathbb{F}} \newcommand{\Gbb}{\mathbb{G}} \newcommand{\Hbb}{\mathbb{H}} \newcommand{\Ibb}{\mathbb{I}} \newcommand{\Jbb}{\mathbb{J}} \newcommand{\Kbb}{\mathbb{K}} \newcommand{\Lbb}{\mathbb{L}} \newcommand{\Mbb}{\mathbb{M}} \newcommand{\Nbb}{\mathbb{N}} \newcommand{\Obb}{\mathbb{O}} \newcommand{\Pbb}{\mathbb{P}} \newcommand{\Qbb}{\mathbb{Q}} \newcommand{\Rbb}{\mathbb{R}} \newcommand{\Sbb}{\mathbb{S}} \newcommand{\Tbb}{\mathbb{T}} \newcommand{\Ubb}{\mathbb{U}} \newcommand{\Vbb}{\mathbb{V}} \newcommand{\Wbb}{\mathbb{W}} \newcommand{\Xbb}{\mathbb{X}} \newcommand{\Ybb}{\mathbb{Y}} \newcommand{\Zbb}{\mathbb{Z}} % mathcal letters \newcommand{\Acal}{\mathcal{A}} \newcommand{\Bcal}{\mathcal{B}} \newcommand{\Ccal}{\mathcal{C}} \newcommand{\Dcal}{\mathcal{D}} \newcommand{\Ecal}{\mathcal{E}} \newcommand{\Fcal}{\mathcal{F}} \newcommand{\Gcal}{\mathcal{G}} \newcommand{\Hcal}{\mathcal{H}} \newcommand{\Ical}{\mathcal{I}} \newcommand{\Jcal}{\mathcal{J}} \newcommand{\Kcal}{\mathcal{K}} \newcommand{\Lcal}{\mathcal{L}} \newcommand{\Mcal}{\mathcal{M}} \newcommand{\Ncal}{\mathcal{N}} \newcommand{\Ocal}{\mathcal{O}} \newcommand{\Pcal}{\mathcal{P}} \newcommand{\Qcal}{\mathcal{Q}} \newcommand{\Rcal}{\mathcal{R}} \newcommand{\Scal}{\mathcal{S}} \newcommand{\Tcal}{\mathcal{T}} \newcommand{\Ucal}{\mathcal{U}} \newcommand{\Vcal}{\mathcal{V}} \newcommand{\Wcal}{\mathcal{W}} \newcommand{\Xcal}{\mathcal{X}} \newcommand{\Ycal}{\mathcal{Y}} \newcommand{\Zcal}{\mathcal{Z}} % shorthands \newcommand{\constant}{\textnormal{constant}} % topology \newcommand{\scomp}[1]{#1^{\mathsf c}} % set complement \newcommand{\closure}[1]{\overline{#1}} \newcommand{\conv}{*} \newcommand{\circconv}{\circledast} \DeclareMathOperator{\affinehull}{aff} \DeclareMathOperator{\convexhull}{conv} \DeclareMathOperator{\conichull}{conic} \newcommand{\pdiv}[3][]{\operatorname{D}_{#1}\left(#2\:\vert\vert\:#3 \right)} \newcommand{\kldiv}[2]{\pdiv[\text{KL}]{#1}{#2}} \newcommand{\bigo}[1]{\mathcal{O}\left( #1 \right)} \newcommand{\littleo}[1]{o\left(#1 \right)} \newcommand{\LHS}{\operatorname{LHS}} \newcommand{\RHS}{\operatorname{RHS}} \newcommand{\reals}{\mathbb{R}} \newcommand{\posreals}{\reals^+} \newcommand{\nonnegreals}{\reals^{\geq 0}} \newcommand{\naturals}{\mathbb{N}} \newcommand{\integers}{\mathbb{Z}} \newcommand{\complex}{\mathbb{C}} % operators \newcommand{\textif}{\text{if}\ } \newcommand{\textelse}{\text{else}\ } %\DeclareMathOperator* is used for defining operators that have limits typeset beneath them instead of to the right (at least when in a display) \DeclareMathOperator*{\maximizeop}{maximize} \DeclareMathOperator*{\minimizeop}{minimize} \newcommand{\minimize}[1]{\minimizeop\limits_{#1}\ } \newcommand{\maximize}[1]{\maximizeop\limits_{#1}\ } \newcommand{\subjectto}{\text{subject to}\ } \DeclareMathOperator*{\argmax}{\arg\!\max} \DeclareMathOperator*{\argmin}{\arg\!\min} \DeclareMathOperator*{\argsup}{\arg\!\sup} \DeclareMathOperator*{\arginf}{\arg\!\inf} % use \command* to get autoscaling with paired delimiter commands % the unstarred version takes an optional argument that can be \big, \Big, etc % WARNING: Mathjax has a bug where there is an extra s in DeclarePairedDelimiter % See: https://github.com/mathjax/MathJax/issues/2758 \DeclarePairedDelimiters{\ceil}{\lceil}{\rceil} % requires mathtools package \DeclarePairedDelimiters{\floor}{\lfloor}{\rfloor} % requires mathtools package \DeclarePairedDelimitersX{\inner}[2]{\langle}{\rangle}{#1, #2} % requires mathtools package \DeclarePairedDelimiters{\abs}{\lvert}{\rvert}% \DeclarePairedDelimiters{\norm}{\lVert}{\rVert}% \DeclarePairedDelimiters{\parens}{(}{)} \DeclarePairedDelimiters{\braks}{[}{]} \newcommand{\image}[1]{\operatorname{img}\parens*{#1}} \newcommand{\kernel}[1]{\operatorname{ker}\parens*{#1}} \newcommand{\diameter}[1]{\operatorname{diam}\parens*{#1}} \newcommand{\radius}[1]{\operatorname{radius}\parens*{#1}} \newcommand{\vecmat}[1]{\operatorname{vec}\parens*{#1}} \newcommand{\diag}[1]{\operatorname{diag}\parens*{#1}} \newcommand{\trace}[1]{\operatorname{tr}\parens*{#1}} % note: span is a latex primitive \newcommand{\vspan}[1]{\operatorname{span}\parens*{#1}} \newcommand{\rank}[1]{\operatorname{rank}\parens*{#1}} \newcommand{\sign}[1]{\operatorname{sign}\parens*{#1}} \newcommand{\tdef}{\triangleq} % middle bar, for use in delimited expressions like sets \newcommand{\mmid}{\ \middle| \ } %\newcommand{\set}[2][]{ % \ifx\\#1\\ % \{ #2 \} % \else % \{ #1 \mid #2 \} % \fi %} \newcommand{\onevec}{\mathbf{1}} \newcommand*{\zerovec}{\mathbf{0}} \newcommand{\neginfty}{{-\infty}} % Positive semidefinite, positive definite cones \newcommand{\sym}[1]{\Sbb^{#1}} \newcommand{\psd}[1]{\Sbb^{#1}_+} \newcommand{\pd}[1]{\Sbb^{#1}_{++}} % up/down arrows for limits \newcommand{\dto}{\downarrow} \newcommand{\uto}{\uparrow} \newcommand{\T}{\top} \newcommand{\inverse}{^{-1}} \newcommand{\invsqrt}{^{-1/2}} \newcommand{\pinv}{^{\dagger}} \newcommand{\transpose}{{}^\top} % unit ball \newcommand{\ball}{\mathbb{B}} % n-sphere \newcommand{\sphere}[1]{S^{#1 - 1}} % Manifolds \newcommand{\mnf}[1]{\manifold{#1}} %\newcommand{\manifold}[1]{\mathcal{\MakeUppercase{#1}}} \newcommand{\manifold}[1]{\mathcal{#1}} \newcommand{\Mmanifold}{\manifold{M}} \newcommand{\tangent}{\mathrm{T}} % Tangent space \newcommand{\cotangent}{\mathrm{T}^*} % Cotangent space \newcommand{\scalarfields}[1]{\mathfrak{F}(#1)} \newcommand{\vectorfields}[1]{\mathfrak{X}(#1)} % Probability % Random variables %\newcommand{\rv}[1]{\mathbf{\MakeUppercase{#1}}} \newcommand{\rv}[1]{\mathbf{#1}} \newcommand{\arv}{\rv{A}} \newcommand{\brv}{\rv{B}} \newcommand{\crv}{\rv{C}} \newcommand{\drv}{\rv{D}} \newcommand{\erv}{\rv{E}} \newcommand{\frv}{\rv{F}} \newcommand{\grv}{\rv{G}} \newcommand{\hrv}{\rv{H}} \newcommand{\irv}{\rv{I}} \newcommand{\jrv}{\rv{J}} \newcommand{\krv}{\rv{K}} \newcommand{\lrv}{\rv{L}} \newcommand{\mrv}{\rv{M}} \newcommand{\nrv}{\rv{N}} \newcommand{\orv}{\rv{O}} \newcommand{\prv}{\rv{P}} \newcommand{\qrv}{\rv{Q}} \newcommand{\rrv}{\rv{R}} \newcommand{\srv}{\rv{S}} \newcommand{\trv}{\rv{T}} \newcommand{\urv}{\rv{U}} \newcommand{\vrv}{\rv{V}} \newcommand{\wrv}{\rv{W}} \newcommand{\xrv}{\rv{X}} \newcommand{\yrv}{\rv{Y}} \newcommand{\zrv}{\rv{Z}} \newcommand{\xspace}{\Xcal} \newcommand{\yspace}{\Ycal} \newcommand{\zspace}{\Zcal} \newcommand\given[1][]{\:#1\vert\:} \newcommand{\simplex}{\Delta} \newcommand{\ev}[2][]{\mathbb{E}_{#1}\left[ #2\right]} \newcommand{\var}[2][]{\operatorname{Var}_{#1} \braks{#2}} \newcommand{\cov}[2]{\operatorname{Cov}\parens*{#1, #2}} \newcommand{\prob}[2][]{\mathbb{P}_{#1}\left(#2\right)} \newcommand{\ind}[1]{\unicode{x1D7D9} _{\left[ #1\right]}} % https://stackoverflow.com/questions/43886576/how-to-do-a-bold-1-mathbbm1-in-mathjax \DeclareMathOperator{\Binom}{Binom} \newcommand{\Normal}[1]{\operatorname{Normal}\parens*{#1}} \newcommand{\Bernoulli}[1]{\operatorname{Bernoulli}\parens*{#1}} \newcommand{\Binomial}[1]{\operatorname{Binomial}\parens*{#1}} \newcommand{\stdnormal}{\Normal{0, 1}} \newcommand{\stdmvnormal}{\Normal(0, I)} % convergence symbols \newcommand{\distto}{\overset{d}{\longrightarrow}} \newcommand{\asto}{\overset{a.s.}{\longrightarrow}} \newcommand{\probto}{\overset{\Pbb}{\longrightarrow}} \newcommand{\lpto}[1]{\overset{L^{#1}}{\longrightarrow}} \newcommand{\weakto}{\overset{\mathrm{weak}}{\longrightarrow}} \newcommand{\normto}{\overset{\mathrm{norm}}{\longrightarrow}} % convex analysis \newcommand{\gph}[1]{\operatorname{gph}\parens*{#1}} \newcommand{\epigraph}[1]{\operatorname{epi}\parens*{#1}} \newcommand{\domain}[1]{\operatorname{dom}\parens*{#1}} \newcommand{\intr}[1]{\mathrm{int}\left( #1 \right)} \newcommand{\clos}[1]{\mathrm{cl}\left( #1 \right)} \newcommand{\relint}[1]{{#1}^{\circ}} % groups \newcommand{\orth}[1]{\mathsf{O}(#1)} \newcommand{\sorth}[1]{\mathsf{SO}(#1)} \newcommand{\sunit}[1]{\mathsf{SU}(#1)} \newcommand{\unit}[1]{\mathsf{U}(#1)} % derivatives \newcommand{\dd}[1][]{\mathrm{d}{#1}} % differential \newcommand{\diff}{\textnormal{d}} \newcommand{\deriv}{\textnormal{D}} \newcommand{\grad}{\nabla} \renewcommand{\dfrac}[2]{\frac{\dd #1}{\dd #2}} \newcommand{\pfrac}[2]{\frac{\partial #1}{\partial #2}} % matrices \newcommand{\pmat}[1]{\begin{pmatrix} #1 \end{pmatrix}} \newcommand{\bmat}[1]{\begin{bmatrix} #1 \end{bmatrix}} \newcommand{\Bmat}[1]{\begin{Bmatrix} #1 \end{Bmatrix}} \]

About

I'm a researcher and writer based in San Francisco.

I did my PhD in computer science at Stanford, where I designed long-context architectures for sequence modeling and memory. In a past life, I was a math & physics nerd at Cornell. In between, I've been a quant, a research engineer/scientist, and even a teacher. Most recently, I was a research scientist at Cartesia, training the next generation of model architectures.

About

Selected Writings