From b2a9218cfb63f6f0d4ed1865d89d0ab9daec48ac Mon Sep 17 00:00:00 2001 From: Oliver Date: Sun, 15 Nov 2026 15:26:47 +1100 Subject: [PATCH] [bug] API row locking (#13274) * row locking on patch * row-level locking on action endpoints * apply row locking to barcode actions * additional row-locking for build actions * rename exception class * updated docs * Add specific error code * Fix translated text * Handle AttributeError --- CHANGELOG.md | 1 + .../assets/images/start/contention_error.png | Bin 0 -> 43548 bytes docs/docs/settings/error_codes.md | 7 + docs/docs/start/config.md | 42 ++ src/backend/InvenTree/InvenTree/exceptions.py | 18 +- .../InvenTree/InvenTree/helpers_model.py | 94 +++ src/backend/InvenTree/InvenTree/mixins.py | 200 +++++- src/backend/InvenTree/InvenTree/sentry.py | 3 + src/backend/InvenTree/build/api.py | 11 +- src/backend/InvenTree/build/tasks.py | 53 +- src/backend/InvenTree/build/test_locking.py | 656 ++++++++++++++++++ .../InvenTree/generic/states/api_helpers.py | 11 +- src/backend/InvenTree/order/api.py | 32 +- .../InvenTree/plugin/base/barcodes/api.py | 19 +- src/backend/InvenTree/stock/api.py | 10 +- src/frontend/src/components/forms/ApiForm.tsx | 9 + 16 files changed, 1125 insertions(+), 41 deletions(-) create mode 100644 docs/docs/assets/images/start/contention_error.png create mode 100644 src/backend/InvenTree/build/test_locking.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ecccadc0b7..fb7870ab2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- [#13274](https://github.com/inventree/InvenTree/pull/13274) API requests which modify an existing object (updates, and "action" endpoints such as issuing or completing an order) now lock the database row before reading it, and run within a single database transaction. This prevents a concurrent edit from silently overwriting changes made by a background task (e.g. the "completed" quantity of a build order being reset while its outputs were being completed). If the object is locked by another long-running process, the request now returns a `409 Conflict` response and can be retried. A failed action no longer leaves partial changes behind. - [#13241](https://github.com/inventree/InvenTree/pull/13241) completing a password reset now verifies the user's email address (when the account has a single address matching the account email), allowing users created by an administrator to set up MFA while it is enforced. ### Removed diff --git a/docs/docs/assets/images/start/contention_error.png b/docs/docs/assets/images/start/contention_error.png new file mode 100644 index 0000000000000000000000000000000000000000..e434de9c274a94df55a7b93f6c064e5634b529b9 GIT binary patch literal 43548 zcmc$`WmH_zvptxE1cF;|3-0co;1b+|2X}XANC>XM9fCUqZJ=={xVyW%HZqs|-kY`N z&`N&UVz(-*MBN+J5paxRB+>D1L{c@EJH|FK4F$)E7KYIT+ zsh8hk+JC+LM)8w01o%w~R`lOsNdKSw;VAmw=@HrWNWCvg?>pvK2nXeAvJI;P<&g*a z`}am?cXOiEI}p#I=c7lGh93~RLo?}l77oO_O_6Tf;L7$V7xyPBhwm7t+|J8(Z=IPn za>OU-;Y|)84Q?puks^q0j270m#q?7+y{ zMH#H|ZrN%mg%#c`9q7;g0$v)Y%7zlO zJca&ZxsNRjrw@{HdF(JJVLF4Qwy8=p9W_~sEuwj4Mp$;h(D&(cVx z2hmPYIGCu_!yPmDEEY7kyoyxhGa91<`({h6ZsNpgVm=bUFKH#$7Mhh7Q^G4bXwG;m z-=s-N$qZ@!;j+BfgLZLM*b3+CaaupvU08#YE)I_ z#20bV>4`td^0fD}m$Kx$q=iu{O53LjSL|5W9c)J*ee}8wDTe9#2OI@6e!;1I5{*8J zr_X z7C(d&(p&Ehex__wNz9{?u*O|Gq8z>R0`CQ*E|I6D5aI$?NEnT zR|dl%N13F0yw0~x=Z9A{GPZoh)m|C3Ax4gqo@f4Vm>QJ|m!3$-luTW11N!HcC5;|= z?~Vg%bgv%!8;Ha2JY?U3_mvrquc5aN?N#p z3vHw2JN5{f@iq47iKnJ8@^6;85xCBm-jkIVN`m}6J{*4MT03wUfe0L3#et*+`c^pW5Zhv6-Q1X|0d``E)03vB}glrQTx>KBrOW zP-IU3#dJ@W1lyx-Dl8|`xy_H_NDSh1k0D70{4J`I&K=C^Do5?{yny6;*e;*SrUQ3N z;)s;dje~lUsJVJ(ge7C#-I;zRa8YscGEd?jFE ztyX9>-JKwcd4G|nFT>5iUu;V?*33H(7Z}iy98_fc!O8C6X;hCWO;9J?Nuzu|$7qMV z@?0qWuV&I2zk`_AC0`PYZCrWz@wN>=I~yAd17&WqFla|>S)3+tr<`JId00D<4AlB1 zWwIoqy&UhP<6a4=S8MNq zVmF7A(UB0*kXIe2Myi@ls_B`Gdt(}<3Z`=E&S_^U}&^ZS5bo=gamx(V5T> zqcedRN?NceUm|16S`LvSUGLXjNdjo|#z&0s$GRT=8a> z-_UfmP+zfDT`@XjY)&55cQJ%JIE>EtjLWJmE9vVi@G}b1w_v}H$q4wsTZ;io^uo#N$~ZT2=th$3#+X7hVtYhc@aZXbK}NC zBtvF`X=@wWEAK7avGqL?h0w5(PqN|R8GPaSRE-@fGpz+-(r43;UDQK3&zYE1wp;xU ze^?Bd34qB2c94^eWVD)FTgRoPRe@$&Zm1&ZFK-c~^hBN^@FlTnwXn z*4sNGsBg&!g=q%8O#gRzcr0X8LKhC&=WE;6_Dj^fd|Vqn&ACm9lte-VIWl&0oOCJ0 zb{YqJSIMb+fnxc2xuzu~J&Uq^Q_AV_4H@-^9ec}abV=Jb{jr0IIb}n_VxmBVN)7Jo ziz@JmO~`nHRtOXGdw5Ry60ote3wzs%usV%jq_LS;v3qE0TZT1jxsezL8?jaULtf*Z z&%iwTx~$tt?<~k(T`GHUS!R_9;S+f$Y&UQw{n`Cukvjro;pFu?cnj=M-^i+Q5e z!PSv)u~h*K<|F3w^uAtv7ci}Za1CtA*=7pyv!o{Xv03PI`O1j9V=6jEGQs5%R($`gjI6(5PXPc z7`Ip#rKzdOPU$jnkvoRq`9w>INeL`=(w^R@{aZT_*Krt+8{=BrUeZe9yQiw%EO$aG zWS8ici0504flNK=oO$eGkyOLIz*V^Rh8g0rCA^rpw4w}#1&%sawoiPjrSljR*Y)oL z^6N+(s_xn_>FeGiC_MfBtKy3s7#zGiMNO5Po4biO(c)x7SZ-EE?)A;~TOy*e%EvS5e8WxO)5EhcDpSBf$J4WC&4`*#{d9LJr{q*8 z<0OTmk}B)^v5LBA>kQXnimzkjB=r$YeKk4dtZS1M0~!9sB$0}mA3Cu9fvHSh5kgfY z;<1Dpq;1WYgo_(vVqD0=9b4iec*pOsUS%lj>{4URP)lgzqBX;9JJI)*ZltEr3?nhA zc}J|H^Ov(Is7$H3w6TUG6-WVg&sOkXFF0m<04F3^A! zpZy`OJCVOYjT)93>0?^`x_4k9M|ikcpK!P4REbJL`}k8iR;9Y8*1t}!^xmbWY}GQi z!N_l$*=B#AX4_lSq&q>fcUb&j&qCvxg?I_PIIl};xENEnTwY`okC0Drg;cJ`H@ETXwuVO2tbJoYiK|Cm6v}rrNYI1pI0>Nn?S*-795HF^1#1I zwYA^OY2s20-N2MoFW6Jeq#{mKsuHp=Iv@dj1M!tB{grj z!{sk8zX22lCB3b`@Oj(fffN*3{3O1mqZ3XIMit#{5dtixiWMS8r?F+UGa~jE77&3|y$p3XAl+MA z@hu1ng2<=jX8dB?*l=1fwq;u8batYD9_Ignhy%-{cX++*)p0RCK2yQw<+^-US)d@S zRV{({!M`|0b0}ptp*&tqUFL6|UUPU;leHEYMX#Z1{Bp0dKt8}`PI;}QVw4xx>7m}E z*^^i$UyPUst;(ml?#GXX&HndJ;w!sDA($!;4|G1yptgs*IwC$?5BJ6AC$AvAW;P6r zKj-)SZl+XJu#9?yzITqMB7Q%r*{SB7cR$vt8_B=ew9F79)D=(+nrTqx+)kdJm3Vo< z!cVuyJDDXT+oKS$hz7Ue3{zHaw|=AlV6@}2(0X)BtjgtBj$45|R#kPavkNOHITdVs zU%rjDjge7+<(JX%JWXloLR4`QA#Z4`0@=;cM0K4!=vMI%lY!6S@kbbT@=S3662)?a zpci(3f4&?6(h78+zEpr;aH-g2peFeJvt~6~Nbb9-9l(yW$1e=&q%BBE>&5vzEKA8W znuCJ7(9SZ8T1zP@T$bz0E}LS@+@3VRL{*FRzDSL$Ki)?tWMpi= z{UED)h2gSpt;IL8qDqnBxlq?HA5-r8_sf^buVO~>0mZMQeh$a>(45<;`OHxCm+M3K zAa}KNC;x_$0fu5LR(uVny*aT8cOzz_I2r73X=w;6^bsj~yT{L8ASs?@Hz1?-Ri@EN z(p{Ox^9PdZRk1M%$yxc=SD0a6U*0DH7(|3fe5IduZbjdy={Oc4v~d%9gZwv<`7CqU z6`!STXTG+LQa0xC!4Y!d_cOY4@L+n#OKI|AtR4*k>2JJ~ujd!Nd``>XqR3-IzS9hg zSj=;-_#9F|yw0w=>hO8f)*_qO)*ZGnoL=qjsD|cmvqWcZQS{u!Wh#aI-aeh>GmS-` zC{`<)HbdN=0R9#aI93xS`W#zC~T6(t~UZ zd(~aS*~-zW{q6rNbA2V(R>-2mJh$RI@J_6GI?#5H4 z$rpbND1Dl_fI!(Sq<8Wwm#KS3Bdg42_m~NgEYY!wIYh?+XHR7~yNP{yJ-vDykF}S( z$7I9)7qr>`I)l$rf^wrtd`g;5dkm@I5;b za9Aks7EHrF79@S*uyo%^Ur_BdA-+j#mf#GbJ-O>2C76^&cTWA*duh zRD;E;7Gnh?3C&*J(ak-P-NF_bQ`&XE+ok*a{ffT}A3okZ>w<@yrDJWN*Nk5)-y8{m z&J2H+>k6lE$)!Dm0}xBVRzZTrb1jeKwN~BHZpSOVGANmR0r#!CWm*|SDQj(^y_g$g zT9-cao?fy9a`MuZh3ZF%cNclANpo59>CC>5oF{s_ySv|pbS1*u$jYEMt=ZSp3=6oQ z%TlEUCVNNHJDHL+%0hE0eR@$WxV%5~tj``NMG50D2az);7ppPh;&mS@q ze)NA`^EQ_2WNyN=?NyE3;z4OCgF_uMznh_asrO<*W;R3)l+KSr*e1HFTA>HnghLz2 zTUbNmu}tCh*LL$II9#DuX|0}Wvh*v7)pd1TuGP{b8C;xJC}|wh*3!`SpO8qxA|V2* zHR%{P2SNjZi?auu8$L=-HNq$}TCJywflG}B+-H}HzWdv5r&AUJDf>|ajYkV%Py53s@Ysa-p>i9Ke>6{)>A7w0eTE57 zP>8yu-EVGJp(ke#y~FEkpV+(3_U)G5wVsBA`jJW}HP{;*^LN)+tb9pkJGxmp%P(T> zT*&l|1QQI}U#zy*$)(EQu7_18GVg%7yo2uy&yAVgZqk1LzLALPNitB$Kjd{Z9?JB+ zPZyZWwbgAZd;9jCK~s6T?H8TJ2Q2#O;Z(5im1Sx5#p#;6Aj!i?E2C_}p|Fqlawyh2 z4t_?)vZddjgHiYs85h8o@p%m%C*T!%H9AgH(fU0uOVtU_tI_Y^`I>e}+5JK@gP>q@ zs*uou>*1}{y|c5%-oXKNUamL4*L7T_5p`j?Pg_4UOyB3ud%f$o(~e@9$4ROk+Pf~n zjw5Q}Cmz?)kB5ihBSKJvIiysc0FGqo5;K3eQ`RL)3VVUc1GXYQ;3PmiGQ(kxrZvBX*-b{;>;)Da7;)% zD$l6B-flBdLpyx$N2mN=JijDXJMz(`-472~P%dwNlfTS}*bFQ6N{Y?;s2KtcP9^GY zQG|GtIdJ1qHF3|KL88lEBg@mS?NP!{0xp*3VsD2HI;sVH$0sJv`m%*Qp`%$O!jB)| zs3`;lS|y&UZI`lC3Pzz1?X#5&Hz&{L_s-#ja}^{RrpgR$v2c~&q$>3A2iBX;hwNHw zEhl$dTc0}C30#lv&qjp1rQPq-8{N@IDwhTl^%uYjvRQ5IAE=4=JgT6(V<3n#ypS*P zi2dM%3f5oLQq75`&p%mV;0)oc)Jg(b?>#+(@u%q2I_|Z(Ed4h=a>1L)r0q?NPcWa% zIk_KcA*~98MsbK_6cqm2S<3)>dqklS*%I;9Sd5X?C@$)=jLzsc+7eASd5n=^8f`q_ z({4U@d{#W>aW=2Mzaq&HgcxssA!JP}4tUUy68BEWMjByldY3iFgTgcH?sM${wK==1 zDF0r7K()quy;ko~@^stT-Cs0Q@38*4=SK>w9dU-mQ-Qo3p^*30bXhevBi-$4$%H${ z6iH&;Pzu)Blx`a)5|M}0vzK~#mv-+p4BY89Lo=6fCM(Pdqr0ZfE%fC0XhGrOT-JN_ z!?^(Fxz`#d)|l1ulKO|ERPJZv2X}|tg4QQK(4O*)>(QhckLxvqUgJ>;ckihgCbgjt z0euNsIUz~uSwHT{WW@etyKWka8=(Nt&5Ey???uFKPwM;ABQSk^g8?qVqtV{DsMU;? z=2sqmo*LWn&GD;)ccpjBKjChZIDK{%&%RWj?qoW64)pc4X{vy@yu~`TA4rOC9IVNN zSXtGtUN?So1A)$l%`FT_m%Gi5X^ncEm2|rUxF0ijMlv}JyA-EVmrh$i4j3Y0zwTc{ zYAhxoPG(4#C)3Dd`-k!LOY`6nJ5e5mjR|5QY%d^yLl&-2dU-1x0}woRx_0I#wCg^q zy)O>s6WYk&@oh0E(|X4(;V1gIcYB6xHPdstsg$E=QSW%5-kZR9MirU!c#x={o$&)E zSdK#EeI z-n7M@LU^4{Wzf9@4!pd`Ka-h#G?-MOrwnCzrdA-vSJ2Q%%N_eUQKUvc32QIJU+q$o z(=15Ec)LeEDb?~L!}Y(l0Q>yIxyqbhJG8~GX`MdZ9OHi4?`R;wdVk0)zlNJrPy(Gn z6%MY(8tr?~<8g19Dph``fEyt4qs7a+N+ja6)qF>U6)*Bu?)+h-fR(SkKfj>O@p-jheKX?9TNTcD*k=Y*?J zO*Tb!S8o-$nNQ~ZLS2a48Iv$Y=D>J!i7Gl~Obz@8BLti9L#D6A!@GtzKYYJ^od4*e zy4FM^fWx7H-?QO&>-Tzbx=cwl{Bwhi)##@bIBhYkf}yq_yW5{mK5?skfgicR1ApH^ z)ryRtEZ5UbQ>`@=hvSEMYf%^KhX!$cXNUVM&$F@KL<5@c0)qdOlfD<8%f(bK3_e?qW z62v}p3FXFiJ_vh$%d1AGiiR2U+w%yq*evlL>0Rs8V!ap69o8er{q2zIh+5_E@p4V} zx9HBXS)D{#U7b_4`IeULaILTQqh!(bm+l1HyR0PgBwtrzkPzkNoqqS>bjsJb*q$#h z|B4WQ+qY{zN7df~AcA00TJStGvg`?H{}k`CU_?kXEi;DHGPak z)FSNo?SI!u8scCHF(Gt|E`j?EOh@n_^Gzyg#;2 z6u&?GbL#cm6Vs7|l+A2}7I(3iP1|xoSnqaSZw+d1_b_wchtKy^w(w4)UoG-O3bGZ8 zvmcGm#6$aae8S8akl}C%lbsZVgEBwu(1&)W-1p+Y9K?y)O)vf%lE<%7tMO|n){F;# z(dh82cj4EI8GUIXp|3KJM&AR?K$2w2y~lMOc(}N$I)1*l9D%5!yF=|kbZ4H?B6C{W z8!8)hd9qWI`vL=$%bW1)O-y&2{zG;lss6nDf3DX^YpovHm9`|Lk_JM)!tB znyS7`Hclq`5Ec4Y-@bZtOCpYeDI+w5OR_{?tI-z~ZQvETPT2JtukFMiGp_w^Rp+P3VrmW{!i zvVCR75V@z%in=~f?JkM!n~rvOf&5)Hv3@yF)2va(%0cDm4DACi$a6im0O2o}2yxYj z?-d?`dodxO%ir0)_q1Y?E{|#MSut->`JPDev*UOkqlA~A9-J-*v3FKW=>L}Rw^Ao1 zfib4^J}W6?tZ{?9e;sN(J$XG+H-lfdw{H;=y~~d2`K9{4iF@q>Ta~Y`H(@*8yJy9j z_(#hShQZ8VNx|p!s`WkmaYGjg^pjaT#3v1%(Ccmj{fj{4vAI^;UX(7L2yT66m@HVT zrAN>0!NF37OQO|_$5)60|dERxt5}4kpgZ=PTICgmjU9U@>-EP)8tExjD$wL&@KWJq;h_U3|J&@g;ft4jG{70wKk+3JSfNGQ~=yL0PN*?B$9{DGI`8 zT|ae1%Wz5+cZ=p~yxayPB680$Lh%d-*6T5(V!x%iy&b_gO-08WBvh!cJNWHx=gmY7Y4IT-DI884l#dxMN%yEv<3$Qdfc|uXbwKrJZ%fXyghBa*?r-R4+6d| zxAH;UpVOp@+kb|cq9mbJ>^KMXs(-=;*cV}QM4pXoO z@(t*KF?>(h;0>Q5!2WB$mMI_?a0oE*;0yWq7zy8r2}r^g?MsV$KRMehU~Ae>N|LO7 z786N1?Z$bh6YKv;!1&ylkdGnEiDFn=dByg*T=x+M#z`3d@*vUe^Ch;2Mp>ib1QmnV z!(v0rNz+ea%o?%Bi)ID^CTJANv(cWi_h6fY*Udr2*=i^x#BBAhM8lzD;DiigERNMk zIZo!XTi`uXSJv(MjQ#==A&stMbD_by+T!2^<+35hC~n}SO#BInbaPC`Rt_C{f={QZlYR~y{Uj9QxM{tVfU&?lv^e& z5E&AIPx-VykkB0l=Xw(9)=yWaeZu2zKRu)GMtj=Z681h*<~f}c@u+CD$zx)<`7FWh zwDnow!h6zcp>*z!h<`MP15k;cbX$GaMuemAcoY>uV;e`LqMlI%W}92JF3&zqS4O#) zn6&Sp0Ix`VxIcK2=DrVVHq~oGT!lf`npv_SAqO(QHyA%J$!RdWzln321KiMq9xuD} z@Q~_i|7v&iHC~|v^t$l3wN{PPLpWe+(H~C?pYG_@C#_eAcLdI~gFF?OY7;(g+UPam z!p)Uhs--c)x||$u9hdvmk0$=;QmHf<90`3B=($CEqlaf{SOruh z^FVxcL*f(cB++`d5e+{5e0OuwB3hX>&=-B+c1W{+A$h3%IU-klrR6qA=(i;n{q|!_ z-EYsY1>r_lV!J$Zq|7|Gv&MHKndhHPAv6| zD30>cd{yGG)Sr{Yo5KcAC>K`>6x7M2t3H%YFE2t(P7DDDqT77IC&3x>jy#l729D=w-FUVWZPae(E*jv$e!@SjFs$H~Q?LSndfwUg7N1m}LHXJGJuabHr)$Sk73h{oVGEJ~l&J z!Ph-%Z9TifsP$FmuMHTF&Hx+436@bk(_b=g{rz!u)u|eaXJAxj$bxmihbyy~~xxU!kXK3lx%9pBz z|D7`G|B~vJ4u9n;zJGb@f0E9kG3Dhi)!+Y}XiiS~zcbRo|MIz1dCF2!Xl!i$>lL9u zf}!x+u8Ame|KJxE7Bm?ZY5$kBboLBN>(Wc+`hTS!{QsHv-v|iE^}0EBIbNLW>nqf6 z^?AuDkIClOlhQIV$)9mBGgqLKODr~ffmh!FB1Nm)$+8gW{_24{UR#Z*a>l@W| zwraR|@I5_q1@iDAA%H)^0|Q-B(Iy{r85wQB&`CxTe!h2bbGx~?;P-tJpb7{ILIw2S zxw*OJWn!%S+1an*))VYldiW>YTyKged0ueG{aZU3oRZOUmrx(YdI zfdYr>l$j_=W=RQs%;@e|PIYxP>vdE$)>-*|7Zb>6NGMJyWAAAj(k5BNyHym|1)K1vPgw> zZ#bQwUZWfd8CjfWuGx$Gej^Z-Byz6H1czBqr${*u*n@S~xhfMHT2ea$f$h89oG6y> z^|5hr9A4LlV|3IC8C}*%6vKg63|cks=lXhjV5Nb#l|(^7x!4}+%-!gX%E1oce1@L1 zCNk>-R}*aFfml)_nOtp3o0eYddD(aW=vHPr^kJ&_zm4RVfA(1WPq;h?Cbj&-Sucsw z>e90BgBa4uX*JzdmXM%+ zG+zsVyYPJ_Ev*#gqOQ6wPE>VtRb8)uIvKP5$+TRG00=}xBM}~`NZCbJWh3ohTohnp zLV+pVYPuNa2j*V$=g+VF>*FP_^UeNo-4CJ08dV+_rn0iK4zrV!s`$*V%hIC*(^V$2 zd*!obRE2M1d%jC)Yis{fkt(sa!K$mfZOv99efp3NfuE|!sz;YVj-Kzv!%KTJMERcI;u{nE(TI8Gn-jfRE>3=E8O zIp8FS#b|*(&6KhUA1c-Uc@+KHFaJI^r|mDFUAmoRWMsmy8TTiPoOVaE0EOLp>u)@8 zd}vjhMo8qrQGH4Jz5r;Sz7zR!fRTzsNbZZNaC=^w=Vk+D$Ee$QnQGV4{&!4L{gaWi zi%T~r;OO@U6|Fue%{P(6f?5@N&17{iac3x6ov6a*78W)uEzQ2q zPXh^zsoZwUfQwP-NteAYdfyRhu_Ot|d2c)~o!<=w zm*w|r&%?vT2#_&f<2Oq~ow{$hEa74HtvX7ICDzDx^jvboyQ*ho2H9Wv{K)b)*CQr-~4n93EGKEb?dP-2RF z;I8pfX0%(Y9%BJxk`#(1fI=h~u=ZOr3vx|DENmHk|3_6i2D`z2>%JjOgK=<3(Qo{# zcCoU@^?E$%qbJ||!g_gg`cT{Q5;l|ePjR$5#V||78R%dD{Yfi38v~i+NxRYg{k=Sj zE5c_|CSqJdzxqH(p`FXg;&k8W>0x(Cw9?pGw9?i9ubcVO7gwB8hCzDw#-$doXNLy3H0j^QUuiU`nbG>Bco4(8yFZsV?$(8a#Aq7G*F7JmS-3sNos&AVJjss?cw_% zr5ay%#5>#Qv4Q^6Gc`Fm*Ub2vrK98DKpO}@*oxEBTv*$x1QaCyEjK{~9SwJ}$Y$UH z1497G(|KcBw-R#N;@fsNFcXNpE9hzd{O7cfe#VZporiy(yIda*p7uobN_}r{&vvn{ zcVfxq@|O=|D#vU+2o(h-zo)B5?fhI!Mby!fJ@Q2V6e|m#GF9(B& z>kwe+sy#CKF&w^obPli5)$guqx2`_2^mWgTb#>u(a+atGp*bjnEWJgtD=VXW$Y|6B z5s)#1tW?W#3rYyEh*G07NE3|rJvD^AACO?ZNd&|!5CSNXxNQ}o72WjR;?_2$1H8QN z@4)(!k_863C#Dhm2Dgz>nQm8f>y(Os2CU!gpxvCv*ji**GJ0E>lmzQyX^4~^)6%k> zGitxdOJrlC06rJ7H5lZh<$VX^c5=*i)=Ox-X?31e6|f$|8ms ze~Aa47dU(_e`PborxZ_4%wPtM5#_T^wS5w%q*OC9R(7n4&icU<{zSW&pPRqGC#V+^ zi2xg;qO63O!fN)T=9{6i<1v}ScASl{@_?VOI+sF|3W7U^RU6W~O>i7(G6cgIo7J4_>V5QK$|6xU=2uuyznOcPcA;AF1JL?HNh z`NkK`+|n}p@2iG!wVAC=6La%l#OSmr{JL~~R;CZh zzqgjs*u#sO$&5~xRRoQc`$4Q)Gwk3;I=W^H%iSU#gA|YmMrC!JQnw=rikUF}maG=_ zNCdt&CXvU_5n>Zyld%<)u&}ULw--~b$@=lR-HPg^u1Be(HTDBKUfCB}r$1ovM;Nut zI8{msA$ME`{_5%+QAPX0*#qZ|gs2`>70g%}ti>6-xqz1H%kc2i;KFCFqrSerQqAV7 zd+TX+?D{LOQ#7zf`4UEj!}Ym6B7D@R?KT#}gjqO^4{`+45#f;yCnVq3isPGY7tf?+ z2jsqK*ey3)ZCctYwX0M@6B4#(?3S<`*$sQ9m*Ch`_ZXQ!l-sUciPwqF9kxdBudibw z=>1#D2w*9twY3+aTx z@OBT#(`#&}rD*U$pyr;d143RO*Z`V`u!so#N>}!MWdJ~e8eNn#t5=Ig!CpIn0cJm= zr@QgIY2-T(!vD;|Mt2VwzH+>UOH&DiXUDY}tI71mK4C%$i!{Ptj+l(qOiG}0<$uz~ zN%KR0F*0&w*_Y~|Yisbg{Sy=Ai;*$mc!^)|@e$!$X{RRImKD^u))Qf7vTtP8XvNoz zE}o;5$^y43?I9oyNLXy|Hy;+|T$T-Di2z7f>fX){R-L83`ae8G{qM&gAHdf?X=X}Y z`yb+UUibx>rIzf@X8e~d4lM4Kw*fnGZZzF8I4=Y01(W1~SUDf&5afHkAtq zt*djLDprF-CHCGMr?jxt(OL9|$1K*Uu(h<*D%0ZpvIMrvQ8dn1pjWk=^*bF2x10?Un%)}&Q1GqAuD~-v{&TefEod4sVRn^t4ciRLUtD{+> z|B7Cp{?^2BSWPo@r2`$gKyBs*00ldPP;q*J&rFGi-3zP?MAmNg5jlVf7W0crrCUszaJ!?0Lt z1tHH%njUION&{ne_xc3v1GATi8@&GXW5M>M;PAGykj5!HhQo0a;Oq~IO%RiTgccCg z)YLS)j>BQSD}sP~x!$33d08(lRGn1hXbExyAYV^UPb@msTL3yqhU5N_2Ds7i@G#hJ z1-lpS?c4gsM#BdH>_?aT`S~R<>px8>2rsqz>g!D9O2?vgsF!JB5fX;WeH1om7xcQW zu~|@msr1uO9~14n@L=VPpPZbuS*XkU=Sa2a&71dzyYuevjaQn}ssC-(-E*$r8x6C&kpFS_u6&7PtL z0-ixrkCa2x;ov_;T1_)?`F*)kolZ)Y-|aX&Bt-Miw;v%P@O<+C+A3ys^7a<66elJo zj*N_~CmS-*)G2*i4I^(1RUmQ7#MyA4iEjY)K=4_A;X;Ey?<&l zCJGd|Eyl(E6~<#TQ&Eu44W{v4++Q6mHM(>z*|prkCigEa0JP>gGCW+se)W~-I!pK_ z@YEo@9>BtkdXNT5(9wU;sg7g_+Su4A`RvY!^RBK;CM=ss8MocwEWZ{GYr z?|1J;mS(yRZTn;RNca*A$ErmhKD|emQE?De24W(0h4FtRmEFmyLDO0n|8dpDUz3%U z!&sSz&)24ZGJvHJGkFya)4p-ObSN(;!%cy8V+a2H#d~;pBDs+IT>v}vzt}0lOaK%E zOQ~M0Fs#Y| z#{*|&WdX7?9GNOWIX!QVBUphzg$0u!X11?dmM`~CSGT?G(@if47N^yJ_FP1gM#;y= zr}gog0M>GJ61v>qGpeaZ6Ln+ z+^z-fTMaZg(?Vxi0kvaDr<&@bqSZNH;Xe%*V*UQPx3}(>zop#(=W7LOh>#|Mv*Z49 zkKJ@oN=^=oTAH-KqeJA2)H}t=qM!b`=Cg{u^I8T*%A_>*0Q{?y z2n-xo4w3F(xFP%^tI%t|Q^!a|RRStG@rUKYf+nUlRyd$Q{Zvi(+W}_0svMZ>LDd1#b+!{KjMJv<7tb-KTr8oJ-zZ7 zol+DOIA*;Lop6&^<&j?#_)`}_L;(xZi5d$AD8W7@DEU<8wv{iKY>zB<;{{! zGsHz~l?U`ReqNi3^H8nTG~IjCE-LB8Ppzhw@9=Z}<`E|M4qZRn&=7yJD8xXF5&-{_ zS!+ov?aRZRWdeWYMWHv=I<2lgsZNeH6BAS2pK9r_lteq)zU%3xa1{Q04L%?4<5;)m zo#VZ-ocUBpSAS-XTNPIu|1QvKrZm($p;)zeDSJ$Y3s4!NNU%zDISKK&JZldCF`;cb^7nM;{#i1ZC4Z^)N?@4<%ZoO-W2^SNi zGAeyDK!TVjUFmR^wDL*6f+m%ng9QyvaaAcl&!CO@+&_vCxq z>S-TMoE}|00<>A1>gJcZ83mNnv2sd?D9jc`eoKMfB#>+y>(QTUDr2iAX3sL-5`RLAdPfNHv&@9DJ2Lf zA)SJRf}}JEQj(I=-Q6IKgh-yby`Sg(zV{j5c*i;8jC1@;iS4!bb*;JP{MF*HXyE-A zo!B)zev;dhB{KU|QSZ2^p`czzH zrt{SPNVdH@eb9{2?%;j-=aS*?8z>9WlkL0+(W%iSDxMwp+|!nl`kdjX-}ivr#@4A} zc_d#x5U!3|gX(5EjGJD$^)mwYb1(KfkMwdn)eGi}@-zH=^aaXw8eE8ep~avvt9n#d z1OC;Mc-5CGJTu(jxVoTuIbWid^USD6U!a2H5m$NTSN&RJR*!Nk&HS*kGRLWu_dO(X z2N_mzZbB;`$&M*PJ~cRxv~Y-fI$JiFja?RWvpAO>?T$#+CCl(>60W8hMQ8UKs9N0P zFSC0aZ9p&$Rpg8NAKl?1!5RI5nA*Us=jF^=_R~^2UE}>wYbYw&mv3=E(Jhyj2tAL?=s(M;;MQ^0tJ!ah1W znw2CJQe!n7)i3Pj<;8#?%b0Jf?#al>;X-@(?unIEUW6j*;RL)k4gvXMApB48^@Asx znuOLeaE0n^*dlUg#ulrj@SFc>QQb?jPUnnEOioV7?C$I(%y9;UI`IRs#euFY4KvSM zI{0Q(=0lGDp`j)D;Z>s^F$xL_{$rB44Ql4Q6Y)oDb7J4c!}qH- ztdbN@6rOw2);+|UQn=0aX5A$=Il0GQJUZcd}Tc)E&))W5f*9v2uy{=>s znNJ?FCv<0&VZsEN7ZMS)HB_=A*d>s|Mm7)}(V5hyehMw|h@rcVQ~qnWctAh8@B-5-*_qqs=p9PEfTn{Na(44SVHTbR^Mq@>=q38DoU&wqbIqNie?shl7m{{U!}hNDrvLE^H` z-rgdh6&Gg?51p5hw-f@l#=;J>O9?!qJKn?zi@Qi-`jJKFgx&z*ZcEpg?ltO+ik5kdH)7Z2m>7* zQt^6vpOQNM50hRIIdmce9T}PNTW;p@>D^(({_e}W#oR7dr|r={BO<9d~9pXIeI;}l%K(m-O+5w$!&k4;9#GShmNQB?Fmaa&cUiCtBFR)kp#+H&Zq&CJYN z+uCLud??GKR(n3s)>w4E9aha%rKkS=2Pfp!P?@Egqu>o}F{pVaYH0#(>#z3|n|YXnG=cP%PL^G231g?je1;c94gkZ3S~SnYwj6Pf z@2E=M9KOuo)2sqELT$J{CMTDm`ulzSC}#ByV3BdA-i(|>s^&C}@hD!@58|>v6G?Qg z6%cY@_7I9pm|Qc7>Wg~M!oq?yGEh-oF1dum!J!00Q}l=farCVcsvWs}G21ObMwpmW zu{G*EjxNrAgLtjEy&Xjq??75o(um%jX>+tNH1UzmI91NZZ5fP>`#4jtQ%6%p{Sf%i zi~jCz1(L|@>%WS$qr`6z>ZQ2R;Sx77Es|s6Q@E*TLJ9k#XoF9{*%F(OKrFaU9O?Zq zmb8SWEtCT%kc9DB0Uj0&1>106ra5|r%VJ9~;B`yEbU98GRyNvSo_=!@8_3McTKG=0 zYkZ-~E?JIz7R0ddI88%4&Evb|SGK2Bue0_(HV)OpLq;6zt+Yjs?BKpX-K_Rtm$={# z)@{NcGO_w9Us@QHTbiy>podBHwhdu8l5G=F;@kWNK%#xcf>_gcCO#?J4f-jt4}VV(TsuW{>7R%LHh&#Z^yk7Mc6F1SVki^@FTb= z7zB$QkoJs_i=De)o9(}9et>^5k4La2bNKV4V4Y2zg3$6z!ok0>0IBMud?SNSQMZT&r4eIwrIfdAu_q4CZ^yV#^{b|`)s8!JIj-M z=Xmh;=1nyUTf0&Rlf518H>4WgeAHd%N}*-qgp{ZG;?Go&-|dm4#PK+^;0RfBwp$ zbsE3Pne!^L`(t%@I8ZdTAxRu0~4B1HmhSsJFzVr_v=o zK;;mNJ6P@V&EEPZ^765kyT|jZ^RK`BurTT`cnUK^WZm(attrj%j%a*Q?8DsEngoK~ z2Va&L00q|49HJJR$~-cUAMr>TWB!SuGJ*acW=f@SERnFsVmJX-oU+p%MflKbXClWAnCo4^)Q*0(!+5@?b8Z6w+}r62Ai>5~{-Gh2}t}C6}Lrx4xlW2m`3v z#y%N*P|15!v4bhI92AhZurOqnZfR)$I8d&>hR&gaQ7;y~kn5X$8~KZ0JI$c`L*x&5 zdI5oty>R@ITe!)D$o5Dd+0ExhI0w|XgZ3gLdYoUFb(!4nR`go`GnOE_H%5ZFUp}~( zxO;e{yAkY417R(3goA~o>pK>ZdK&e*-S>HFbdFn|i2Om(oj(ND-xpEDwUtejR4xcI zh*PL#ysKK-+hlYot$DppAEpzJzHSTB@35daE&+IMoJdf0@#9#Leluoz53dAqo1lB| zGaOTH20CWZ%X^$!2N-VKCZA0>f1OLCpKER~mDwH@dZti}osNG$32SM%&@#Ffy!N#d z52NfTd-h}{^h1O0dZL%Ar63{M_Gm+dW<@fK(~Kv1*9#xhzhiMVm4w_h2ENLIQ7C>JJwJz z%e&dw$)3i}sJKpbfkBxPJ%>RnWlGhZI`o8z`h+*+5n8Q)Y&HhY&SIVV`3YSulAI77 zJs)4!WoQW}7Pj%PGp#-T{ndzrEceN_8rGeVIBG|XCOk!bRqmW?yt_1b^Mb5`=&w93 zcP&qz-hU$b`n#ou4eIQ=s19ZD>(mlcxoL#*qV<^p7W&@Ti9V|N+vDnI7r=9av;7EWZ| z_7cZjEjECi!V)7c#kcrZ+fVGGO0C!vaQHWO>YSi2#)f8)y!3ubftUVhSU6Z z52H~^9tR-*>?*EQOj)hXoCxaM{WlGf1#4{@1YpLVvm8!Zh) z%TY;EAC;y>_>!@N|KO0G6_a;Z$Mzd068(vii}~&gTMq6*II>;~hp2ng!JQv>TjOd? z*hy2?j2Ssy@dl*BSc0;7SrRBejVm`a>WdRNW?0-5mYGj@mf^mpD*Vk^&DGXD;R`d% z{hf%ml*M~=b~u&u#Vg0GBQTKs5)JKfP1zL+GtKW+_F73?*>z-UDA zWDHSMI9M6~;EK#csYw>t*&lP;nsz9i`#Jjvhm@)gY1xuu`tG*>856$7qkN0h zEW@t!jp%l+>jkcyhs`Vl;*ISU z>VMfPczg_vR#TP>@_UnyOA{YQ*Ws?y<7G3^Lt-kI5B_d_-OYk<|3fU zX0~7`U2^s6_r)9I9uj@q?QG$?M|K~}%|eTm#ae!Sb(ah|ob^3G-&riYCe6^$^`c8V z*R66}|C^|5t+6j;X#PWtz4gXXVo!P9 z_fm(s4_<|{5G$&$4>wz4WNVXAmh9e)LlyksSmyXVhQC(ABCO0wf#57<@97VDt?CC9 zHTN**l}&^;$BMNKZ%VC>5igTUT;Ct@4f8F<6Mp&Z+M4|`1%AFm)WEau*G}_BY1&USD`@egp4LR zh^M4XFSCpxdQ=uGWfeekutd(O2(X2EV)K`Lb`^}f9-yj_B{3{mn_!ty1(nV<2PG#nryH7 zc-HKwEJZMpfIqLMdP1zCNNm+Vzp$U5@`YKfUc&*079Q6NYsNK`BFpY2t7XBuQ|ZdZ z-Jbf&aj7du!wNBFLCe7%*%VscDQ?9MIf7pkIBD#wt9)m_~sU3Yi3Ce-2il`6S&jf)k-W#W(M`?C0f+OOhU zI3&#|eh^9Cz-JCA#Z|{wcK?2ll2PR6eairYGo8H^VO~q>A6s(`lxys-+IsY)ViwVR z3CG`Wjcr2cVV!ooDjSs%v7@s4tQ4Vw4FM)guhimgFAkr13)>qqG}6ih6pA2=2haiVhBzLJ!!w=-mmh#T|jj@pw^Hy)>wI9r4V=F zG4&`F-vTyuLC{MuEl{NoG8i~w1*K_6633vuzLXHmZ3#_|e$x^iB(ToZ@{V#QX`M)% z?L!+el4DCWaB4ZlXgG!Qwz9b>n~e)GvtPetzZ`pOp!IW7z$~R0ed%#9fuGUzRioOaw9PMyS_55|q_wR5yb!EgV60V2 z38}8)Hk~`?7okD+a&Ym8i1I@s{D@1K{_5Q2?(LgXiQEn2I35lw&8$_VMPVUbGT&WQ zN<-u#CH2vE%RkMX+_kR~(YRWEel9dYj)S(eom}U%a#eiC$*s?_pGtm-GwQxp*}zYR zSh8Az`EFl-I(;eMo4)UedUpqZj1lcR>HNZQ!cUmr|hZX^2MQPb~FCwMQbOw~_Wds0M#&Op?6 zo<^V3_xi^Xb_V-6%*X&|)s{pB!LK1R4+evuRPUkJgxzhKGsafWHVWUNb3w7nc?2dE z^+`A9wFS{1G>ek#?kxi;(B4oJQ^#~hvtN{}|p5BJ_%@VdhqAF_15f{jN460%gJkN zHRv&4Re2s7e98XfH0T}Bzq3@IT~M#KaSslZpZj-*vBx+ zJfB~ZE3)}FDyNk^lv~ILlry9VXM?qh&VB4jLcTF7S{T+G^U|>x7KIX3R=S}!bUJ)~ zOxLujRV#>YugL~Mod!Qe96~?7`TgW$S!O-O$M%8WZ`>Aq{UY28AH8oqV-t=+%lh8a zvkk+<`hLe57g8TuHjO&wunDCTvK$HuwyN(vN0htfeJQPpN~eEr+Mv%o;>B8-VM-a| z-QrzX5Dcsi7#Hzn6k3&tw|K|!!~xsE72mwg5{pqr{rcuee&7#GM!qnKR2*>v#gg$H zUY492G$D637HMt>y_P21jj_1)9FNkX zfD$X+J&o7{Yk&Va5Z4K}r4?#Vv5U(=iTfLwb}C{Nj`^a8Ot-;%iKslVn^Jo%$g~2 z%*9VpSmL%9$D=*|O~VC=Gr4N3YoACTTE7qG#BFt2cTI+Bw@=UcE zh$DW9)P-wt82mcPxo6<4S1R&A?Nza>Ow)woMgFBbQJcAAZT@Oyy9DOE}Ymv#!dunM*P;}$; zMulC_sIh1reSa9>XT|zv?cI>NgaJ5S8;OwZu4bSz?CP%=E_gLnc=Ij7@O_%qE@G-` ziTE_s28iz+O%SjnOA{+yJK_i;67o324&1*cyVOc)81O*9+M+Yl)nAQIzc75Q zhySGeQRTm}fMN&wU;`)f-jAY&So~YKcz6SBv|f$dF23Xvwz!hHW4qbk4Uw7dq|f`i z`_qLm8P>SkzRAI$HaOSaKe!UBw0}8r`Rc>_)OE|b>v8Ob=HGsSV*xBsEB$Fd2Pvz_ z*{N=lJGgAtfAg8I7#5Got)Su9oJi3tdYhGimfH=g`?c{B6H*=TlsFs5GMQKd4L@Ux7V*YQ$ zycJ*VbURM?1ioyHbQNi_=~Kn2?%H`KB)QOXaM>$%7T4TCwM_S%i#QIyl|F{!>Dze? zi*q2fX{W+`$tow@$4PC&=VvQ=Bq0Cm^sT}-(mAsVv*J=NyBkboB?FtgW|e4|acUht zGauBM!4g`Ng~VMJy8Rx;LGr#f2Df{z2S5a#q&+U(>zfAhgSXi7*Sn zH2=rJ2rAQ2pukN|zF55^M94Pku!ATkv(9fjhLXl$U|@v&3KT>9mK^rtZLuJr5J(Y7 zd}(PZ#IEQl0Sc7P`MUkh{F@V%$p&8KX%YDw7G|ods=B%)$bW?jgZ>6Q01nY*4({09 zfAjYchQrf^%D=S6e_YA`H%BxSo62-{cdIkuKX~vUs~QFI4m&_@9vm9_wX6(3(8a|? zUtd38tN0SE9b*MLYE0a^y1IaJbMf#zaIeV8Ve<4m10b*x0KTA5gbX1V9~2{E96{8; z`DOzmDwp*Oq#&rMs4{dgd+ZKnNA8J%Zr^!pyad#F5;A|!s7R6QaIQ=fdjS|X6zx=Z zE>Om8ZEXRr4Pdabs=NCM2+NPxKwWz7)&&+;efxF+ zZVh4)vrc?EI*c;wF}v|EMkF(~7Rb6mPd6T}lzVF>3miK}6|u4SEy*=*2zdy}xh)k_Kq2~9uHt6%{y z5SO%+)FMz4U$uP@vJzB1_-nS50B8rppJadjj7cHLOGR~!12a69$Dbk1cAa0)ZD$%> zEzy4vPnt(F{_i9@opS3jkPqfWMq+XP2tX&o7X7T49I_8TsPoN`qo8L)x*8Gv^9u)tLxP*@^~g%t@Z z7u382Zm3Xzhh4%a1!%CiKAN?-+lNyc@Z3loz&%y&5!(<;&b1u+3^EUM5P7xSp-uMi7mFj&9m(Z(h`l0N-BurPvxtby>{2 zcN`skeM-t^_|qHf>o!_Ae(0&~v2zvRSz*8oA_}{G>lQ)(7m$uE5fa>ko9piBiJrG3 zVu|cXFJmv`NgNdMH^CJQN6c}V2=r}qW=2R@*g*Nn_DPDMPLeLr4$D|bw1@R+z0T(+ z`y3{1pxn!NPTTuBw=|!%(`_hILW2oZhK^JW3{oaB;{n;if8AC|F~xLg(p%t-Vj2Wh z%k{cv#C|!$a;>AIglt>r%mG&?t z$H|$ck=D6vzUOtgICYuM`yBz)h-LTI$_jl>5jN3o3M6w@!utA0#>kW?hs)p9Z_SVs z(~~A1$j)xwOXg4QY&nvvcH2bwZ0jo^*h(LH9s5&7W#nsIw>Kus?{i|eQQb_t*$V)E zFdBW9q0fcOK)NKXR1{Sx>?FJnsc+thrrmjriC6sP5*CbuRxQSKZ!cygsGs!Q-FYkS z_k`^TmMy?D26Wty8JEpb)V$Dg#K|=2jB+48>`(Qe!=%Ny*lWf@?Dgv8=J3|8;9v2q zdT(bREfYxG4ci^e>YD!cf@7*JoHT$@%k)DI8J`oX%H4m}j7fRLfSB^au?LlkgL{I8=)*_*-2WL(WWM z*vNbp;T>aVx8yhdE!9Y%rGY{f{qN&#ieI@wraX>{@b&Te z?;lNOo0U5eL1Ci_Ybq8XuE_tT*Sum~`MgiB+GLk{A~$Uz`WDMs!Sl;+yu#EfuBwWP zy`XRiXwp3+bKMsC1#*klIBT2xf}B=6=_^fNc|G^#yno0gml2g76wj5)*9oP^4i8Iq zUCPTS;D&rQjZ7F<6y&$KsY?Tew$6fHO*V=5`T8&xLm0(iU(5L3_knjLYRTl=9OB+x zr?&)cMhc9;7!)x0yJOjL_4()0cTG*S_A_;i_@k3+4t>8H#}{ta=4Gcd6w+< z2O}R@B_<}}J6;a*8oN7hjN}HtM_5bQbUV(sd7^WJ9b3^T0wYpz)y?~p&#5?t}>}{=WKed zHpTs(yBn-KHbslfke!)s@rL_H>yPZL>`J@Qs#kp3pogMo%(j`X$aeJOSG899)DT9N zs%)X7mt+#EEiyWqPBi4%MicQ!}!46+t0gl?Av zgZ{q4;_2Mb48Z|#>@rhUvzhf3??+42#v?!QdK4yc|N7(A3Plc&3(XT_q=@ zZd-3#Pe(_$V4|u{YDDfNFn5t%O3stIiSW_(czj}eU3Wafb9hdWOT-6aSN2|0{(rl0Y$*}}+%%`L7{98ppKu!a`+sr>>~$lJrs zH#Uw_KZw5CPL@>0#GhrBG)MOxQ+5s-R$tma#9Si)Q6nsKrz;7f}@BaOC&5 zww_TRrSANkDiS}wv@N6Co-?!c;BY}>Fly-T@^OIFs_1fz>U zto87RFP18l93v7^v72aUZXa%hxQFbWQn52}alM#JTiVv;zWd0SDyIMEi}+!^^uj{@ z`i}=QXO|$t89>+Iq_(^q)ZkiWIr;OuLB-foy|{vxlTjesV4Gw^BT7KnyB!<+b-Z`S zikFh-g0a_Bh|Hz&@Jukq3N-l|%p7U@dOPdgSCCcZ&Di=*U94zgTUzN;{rx5;%dPlR zwTK*b)HM&CT}B?IBZPsgYHVmqVFtsp8s3Ppr!j74n*qY z;-?-Sva>FGjN$UFr|)t`uoH3u|-fTY}J@>*i^Y%j^1Tqfy`iH_@O*m2qX zvm+Vl#{TK37-+c$u(Hz?6H= z#HN#;l9`claCxGF(x2den}1isI0DmkuoO`uY7SJh$dxNw5)ls+sSOCc z?|%E@{;9F@Yn)x!)Lm_JbLA{`XvLe)-*Z1?Cl>xPHW8`?`nk#S+83M8!?8IJ4{X%i zg(fzw!#LR81(99QKYksUhzf$|@=PRVx=D@imR?s_DztJudiW4KP($F-omhZ^0*4-dH|VEG z?m}8#?7L*omfojTueMi{$E&6}shwilJi;*U#HhJ%L+)AQrHP4n;wZ`8&q91X8F$;>z7>0;5Eed6D{p0$R8ms1w}mDo zXK5K(={s$bV?_{RH8?jf-`5_cFDImuW-E664m+oM-fmS%v~s$1HyahUzI(>oL-Bi< zH)Z4zPiR-RzYpU*8cvcmE$8)iuwqVZwvkirlldI2lDFKOnCQpB!zW`t7MiJAc{f+u zNNtq=$Og~JnpC$p;&{D8jJs>PTcMU0wLIjVIos-3Z+AD#{s*0ZWNNgJQ~A>!`M9b_ z?567(4P2a$+HBn(5v4uW#;X2qH`}4l>JbWp$AeIdmz7yX{TYW$56SdBiK|@JC|sYm zZ=Z3626bMn1c3H(eO-HE1U#Ix)$eNZKrh>F(eugjF?-yG0Rc3;`gL!v$A-SzIrFTJ zCz3l1l9Xy4*Vdt-HgfdOtPP~0*RZrvm!X5B%$RLq6U6mlxJ}N%i}!nH5=Te^Je@Di z^XGR!2H0S{{XCWo>>Fp!n{+ii1ug%(nTsDythCGc`$+4{g9xvV^;KDoW6i=s``?P7 zyPd3Gqv|ca`1JH$EK!I7PBaHmkxgfN`+9#hUt8=8r*Tgv3W33;!sA6j4Q9=t;Bdo= z=kHX5UsR~XW7_&R9qvw;{g@R8+h}-5#CHGZZ)tDpw`>M0PMvoT3#3sAXxWr1L0(ED zk?uMI`q{7o{jbcw7BiyQm*w}H{?sNtEI6C{OkRI^9aLlE5)n}u!XpE{JP*$|EBR0V zjRiO!6+P)<0H>@@sgqE;wiz21cp4MbeWeRL%-EbO()2X7v?}ciNd%6`1idafLyQJ7 zf)Z9fl0#dRPPB0~^Uxmi2?hW2f3&b1@T9eM5K$|*e`J2N?v`X%et-y zaW)=5Dte$1OeflY@;4rXh8h3=mPXIpQGk|~i2Yz9|E1KwsxkfPT6=rxb;`vfqKUuX~7OWgb z(Hz=-8Oh~;C4_(7Xup#($4?~WUtHcog@XiieCpdUK!B4#4LbVfJ0uU=1)4XPmm)XsEn5m(5f~Pe+HWJE3;VzyQbYNi?l={n<_pVqNFwZi%V(Bykfk zPfkuk-?aaiSy#1Ph?U3 zusCfJKbyv-QdUe{e9mK&6%2hhoN zFU`;I4M3JgyzvOX6_S}=JG*GxI=m2|nfy7CHduo{+{ z2FJ$}F$aYbv88aDb+!cKweF)6J!r{@^b+mHr7U1;ICq+aM8vOEy$Byz>^3l_;!1Tmxn7+oeivjQqliJ0vE;jh3rbKrL zRp8YyH#McAd1y0^eZd<<3yzH+HYDgYS=2IxI$u}ecELL%=XH1n;#jaFf!?piWs{JY z7?Nxo08RGg9sXpw%X3r!9@%{J3jj&fN}AQs8jEKGv@cW}C`{@KVcE_(XSccMSv z!rne|_HlUC!TJEL4N6Db*cj1BvSKp7QR8_9(>-V8J3*O%1AP*3d$8E`b0{04OTztV zKaBtK>-SUa19zkG$zp4$%yaEX-k*Ru+D?{5uhzP5lekg}M3`u&iQUS@!u*33h|+-@ zc>mr#OSsy_{W-vP!0Y(>IQ0xG5N&e=-Qg`J3u=eji{q(^iV6!-P9)ba zuUSLTlBwCqZmXw!Ay{~6BrX%>yHULh(!`9s*r=%2H9sOLrQ>rRlKQgcxglaPriA}2 z;|VGZx~uz1MZzhAcROWNwIyDnE}$l2k-OqCD0~+rIr9y~;u7HFOH4>e9rmwbIzm8L zX6XvzxYhmMWSTNFRipkZN2yJ0znxf6LkD*c+)pg;x+X>4_k{Q3EL@bFMy z1T8@qTQeMlLkpG?l>t%hi3XnrVz6@Eh%(81In4I*J!fD_3I*qD;`v`Dxa(RQ&*g@ezoA#;NBWmNVQM- zXO(c?kV?S zgiM#5Bv^enacxqC-1eWs3-{FVY!HVOB#}vBKA{t2E4jcV^9it;9C~F`tu9WQBleXfH#*G7hS;(4Qvpsa6+?CruSi#BA08Qe54W{ z^N8`8Lh%J(?#970j18I>YrWw?LEk?jwt=`fjWoS(dt|cagSEcU4-jL6%zqu-j(Sy^ z9T%Nz-GJrfQGP1GRZw>E1;(t#io$l*@m)bBPHjXVN&c58VczYZtHfCk(F z8#`dF=L6#{-m+J4L4XxX@QCS(Eqk2orKN6kN{ov=Z_R|=H8ehPNmSZ8^rS`y7FX+U z;|J4H_ZEQ*%|Qe3(L~Tm394mp)%x15Ccp^KBuGew#UTEd1=SoLXF#W-2&*rDg zglC3~-?@Ey`uv@eu;{CfDf36*Esym{;;{``4JYQffr_z$7b^wY^%~0|1|$a40@&ER zzrVgNHgFE`8i>tF;JlO#rXVw!n~U74xaQ{OWN*OD-q+QIE`9SL)l_wz@ZiIBQn8fB zS42)2m^h<4d(pYdX*YhYe=%xKf-(rsW2%xgwSw1wIHmw6Zn2&nXay>h+n+exP&G#_ z8<*%9HD=nz{v~z(299lH)+!3aq{2u|DKq(jdBj$vW_fUQBzXt1Ux40z-C10OJW>Q^ zw{x_s*&*x4=dHAdldp~^%w^<3>zK@G7V_BamXI01ufO%|>$23DRiwmr5e)$1zlopv zF#b3DtoXX?U-xa@wg2hdY@mlsQ;V||Ut2O``-9fi&656hq@CD&*n*3A?`#XY1Heg4`OZHqtUG7i9f|+A<}WHKj<5;y zykD)Z&N>Sem!tZ}*6e+<{r~cqv;XOB{dcaZ4INz;x(QXt7=L@Gc@LP4z=hUOqP1j5 zz)DKMCm^7&C4-K|?-*j)v=G++F}&X4uTS^kQbugVium zKe$UY9zTDg&^LbnJ`dSq->(^WLO1~j89x3)pUd+bgy3O71Q`UZl|wijkdqLpqN}Zq zATVKz2A+HBt9jr4(huw59O~E{uNJ}%cWqtmv_>mU}75ZdvF}^jpsJ|I>%A-l_MmL^o*T3S^ zj5rE9Xn>x-3bD4q8;T4)aV_k-ImmbJaw4&|v}b10b@ziM8fqJC7Zbg`c7vI+V8+#E zZ8M?RiWu)=aH%w613!PL8A6y|w?0$zsBL$)PH%dV(&zk54M62_lOGS}MUVHFbgakF zN=kHItUWhuK%77{AYZUtEL-gkCumuZ&W<$-uscQj#q4mcP`FQ%Y_kJnUoLk(P z_-bCF7+b*pwEor0;f`T(Tr@=(Lw;QQr3vz;{@>=-)`(yY$SnYBZ@2>+4`*rkQ8J(P z`=$vU&%w;Ci?bJYGYD#Pon8~H=xOK}`YO=&N_!!Qji#^YXlPBW)y)pp#>T#imB5Nv zchf#6p(768hp}sL`22s_z8`-|=0&DP-TraSV?tS~=`}I9l#Kmp5}UA?{*46;jf_Zg zSoVJ8TmOQNjt+-C5W`s6CdjXry-F4Jt>(V73&Fo)>TvHTM$s9(qnnZZ15zR(6j>R% zs?gc{j;~;7z|jgCj0IDDYPAGJKs|c&NYthAU7I@$Hq+_C%S%gs*Xantw`ty_Et{B` z1yoRvhTY{t*0&j4^jJst^q5b?pcE#fwglozM!!nn+4pNbeE&UB&cH6~mw+r-p?o*S zuK#tJxE-|`B2{@DS*3M_OsrRLRIdgFp>X`RKx{A11w0PdVV28`hABhLM+eGJ)SH%V z10N^HL+Gqqu7`;lYD39}nR5<2Qn9btamh`^j7+~Kwb30GgD2+=^+K#jsR72y=P3w6jxhGr?)fTXt8n_t56E@%vHDUf z%K)7Q4>~U1J6#^z^%rv@3=d`@HB4&^6Z!NsVd%%$)pVA%PGTaM%WXz;DB6(z9YPQm zhEd~YXKiZXe!Qq*TDsSs^K0d!K$#tb?ZAH}1GL*hFhEJ!!?|z2zi_hJwobBwJ)Va_f zo1}y!p8&tJ)l}19)6>btfp_N3H0j8P8@@|a5sixj93{sUJR6&}4u4MLNGrdfujqgL zqS*a(&#~WvdXlP0P)Nw+Y*xIZ;~r_{_Rpz|uWgt7{Nt8_5C1cAg4DhsuGBemUNPxF9*Fo035gJ;8vkPJw&q$Wg|%&NZCeg(>a^5-{OfNT zlP4?Gm1JaPJM`Qr*C(=?a~i$wYwUezA8gYj*Zn|DibXjtFDKJ+FSnh;@6=g5Q{1j< zdub}QsPbE`5aQ2-__TAx{54A?2?0}9>N^=SKO;+s-$Pz}U0-jYtLvfbaoU_J z4g6mep^}1JZ*pFCc8MWizbI`SJvXWYR|p(+}>?n!fAGia3Jj; z6==ybmEO?R-n-1}qv9srY*aTfX-*fUVWN_K=YWKmsvc>`UMk|EQOu6yrz6qKma}s~ zh#!vn$mj3#(vnX*jy||Kkm8XB*7G>$ynI;|;4dwaQ_oLCi1YH~B=MY3VoupXhM2Hr zZzccHfOvK`;}FidYy zMCPe&+8JbP(tBg=tddifjXpMPV2TWSmvNEo>78($o?c|6bqR3`KF1W$iGGzjDw?Sc zuk|-7M%!68(XD+|7+#*0U1P1{{SamCIlc-Z>lqs>ZHDB#QwD}_SMud4_!x}+!Y;dMC$L*VSZvwr3&{hJ7Kk^~C?(fr_NxYIRS~V=?5nl!gOAI5_{=)R~*3!4!1;y86G|B=h^-xIz4{*ULD((jqXfc8SAWZw~0 zR`PuF8De+t>v?S0*VNt8HuzMl(%%P9Pu$T*fs>YNNQ8a3y9K3(o2ALn6KKEmN7He7 zzpb?na8tDMIPYX7wq)Y}{uMDu8O(O4Ro}(g<_2!ezZsuX<=94%05Kc_Ji|QH%&IZE z);31nq!0hZIicul=P;sXMVFqUF{3K_mik zI82Yzr$@b225E4gRkP$a4A+{!K=$PIJZ3~O$e3gnc6LP^oYd5!N=o(+{RI6T6K79~klsJ3O||prl}k%Y&CSg!1{*`! zeUOfj{p7>*XV2Ij7bOs#-+F*YBei8h2=8=}KH?ll=;%2Su*LxTZ>NiGp2MnJ$jEk8PW92a}yITz{SkX*Ff?{0X;hw!QWSxyk8S# zYHdvlzyxOK+#f77G%Ltuhd>4IKP&OY4VNTdwulTpPSc=&e;zU+VPUc4bFz|u-#}vR zQzAQf#&PQC{)We`XGJ9?RReZZyTqZItBz(C@z1SAlMw^qn_-D@z-c=|%tt#kIOxy$ z+xK7L`K4uLAi;bJ>C*4sy<>zRYYsTTF##r1snv_F=Vq_}^?bmS0K=H1qS#Lq6*$3M zqsrRcoUs<=FFbRID@w-?LD`_kV-LFy)7`=?qkou-`>OnH`$v8tSfgmd1nTj#H@ zBi^m$e?d`lzum6?)$#5FMIzPOM%!O92k2^$94Snp;-w`I$3|FZc0a{EMqCVy$Ho62 zXV@c)p0E4goM}m%alX>qB5Y<>Hv{l9FoWYubXu^Y`xDUnCDLgEkh4 zoIDvYPoryDsYA@fDL{1%jImn{GcCN_p8hrjQqkQLP%sY{8;bDrPxkkJ0jL_{#_BMdJQIye?WaKGg)T=EFOH1J?S ztX^gr%-@dZuo#uGxVX7(XR5i=<4*an{;W|Vu_-id!{_;TkbV-k$1Mw6E?iCPH78U% zkX|{SO_+ZI*L#In02=W`x@0JfaqnT61T+T{{o3?R$;0EQs1d9r6sQy~^9T?im#RZj z%IU7hF`Tn(0Vf+98fs24sB#ef zi%zP5R!R2!{M`F&`w_-4IAkhwcII7A_7*6;4sno`6~wmTprqN!5|covix4IX8tH0_ zfph?w40LaH%D^gq93p1bsGs@Xaj6Y5iUb)z;I?KqJ7#<%Ck#LKPcd ze|c(s^q$I-S~tzq%-T9%*kB{!1JnRJ9~j)ruHOVDJRD$9sU0n5nFoNh}@HLzfoLa_XBbyX8FiP=qR0-cQVW6;3C%J7VHz)ZkGDHW-*l z0T4iDW@bEQ&F6PK@i0X&oC@sOD>ry8!Tymt+IS5r!9ZL#sI>7nbe?Grz~5nj5W|<& z@J6y6AbRcvYywGUr`Bl>9nZ~O1nTCEs8iznnQUS$%0TvCrYq{3$EJPO|{{0 z?kuA7+M}OWq?t=^c3COOGXFoTNsV7<2OJZ_B&jy z0bYW2OwE^I^Xs{vr$zB5TWGIQ5^k#-L6YE4qDjd+07xG)WMrVPsg&%E-Pe;Zz z#Fy1A;-gtq6Bz_=$KQ85724BOkeHufdJ_IL;y@edoW_~k#v#hlproLzj0RBJ`!9Cw zPvbrFM!C!HT8%b(yBP-FnxS{$ceMD+ZDBS5%K-?ayCgjAi>@eS-LSc`Kfm(+n2{1g1wU|t;W(@f zjc&Q`GMvV)lcr|P4LXt^ozJtlctiM}@HZeH+5Tnp2F^W{E!2D>|0%HQ2^BJlvQ!)7 zRNIu+I{yZ^$3DlMOurzX2M}zI;L^M;caER^800_I<%?d7^j-C^q&* zML-xxZ>0RXnX*<4K%>CT{0VFq$CS1s#p|mE0zTHQ4Cmv)U=LTDA>yZ&`yY)pWzHry zJoW(s(7>qKaTIaH2XWR2mEY4MKoboKX4gZngcqtfT}(H=GVb>fHMjCNLpc8qpafp8 zvu&SJHeQQqBUNDBqZbJ-wSq&(%TzD#yos%W{ilNU4C#Cxmzb?u0g+|2T+Yz?bY}VHYR(^HI`dK1Y)D7 zY)(w0gyj#hGc12)O9TbM=qZQR$&HA#{k3L-E@jaC@wIam!#!jVj5$7 zt^H=0bY-&2q^AZ^Q^?8bsNE?dX58qe1){yzx!8{aN+_gjtL@?T!K2O*xba30bTj0} zG~pGx)z#K$**StLr<-wD1;^l!U|WqGmO%1%8cPq#2)D#WF`l*W6vjYnLfekre2oaJ zf|!Jn5wUiGCJ!&WKJC*?D05ht+#4mu*86Rvn6c5ZCz-{JoF47%SKdQ*SYfatoNL4S z)Skkl*;xt=$L23?MTw*fLvn7Ro|wQ%=^q_H2Ghc`JiD{m-~CDMTq&(4x&@ytm)tx4 z>?1wcGKi6EEKjOkUN$LmmUC)qI2FUl+UNEudpCn%cyo702j1MRiZ7-U_v%OY z)8_OCNk=~Gddkop$R+3m$$9S5fBdsu#+IU)1S`}#!hyyTlMuI_Ip^a8Y}LYrU=mdE zQxh0-U?A=b*E4k)d1WXOu0k!jj$um9t2aSP$`g|NgnVdi*DC?iNwa>HS$a;CARJ-*N|$XEt6e~1~_ zffWK{d=f_Yt?UHvzEh6vt#|pd$DZM=M*8B)=N^-Kq6_&)SZRa|H74a;J0b=kJ;kbE zx82^`ZaBJ*RUbYulrfL|y883h4g4*fe8(XVYvl$;A$`eBfPWaj=0m=d3IWS`hdqO$ z*71-z9+8k+P`{7>vccR1WCwXHISM*qM4M{sE2pLwffpzqs#}~k z8~%6m;Xfv(|CfLA&mXjNa3+C3EB zdG$}XJPOi&E6elOHBl#n!VMg*6o;`!!sw0DBR32~>O3|#+nIvjE`(QO_0fq19iKLz ziqtjr<_;he`#vVh>l69&SGvRd2C6cHY&~URufKiQNcCtsaMyj9^UnMWlGc?`jCI*< z!iN|`Gj0#7*$xeDqP--+8B2A_*?sQg%3yqS*~YC+KQ2(erkSt!O)LGT9@m-`Ga~Wp z4}Cv_!`H20-!DJb)8c(sRaD^WWNVk_usKt=9-#T0e}(v0aLa0PBFCHZh+_PH*bc0x zjrM<&mBV>&5jHfaLvIny=Zq_g5WRuGQpYUX{!uZj{IeR9=g!FYp{tVc1xNHjZ{*66}>)N&erE+XyI*;lO23RY0IJH!5Ytiz~r=Tm4nPj^HhO3 zc9P+dVUyVkEWBZzaHxH#w9s4hXm-26FU#6G4O!ImV&uR*4*9;mL8q8RS0Hn0p0$PB z+uOHZgT+G?=!K|svOq^nL1I(_;^Py&vdYMvp^gYe#dsNBL{N19^F{~j2qh_6C9;#W z!j$}m`*Sw099&p<5$f95cBBP>!DW8T4eulLewXbPGJK0bY%QDgDNVSfI8%<nu3jlL{qTupxW`bE|71aIteYDo__y4+`CirspJ3~^n1 zYV;sUR7=G*e4yMpvf)E|_dZ&-&urd&MR@3Nl787YE-`PguV1?9)~~#h8T-~BAHOTS z%CB}2Un$hk#jDW^BvjYq4++gtg{e#0PW(gte!S-1vN3r@T{vAKS}pFHK$qLXbT#HA zxy2Q?meiAi#;-53lf6dxt4XNHnwAUojq{=5YSf|Ru3@mSsv`^D$8O3!6fFGwH}pFf z)>Gn!<@ISC8od0h+`^}ak(<73TlV%yN1e_(6EAi6{9xhRW0Uwy2NjXS$u|QghM{#9 zS+LQG#&mW~BE?7aDM`D(#vI7&00@%&Ww!B=C#K~%LAzr@%ZKEUp6-ewaofknZx)zy zS1wuwOv3ORmVJL|MOjcZ3I0rdl7>1enKMF3ttT#9w`zcD;xa!OIpJ z9Q~cEW>*_+Em*!D+!quIQ58#m={K7o9Kg=lplLoZ*Pgt8QQ)?Rw+)Gv7mhKbJ88Rr*w5_!&X|69q;eHRk(gpja2Sr66J9+@jk}cTCh)m!uag( z+vsO0TH?}FC|aA|lZ$NMK|ihE4R{aWjcZ73I0K~HwUvB!h;#m>;5 zmyOr?@7vDivrcyV{?LqOu>4_J;#aCg>&EAFP|&dry+*Y>=Apoz$s@0~?3B?v&_DD< zVq@m(Pr?0red3zfw|~0 zji>1SW3MOhdHs`{^4*3zRoY{$IUgT9cDB9^!7Nz!e2I6B%*R=MWaMy6X@H zq!$l!xXGl0AfEZv2t?WwNBQaC?9IA+a!x$jv-nM0{OPTY1VexH^HM z6HOe&ZEHNFF(Av2 zn^`0gO2}%-RfiC-sE{*aHJW+n%#4eRe}A~7>t`HZM^_Y9^ViZx|*IQlPRxWUl!%zRYj2vI&&$h2h?>pL4meF?Gdoykbg)1CBm-Ete z#-D)bkRddZ!>MA8vU>bUznzFR>y_prMKFwis}!s)))sn~YGzD3$ zLj#)&p6zzaNtvX&Gwv4WxHmv^$=Orl>q*gtY%Vp2KY!JVCc^HOwp2qPv0jyCx( z;na0!WLI7rEH`yy@}UBqAs2N_jFR6cABc|>Ib0fjAvGf{(5Vd0|I1c#Q88OYyP@D{ zlbAU$$enJFXil0Jack&{SZk(Pv9DR};g|Y9blUxM;&QtAp#`Qz&!v`_aRs?n=a6A^ z)G1MA5m6i2UJ*XlAf?FAGl*9vKzheE(>&{uu$m7cXy$X~b0weB?kQXU{Re0?1^|_^ zvM%T9G{TF*?W)*|I}9+y9$&nCpxy#wLmmj#2YAHI!aM4e*q!MrN=iNNOum2-I(%_S zMh2Ww*HR`sHZF6lIc%#FtDse`O+i>h+fFQcNW;kpGT1_Xok88Kc&!*n;TY?mE!RH; zzY!ef!SXAaeXYxAd}|NqduzR3k?9QNyc-SV0YfwdayIHbgzPO70>BGzXhj?U-%eV&d?y|62SKyq(d9M!-GA&`@km)BdlAP@)vKFK-< YDHUKKF%Qs3fL9<&a%vBYWlUfF7di7|&Hw-a literal 0 HcmV?d00001 diff --git a/docs/docs/settings/error_codes.md b/docs/docs/settings/error_codes.md index 8af8713054..68b0c71a62 100644 --- a/docs/docs/settings/error_codes.md +++ b/docs/docs/settings/error_codes.md @@ -128,6 +128,13 @@ An InvenTree process (web server or background worker) reloaded its plugins, but The affected process stops reloading its plugins to prevent an endless reload loop. Restart all InvenTree server and background worker processes to resolve this error. See [reloading plugins](../plugins/install.md#reloading-plugins) for further information. +#### INVE-E21 +**Resource locked by another process - Backend** + +An API request attempted to modify an object that is currently locked by another long-running process. This typically occurs when multiple processes are trying to update the same object concurrently. + +The request will return a `409 Conflict` response. Retry the request after the other process has completed its operation. + ## Warning Codes ### INVE-W (InvenTree Warning) diff --git a/docs/docs/start/config.md b/docs/docs/start/config.md index 57024767e0..a468132e23 100644 --- a/docs/docs/start/config.md +++ b/docs/docs/start/config.md @@ -312,6 +312,48 @@ If running with a SQLite database backend, the following additional options are {{ configsetting("INVENTREE_DB_TIMEOUT", default="10") }} Database connection timeout (s) | | `INVENTREE_DB_WAL_MODE` | database.wal_mode | True | Enable Write-Ahead Logging (WAL) mode for SQLite databases | +### Concurrent Updates + +InvenTree is a multi-user system, and it also runs [background tasks](./processes.md#background-worker) which modify the database. This means that the same database record can be changed by two processes at (almost) the same time - for example, a user edits a build order while a background task is completing the outputs of that same build order. + +If this is not managed, one process can silently overwrite the changes made by the other (a "lost update"), as each process saves the data it read *before* the other process finished. + +#### How Contention is Managed + +InvenTree relies on *row-level locking* in the database to prevent lost updates: + +- **Editing an object:** When an existing object is updated via the API, the database row is locked *before* the object is read. The object is then validated and saved within a single database transaction, and the lock is released when the request completes. +- **Performing an action:** Actions against an existing object (e.g. issuing, completing or cancelling an order, completing build outputs, receiving items, assigning a barcode) are handled in the same way: the target object is locked for the duration of the request, and if the action fails then none of its changes are saved. +- **Background tasks:** Tasks which adjust quantities (e.g. completing build outputs, consuming allocated stock, receiving items) lock the rows they work on, and re-read the current values before changing them. + +A request waits for a short time (around two seconds) for a lock to be released, which is long enough for most operations to complete without the user noticing. If the object is still locked after this time (e.g. a background task is processing a large number of items), the request is rejected with a `409 Conflict` response, and no changes are made. The request can safely be submitted again once the other process has finished. + +In the user interface, this is displayed as an error message at the top of the form: + +{{ image("start/contention_error.png", "Error message displayed when an item is locked by another process") }} + +!!! tip "Try Again" + This message does not indicate a problem with the data which was entered. Wait a few moments for the other process to complete, and then submit the form again. + +#### Where Failures Can Still Occur + +Row-level locking prevents the most common lost updates, but it cannot cover every case: + +- **Stale forms:** A lock is only held while a request is being processed, not while a form is open. If an object is changed by someone else after a form has been opened, submitting the form overwrites those changes with the (older) values displayed in the form. +- **Operations across many objects:** Operations which act on many objects at once (e.g. bulk stock adjustments) are not locked as a single unit. Two such operations which overlap may have to wait for each other, and one of them may fail. +- **Deadlocks:** If two processes each wait for a lock held by the other, the database aborts one of them. The aborted request (or background task) fails with an error, and its changes are not saved. +- **Plugins and custom code:** Code which loads an object, modifies it and saves it without locking the row (e.g. a third-party plugin, or the admin interface) is not protected, and can overwrite a concurrent change. +- **Serializable isolation:** If the `INVENTREE_DB_ISOLATION_SERIALIZABLE` option is enabled, the database itself rejects any transaction which conflicts with another. Data is never silently overwritten in this mode, but InvenTree does not automatically retry a rejected transaction: a request or background task which conflicts with another fails with a server error, and must be retried manually. + +#### SQLite Limitations + +SQLite does not support row-level locking - instead, the entire database is locked while a process is writing to it. As a result: + +- The protections described above are **not available**. A concurrent edit can still overwrite changes made by another process. +- Only one process can write to the database at a time. A process which cannot obtain the database lock within the configured `INVENTREE_DB_TIMEOUT` fails with a "database is locked" error. + +This is one of the reasons that SQLite is not recommended for production use, particularly where multiple users (or a background worker) are active at the same time. Use PostgreSQL or MySQL for any installation where data integrity is important. + ## Caching InvenTree can be configured to use [redis](https://redis.io) as a global cache backend. diff --git a/src/backend/InvenTree/InvenTree/exceptions.py b/src/backend/InvenTree/InvenTree/exceptions.py index 633612922d..123bb7d3bc 100644 --- a/src/backend/InvenTree/InvenTree/exceptions.py +++ b/src/backend/InvenTree/InvenTree/exceptions.py @@ -12,13 +12,29 @@ from django.core.exceptions import ValidationError as DjangoValidationError from django.utils.translation import gettext_lazy as _ import structlog -from rest_framework import serializers +from rest_framework import serializers, status +from rest_framework.exceptions import APIException from rest_framework.exceptions import ValidationError as DRFValidationError from rest_framework.response import Response logger = structlog.get_logger('inventree') +class ResourceLockedError(APIException): + """The requested object is locked by another operation, and cannot be modified right now. + + Returned (as HTTP 409) when a database row could not be locked within a reasonable + time, e.g. because a background task is updating the same object. + The request has not been applied, and can safely be retried. + """ + + status_code = status.HTTP_409_CONFLICT + default_detail = _( + 'INVE-E21: This item is currently being updated by another process - please try again' + ) + default_code = 'resource_locked' + + def log_error( path: Optional[str] = None, error_name: Optional[str] = None, diff --git a/src/backend/InvenTree/InvenTree/helpers_model.py b/src/backend/InvenTree/InvenTree/helpers_model.py index d5ce76fe41..e4f61b288e 100644 --- a/src/backend/InvenTree/InvenTree/helpers_model.py +++ b/src/backend/InvenTree/InvenTree/helpers_model.py @@ -5,11 +5,13 @@ import contextvars import io import ipaddress import socket +import time from typing import Optional, cast from urllib.parse import urljoin, urlparse from django.conf import settings from django.core.validators import URLValidator +from django.db import connection, transaction from django.db.utils import OperationalError, ProgrammingError from django.utils.translation import gettext_lazy as _ @@ -433,3 +435,95 @@ def notify_users( target_exclude=[exclude], context=context, ) + + +class RowLockTimeoutError(Exception): + """A database row lock could not be acquired within the allowed time.""" + + +# Default time (in seconds) to wait for a contended row lock +ROW_LOCK_TIMEOUT = 2.0 + +# Time (in seconds) between attempts to acquire a contended row lock +ROW_LOCK_INTERVAL = 0.1 + + +def _is_lock_unavailable_error(exc: OperationalError) -> bool: + """Return True if the database error indicates that a row lock could not be acquired.""" + cause = exc.__cause__ + + # PostgreSQL: 'lock_not_available' (psycopg 3 / psycopg2) + sqlstate = getattr(cause, 'sqlstate', None) or getattr(cause, 'pgcode', None) + + if sqlstate: + return sqlstate == '55P03' + + # MySQL / MariaDB: NOWAIT lock failure, or lock wait timeout + code = exc.args[0] if exc.args else None + + return code in (3572, 1205) + + +def lock_row(model, pk, timeout: float | None = None) -> bool: + """Acquire a row-level lock on a single database row, with a bounded wait. + + A plain select_for_update() blocks for as long as another transaction holds the row + (e.g. a long-running background task). This helper gives up after a short time instead, + so that the caller can report the conflict rather than hanging. + + Note: Must be called from within a database transaction (transaction.atomic), + and the lock is held until that transaction completes. Any instance of this row + which was loaded *before* the lock was acquired may be stale, and must be reloaded. + + Arguments: + model: The model class + pk: The primary key of the row to lock + timeout: Maximum time (in seconds) to wait for the lock (default = ROW_LOCK_TIMEOUT) + + Returns: + True if the row exists (and is now locked), False if the row does not exist. + If the database does not support row-level locking (e.g. SQLite), no lock is taken. + + Raises: + RowLockTimeoutError: If the lock could not be acquired in time + """ + queryset = model._base_manager.filter(pk=pk) + features = connection.features + + if not features.has_select_for_update: + return queryset.exists() + + if not features.has_select_for_update_nowait: + # No way to bound the wait on this database - block until the lock is available + return ( + queryset.select_for_update().values_list('pk', flat=True).first() + is not None + ) + + if timeout is None: + timeout = ROW_LOCK_TIMEOUT + + deadline = time.monotonic() + timeout + + while True: + try: + # Each attempt runs in a savepoint: a failed attempt must not + # invalidate the surrounding transaction + with transaction.atomic(): + return ( + queryset + .select_for_update(nowait=True) + .values_list('pk', flat=True) + .first() + is not None + ) + except OperationalError as exc: + if not _is_lock_unavailable_error(exc): + raise + + if time.monotonic() >= deadline: + raise RowLockTimeoutError( + f'Could not lock {model._meta.label} <{pk}>' + ) from exc + + time.sleep(ROW_LOCK_INTERVAL) diff --git a/src/backend/InvenTree/InvenTree/mixins.py b/src/backend/InvenTree/InvenTree/mixins.py index 92aeebe3f4..3dd9bdeac9 100644 --- a/src/backend/InvenTree/InvenTree/mixins.py +++ b/src/backend/InvenTree/InvenTree/mixins.py @@ -1,12 +1,20 @@ """Mixins for (API) views in the whole project.""" +import contextlib + +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db import connection, models, transaction + from rest_framework import generics, mixins, status +from rest_framework.exceptions import NotFound from rest_framework.response import Response import data_exporter.mixins import importer.mixins +from InvenTree.exceptions import ResourceLockedError from InvenTree.fields import OutputConfiguration from InvenTree.helpers import remove_non_printable_characters, strip_html_tags +from InvenTree.helpers_model import RowLockTimeoutError, lock_row from InvenTree.schema import schema_for_view_output_options from InvenTree.serializers import FilterableSerializerMixin @@ -23,16 +31,147 @@ class CleanCreate: # noqa: D101 ) -class CleanUpdate: # noqa: D101 +class RowLockMixin: + """View mixin which provides helpers for locking the database row a request acts upon. + + A request which reads an object, and later writes it back, must lock the row *before* + reading it (and hold the lock until it has finished writing). Otherwise, the write is + based on a snapshot which may be stale - silently reverting any change committed in + the meantime (e.g. by a background task, or another request). + + If the row is locked by another (long-running) transaction, the request is rejected + with a 409 (see InvenTree.exceptions.ResourceLockedError) rather than blocking. + """ + + def lock_row_or_conflict(self, model, pk) -> bool: + """Lock the specified database row (until the current transaction completes). + + Must be called from within a database transaction. + + Returns: + True if the row exists (and is now locked), otherwise False. + An invalid primary key value is treated as a row which does not exist. + + Raises: + ResourceLockedError: If the row is locked by another transaction + """ + try: + pk = model._meta.pk.to_python(pk) + except AttributeError: + # This class may not have an associated model or _meta attribute + return False + except DjangoValidationError: + # The primary key value could not be converted to the correct type + return False + + if pk is None: + return False + + try: + return lock_row(model, pk) + except RowLockTimeoutError: + raise ResourceLockedError() + + def lock_instance(self, instance): + """Lock the database row of an instance which has already been loaded, and reload it. + + To be used where the instance is resolved before the row can be locked + (e.g. by a serializer field): any data which was read before the lock was + acquired may be stale, so the instance is refreshed from the database. + + Must be called from within a database transaction. + + Raises: + ResourceLockedError: If the row is locked by another transaction + NotFound: If the row no longer exists + """ + if not self.lock_row_or_conflict(type(instance), instance.pk): + raise NotFound() + + instance.refresh_from_db() + + return instance + + @contextlib.contextmanager + def locked_action(self, model, pk): + """Run the enclosed block in a transaction, with the specified database row locked. + + Any object which is read from the database within this block is up to date with + respect to the locked row, which cannot be changed by another process until + the block exits. + + Raises: + ResourceLockedError: If the row is locked by another transaction + """ + with transaction.atomic(): + self.lock_row_or_conflict(model, pk) + yield + + +class ActionLockMixin(RowLockMixin): + """Mixin for an "action" endpoint (POST) which acts upon an existing object. + + The entire request is handled within a database transaction, with the row of the + target object (as identified by the URL) locked before anything is read. + + Attributes: + lock_model: The model class of the object which the action is performed against + lock_url_kwarg: The URL keyword argument which provides the primary key of that object + """ + + lock_model = None + lock_url_kwarg = 'pk' + + def is_locked_action(self, request) -> bool: + """Return True if the provided request should be handled with the target object locked.""" + return self.lock_model is not None and request.method == 'POST' + + def dispatch(self, request, *args, **kwargs): + """Handle the entire action within a single database transaction. + + Note: This is implemented here (rather than in the 'post' method) as many + action endpoints provide a custom 'post' or 'create' method. + + If the action fails (e.g. validation error), the DRF exception handler marks + the transaction for rollback, so a failed action does not write partial changes. + """ + if not self.is_locked_action(request): + return super().dispatch(request, *args, **kwargs) + + with transaction.atomic(): + return super().dispatch(request, *args, **kwargs) + + def initial(self, request, *args, **kwargs): + """Lock the target object, once the request has been authenticated and authorized.""" + super().initial(request, *args, **kwargs) + + if self.is_locked_action(request) and connection.in_atomic_block: + self.lock_row_or_conflict( + self.lock_model, self.kwargs.get(self.lock_url_kwarg) + ) + + +class CleanUpdate(RowLockMixin): + """Mixin which cleans the provided data before updating an object. + + The update is performed within a single database transaction, with the target row + locked *before* it is read (see RowLockMixin). + """ + + # Set to False to skip row locking for a particular view + lock_on_update: bool = True + def update(self, request, *args, **kwargs): # noqa: D102 # Override to clean data before processing it partial = kwargs.pop('partial', False) - instance = self.get_object() - serializer = self.get_serializer( - instance, data=self.clean_data(request.data), partial=partial - ) - serializer.is_valid(raise_exception=True) - self.perform_update(serializer) + + with transaction.atomic(): + instance = self.get_object_for_update() + serializer = self.get_serializer( + instance, data=self.clean_data(request.data), partial=partial + ) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) if getattr(instance, '_prefetched_objects_cache', None): # If 'prefetch_related' has been applied to a queryset, we need to @@ -41,6 +180,53 @@ class CleanUpdate: # noqa: D101 return Response(serializer.data) + def get_object_for_update(self): + """Return the object to be updated, with its database row locked. + + Must be called from within a database transaction. + + Raises: + ResourceLockedError: If the row is locked by another transaction + """ + if not self.lock_on_update or not connection.features.has_select_for_update: + return self.get_object() + + # Preferably, lock the row *before* reading the object. + # This is possible if the primary key is provided directly by the URL + locked = None + target = self._get_lock_target() + + if target is not None: + locked = target + self.lock_row_or_conflict(*target) + + instance = self.get_object() + + if not isinstance(instance, models.Model) or instance.pk is None: + return instance + + if locked == (type(instance), instance.pk): + return instance + + # The object was resolved some other way (e.g. a custom lookup): + # lock its row now, and then read it again as it may have changed + self.lock_row_or_conflict(type(instance), instance.pk) + + return self.get_object() + + def _get_lock_target(self): + """Return the (model, pk) which the URL of this request refers to, if known.""" + if self.lookup_field != 'pk': + return None + + value = self.kwargs.get(self.lookup_url_kwarg or self.lookup_field) + + try: + model = self.get_queryset().model + return (model, model._meta.pk.to_python(value)) + except (AssertionError, AttributeError, DjangoValidationError): + return None + class CleanBase: """Model mixin class which cleans inputs using nh3.""" diff --git a/src/backend/InvenTree/InvenTree/sentry.py b/src/backend/InvenTree/InvenTree/sentry.py index 20dcf562af..70326e41d8 100644 --- a/src/backend/InvenTree/InvenTree/sentry.py +++ b/src/backend/InvenTree/InvenTree/sentry.py @@ -29,8 +29,11 @@ def sentry_ignore_errors(): # pragma: no cover These error types will *not* be reported to sentry.io. """ + from InvenTree.exceptions import ResourceLockedError + return [ Http404, + ResourceLockedError, MissingRate, TemplateSyntaxError, ValidationError, diff --git a/src/backend/InvenTree/build/api.py b/src/backend/InvenTree/build/api.py index 4ca53fc915..0b186c1f20 100644 --- a/src/backend/InvenTree/build/api.py +++ b/src/backend/InvenTree/build/api.py @@ -45,6 +45,7 @@ from InvenTree.filters import ( ) from InvenTree.helpers import str2bool from InvenTree.mixins import ( + ActionLockMixin, CreateAPI, ListCreateAPI, OutputOptionsMixin, @@ -739,8 +740,14 @@ class BuildLineDetail(BuildLineMixin, OutputOptionsMixin, RetrieveUpdateDestroyA return None -class BuildOrderContextMixin: - """Mixin class which adds build order as serializer context variable.""" +class BuildOrderContextMixin(ActionLockMixin): + """Mixin class which adds build order as serializer context variable. + + The build order row is locked for the duration of the action (see ActionLockMixin), + so the action operates on (and saves) up-to-date build order data. + """ + + lock_model = Build def get_build(self): """Return the Build object associated with this API endpoint.""" diff --git a/src/backend/InvenTree/build/tasks.py b/src/backend/InvenTree/build/tasks.py index 588b36c4b1..948a7474b6 100644 --- a/src/backend/InvenTree/build/tasks.py +++ b/src/backend/InvenTree/build/tasks.py @@ -121,6 +121,11 @@ def delete_build_outputs(build_id: int, output_ids: list, **kwargs): build = Build.objects.get(pk=build_id) with transaction.atomic(): + # Lock the build order row first: every operation against this build order + # (including the API endpoints) locks the build before any of its outputs, + # so that concurrent operations cannot deadlock against each other + Build.objects.select_for_update().filter(pk=build_id).first() + for output_id in output_ids: # Lock the output row, and re-check that it is still "in production" - # it may have been processed already (e.g. by a duplicated task) @@ -167,6 +172,11 @@ def scrap_build_outputs( user = User.objects.filter(pk=user_id).first() if user_id else None with transaction.atomic(): + # Lock the build order row first: every operation against this build order + # (including the API endpoints) locks the build before any of its outputs, + # so that concurrent operations cannot deadlock against each other + Build.objects.select_for_update().filter(pk=build_id).first() + for item in outputs: # Lock the output row, and re-check that it is still "in production" - # it may have been processed already (e.g. by a duplicated task) @@ -229,6 +239,11 @@ def complete_build_outputs( required_tests = build.part.getRequiredTests() with transaction.atomic(): + # Lock the build order row first: every operation against this build order + # (including the API endpoints) locks the build before any of its outputs, + # so that concurrent operations cannot deadlock against each other + Build.objects.select_for_update().filter(pk=build_id).first() + for item in outputs: # Lock the output row, and re-check that it is still "in production" - # it may have been processed already (e.g. by a duplicated task) @@ -426,24 +441,32 @@ def update_build_order_lines(bom_item_pk: int): # Iterate through each build, and update the relevant line items for bo in builds: - # Try to find a matching build order line - line = BuildLine.objects.filter(build=bo, bom_item=bom_item).first() - q = bom_item.get_required_quantity(bo.quantity) - if line: - # If the BOM item points to a "virtual" part, delete the BuildLine instance - if bom_item.sub_part.virtual: - line.delete() - continue + with transaction.atomic(): + # Try to find a matching build order line + # The row is locked before it is read, as the line is saved in full below: + # a stale copy would revert a concurrent update (e.g. the 'consumed' quantity) + line = ( + BuildLine.objects + .select_for_update() + .filter(build=bo, bom_item=bom_item) + .first() + ) - # Ensure quantity is correct - if line.quantity != q: - line.quantity = q - line.save() - elif not bom_item.sub_part.virtual: - # Create a new line item (for non-virtual parts) - BuildLine.objects.create(build=bo, bom_item=bom_item, quantity=q) + if line: + # If the BOM item points to a "virtual" part, delete the BuildLine instance + if bom_item.sub_part.virtual: + line.delete() + continue + + # Ensure quantity is correct + if line.quantity != q: + line.quantity = q + line.save() + elif not bom_item.sub_part.virtual: + # Create a new line item (for non-virtual parts) + BuildLine.objects.create(build=bo, bom_item=bom_item, quantity=q) if builds.count() > 0: logger.info( diff --git a/src/backend/InvenTree/build/test_locking.py b/src/backend/InvenTree/build/test_locking.py new file mode 100644 index 0000000000..790b9e4f43 --- /dev/null +++ b/src/backend/InvenTree/build/test_locking.py @@ -0,0 +1,656 @@ +"""Regression tests for concurrent updates of build orders (and similar "counter" fields). + +Background: the 'completed' count of a build order was reset to zero when a user +edited the build order while a background task was completing its outputs. The edit +loaded the build order before the task committed, and then wrote *every* column back +(including the stale 'completed' value) after the task had committed. +""" + +import threading +import time +from unittest import mock + +from django.contrib.auth import get_user_model +from django.db import connection, transaction +from django.test import TransactionTestCase, skipUnlessDBFeature +from django.urls import reverse + +from rest_framework.test import APIClient + +import build.tasks +import InvenTree.helpers_model +from build.models import Build, generate_next_build_reference +from build.status_codes import BuildStatus +from company.models import Company +from InvenTree.helpers_model import RowLockTimeoutError, lock_row +from order.models import ( + PurchaseOrder, + ReturnOrder, + SalesOrder, + SalesOrderShipment, + TransferOrder, +) +from part.models import Part +from stock.models import StockItem +from stock.status_codes import StockStatus + + +def create_build(quantity: int = 100, n_outputs: int = 4, output_quantity: int = 5): + """Create an issued build order, with a number of (incomplete) build outputs.""" + assembly = Part.objects.create( + name=f'Locking assembly {generate_next_build_reference()}', + description='Assembly for build order locking tests', + assembly=True, + ) + + bo = Build.objects.create( + reference=generate_next_build_reference(), + part=assembly, + quantity=quantity, + title='Original title', + ) + + bo.issue_build() + + outputs = [ + StockItem.objects.create( + part=assembly, quantity=output_quantity, is_building=True, build=bo + ) + for _ in range(n_outputs) + ] + + return bo, outputs + + +def run_complete_outputs_task(bo, outputs): + """Run the background task which completes the provided build outputs.""" + build.tasks.complete_build_outputs( + bo.pk, [{'output_id': output.pk} for output in outputs], None, StockStatus.OK + ) + + +class ThreadedTestMixin: + """Helpers for tests which run code in multiple threads (each with its own database connection).""" + + def run_thread(self, func, *args) -> threading.Thread: + """Run the provided function in a new thread, recording any exception.""" + + def target(): + try: + func(*args) + except Exception as exc: # pragma: no cover - surfaced via errors list + self.errors.append(exc) + finally: + connection.close() + + thread = threading.Thread(target=target) + thread.start() + self.threads.append(thread) + + return thread + + def join_threads(self): + """Wait for all threads to complete, and check that no errors occurred.""" + for thread in self.threads: + thread.join(timeout=20) + self.assertFalse(thread.is_alive()) + + self.assertEqual(self.errors, []) + + def setUp(self): + """Reset the thread state.""" + super().setUp() + self.threads = [] + self.errors = [] + + def hold_row_lock(self, model, pk, hold: threading.Event, **changes): + """Lock a database row in another thread, until the 'hold' event is set. + + Any 'changes' are written to the row (and committed) when the lock is released. + Returns once the lock has been acquired. + """ + locked = threading.Event() + + def holder(): + with transaction.atomic(): + model.objects.select_for_update().get(pk=pk) + locked.set() + hold.wait(timeout=15) + + if changes: + model.objects.filter(pk=pk).update(**changes) + + self.run_thread(holder) + self.assertTrue(locked.wait(timeout=5)) + + +@skipUnlessDBFeature('has_select_for_update') +class BuildCompletedConcurrencyTest(ThreadedTestMixin, TransactionTestCase): + """Cross-transaction tests for the 'completed' count of a build order. + + These tests use real threads (each with its own database connection), + to reproduce the interleaving of a background task and a concurrent edit. + """ + + def pause_task_before_commit(self): + """Patch the output completion task so that it pauses after completing an output. + + At this point the task has updated (and so holds a lock on) the build order row, + but has not yet committed. + + Returns: + A tuple of (context manager, 'paused' event, 'resume' event) + """ + paused = threading.Event() + resume = threading.Event() + + original = Build.complete_build_output + + def complete_and_pause(bo, *args, **kwargs): + result = original(bo, *args, **kwargs) + paused.set() + resume.wait(timeout=15) + return result + + patcher = mock.patch.object(Build, 'complete_build_output', complete_and_pause) + + return patcher, paused, resume + + def test_concurrent_output_completion(self): + """Tasks which complete different outputs of the same build order at the same time are all counted.""" + bo, outputs = create_build(n_outputs=6) + + barrier = threading.Barrier(3, timeout=10) + + def run(task_outputs): + barrier.wait() + run_complete_outputs_task(bo, task_outputs) + + for idx in range(3): + self.run_thread(run, outputs[idx * 2 : idx * 2 + 2]) + + self.join_threads() + + bo.refresh_from_db() + self.assertEqual(bo.completed, 30) + + def test_duplicate_output_completion(self): + """The same task delivered twice (and running concurrently) only counts each output once.""" + bo, outputs = create_build() + + barrier = threading.Barrier(2, timeout=10) + + def run(): + barrier.wait() + run_complete_outputs_task(bo, outputs) + + self.run_thread(run) + self.run_thread(run) + self.join_threads() + + bo.refresh_from_db() + self.assertEqual(bo.completed, 20) + + def test_complete_build_during_output_completion(self): + """The task which completes the build order must not revert the count of a concurrent output completion.""" + bo, outputs = create_build(quantity=20) + user = get_user_model().objects.create(username='locking-user') + + patcher, paused, resume = self.pause_task_before_commit() + + with patcher: + self.run_thread(run_complete_outputs_task, bo, outputs) + self.assertTrue(paused.wait(timeout=10)) + + # This task waits for the build order row, which is locked by the task above + self.run_thread(build.tasks.complete_build, bo.pk, user.pk) + time.sleep(0.5) + + resume.set() + self.join_threads() + + bo.refresh_from_db() + self.assertEqual(bo.status, 40) + self.assertEqual(bo.completed, 20) + + def test_lock_row(self): + """The lock_row() helper gives up if the row is locked by another transaction.""" + bo, _outputs = create_build() + + # No such row + with transaction.atomic(): + self.assertFalse(lock_row(Build, bo.pk + 1000)) + + release = threading.Event() + self.hold_row_lock(Build, bo.pk, release) + + try: + start = time.monotonic() + + with transaction.atomic(): + with self.assertRaises(RowLockTimeoutError): + lock_row(Build, bo.pk, timeout=0.3) + + # The transaction is still usable after the failed attempt + self.assertTrue(Build.objects.filter(pk=bo.pk).exists()) + + self.assertGreaterEqual(time.monotonic() - start, 0.3) + finally: + release.set() + + self.join_threads() + + # The lock is available again + with transaction.atomic(): + self.assertTrue(lock_row(Build, bo.pk, timeout=0.3)) + + +@skipUnlessDBFeature('has_select_for_update') +class BuildUpdateLockingAPITest(ThreadedTestMixin, TransactionTestCase): + """Cross-transaction tests for updating a build order via the API.""" + + def setUp(self): + """Create an authenticated API client.""" + super().setUp() + + user = get_user_model().objects.create( + username='locking-admin', is_superuser=True, is_staff=True + ) + + self.client = APIClient() + self.client.force_authenticate(user=user) + + self.bo, self.outputs = create_build() + self.url = reverse('api-build-detail', kwargs={'pk': self.bo.pk}) + + def test_update_locked_build(self): + """An update is rejected (409) if the build order is locked by a long-running process.""" + release = threading.Event() + self.hold_row_lock(Build, self.bo.pk, release) + + try: + with mock.patch.object(InvenTree.helpers_model, 'ROW_LOCK_TIMEOUT', 0.3): + response = self.client.patch( + self.url, {'title': 'Edited title'}, format='json' + ) + + self.assertEqual(response.status_code, 409) + self.assertIn('currently being updated', response.data['detail']) + + # Reading the build order is not affected + self.assertEqual(self.client.get(self.url).status_code, 200) + finally: + release.set() + + self.join_threads() + + # Nothing was written + self.bo.refresh_from_db() + self.assertEqual(self.bo.title, 'Original title') + + # The update succeeds once the lock has been released + response = self.client.patch(self.url, {'title': 'Edited title'}, format='json') + self.assertEqual(response.status_code, 200) + + self.bo.refresh_from_db() + self.assertEqual(self.bo.title, 'Edited title') + + def test_update_waits_for_lock(self): + """An update which waits (briefly) for a lock is applied to the *latest* data. + + The other process changes fields which are not otherwise protected: if the + build order had been read before the lock was acquired, the update + would write the stale values back. + """ + release = threading.Event() + self.hold_row_lock( + Build, self.bo.pk, release, priority=5, batch='ABC', completed=15 + ) + + # Release the lock while the request is waiting for it + timer = threading.Timer(0.5, release.set) + timer.start() + + response = self.client.patch(self.url, {'title': 'Edited title'}, format='json') + timer.join() + + self.assertEqual(response.status_code, 200) + self.join_threads() + + # The response reflects the latest data + self.assertEqual(response.data['priority'], 5) + self.assertEqual(response.data['completed'], 15) + + self.bo.refresh_from_db() + self.assertEqual(self.bo.title, 'Edited title') + self.assertEqual(self.bo.priority, 5) + self.assertEqual(self.bo.batch, 'ABC') + self.assertEqual(self.bo.completed, 15) + + def test_update_during_output_completion(self): + """Editing a build order (via the API) while its outputs are being completed. + + The request is rejected while the task holds the build order row, + and the 'completed' count is intact afterwards. + """ + paused = threading.Event() + resume = threading.Event() + + original = Build.complete_build_output + + def complete_and_pause(bo, *args, **kwargs): + result = original(bo, *args, **kwargs) + paused.set() + resume.wait(timeout=15) + return result + + try: + with mock.patch.object(Build, 'complete_build_output', complete_and_pause): + self.run_thread(run_complete_outputs_task, self.bo, self.outputs) + self.assertTrue(paused.wait(timeout=10)) + + with mock.patch.object( + InvenTree.helpers_model, 'ROW_LOCK_TIMEOUT', 0.3 + ): + response = self.client.patch( + self.url, {'title': 'Edited title'}, format='json' + ) + + self.assertEqual(response.status_code, 409) + + resume.set() + self.join_threads() + finally: + resume.set() + + response = self.client.patch(self.url, {'title': 'Edited title'}, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['completed'], 20) + + self.bo.refresh_from_db() + self.assertEqual(self.bo.title, 'Edited title') + self.assertEqual(self.bo.completed, 20) + + def test_update_not_found(self): + """Updating a build order which does not exist returns a 404.""" + url = reverse('api-build-detail', kwargs={'pk': self.bo.pk + 1000}) + response = self.client.patch(url, {'title': 'Edited title'}, format='json') + self.assertEqual(response.status_code, 404) + + +@skipUnlessDBFeature('has_select_for_update') +class ActionLockingAPITest(ThreadedTestMixin, TransactionTestCase): + """Cross-transaction tests for "action" endpoints, which act upon an existing object. + + These endpoints (e.g. "hold" a build order) load the object, and then save it again + in full. The row must be locked *before* the object is loaded, otherwise the save + reverts any change which was committed in the meantime. + """ + + def setUp(self): + """Create an authenticated API client.""" + super().setUp() + + user = get_user_model().objects.create( + username='locking-admin', is_superuser=True, is_staff=True + ) + + self.client = APIClient() + self.client.force_authenticate(user=user) + + def post(self, url_name: str, pk: int, data=None): + """POST to the named action endpoint.""" + url = reverse(url_name, kwargs={'pk': pk}) + return self.client.post(url, data or {}, format='json') + + def assert_locked(self, model, pk, url_name: str, url_pk=None, data=None): + """Assert that the action is rejected (409) while the specified row is locked.""" + release = threading.Event() + self.hold_row_lock(model, pk, release) + + try: + with mock.patch.object(InvenTree.helpers_model, 'ROW_LOCK_TIMEOUT', 0.3): + response = self.post(url_name, url_pk or pk, data) + finally: + release.set() + + self.join_threads() + self.threads.clear() + + self.assertEqual(response.status_code, 409, f'{url_name}: {response.data}') + self.assertIn('currently being updated', response.data['detail']) + + def test_build_hold_locked(self): + """A build order cannot be placed on hold while it is locked by another process.""" + bo, _outputs = create_build() + + self.assert_locked(Build, bo.pk, 'api-build-hold') + + # Nothing was changed + bo.refresh_from_db() + self.assertEqual(bo.status, BuildStatus.PRODUCTION.value) + + # The action succeeds once the lock has been released + response = self.post('api-build-hold', bo.pk) + self.assertEqual(response.status_code, 201) + + bo.refresh_from_db() + self.assertEqual(bo.status, BuildStatus.ON_HOLD.value) + + def test_build_hold_waits_for_lock(self): + """An action which waits (briefly) for a lock is applied to the *latest* data. + + The other process changes fields which are not otherwise protected: if the + build order had been read before the lock was acquired, the action + (which saves the entire build order) would write the stale values back. + """ + bo, _outputs = create_build() + + release = threading.Event() + self.hold_row_lock(Build, bo.pk, release, priority=5, batch='ABC', completed=15) + + # Release the lock while the request is waiting for it + timer = threading.Timer(0.5, release.set) + timer.start() + + response = self.post('api-build-hold', bo.pk) + timer.join() + + self.assertEqual(response.status_code, 201) + self.join_threads() + + bo.refresh_from_db() + self.assertEqual(bo.status, BuildStatus.ON_HOLD.value) + self.assertEqual(bo.priority, 5) + self.assertEqual(bo.batch, 'ABC') + self.assertEqual(bo.completed, 15) + + def test_build_hold_during_output_completion(self): + """Placing a build order on hold while its outputs are being completed. + + The request is rejected while the task holds the build order row, + and the 'completed' count is intact afterwards. + """ + bo, outputs = create_build() + + paused = threading.Event() + resume = threading.Event() + + original = Build.complete_build_output + + def complete_and_pause(build_order, *args, **kwargs): + result = original(build_order, *args, **kwargs) + paused.set() + resume.wait(timeout=15) + return result + + try: + with mock.patch.object(Build, 'complete_build_output', complete_and_pause): + self.run_thread(run_complete_outputs_task, bo, outputs) + self.assertTrue(paused.wait(timeout=10)) + + with mock.patch.object( + InvenTree.helpers_model, 'ROW_LOCK_TIMEOUT', 0.3 + ): + response = self.post('api-build-hold', bo.pk) + + self.assertEqual(response.status_code, 409) + + resume.set() + self.join_threads() + finally: + resume.set() + + response = self.post('api-build-hold', bo.pk) + self.assertEqual(response.status_code, 201) + + bo.refresh_from_db() + self.assertEqual(bo.status, BuildStatus.ON_HOLD.value) + self.assertEqual(bo.completed, 20) + + def test_build_actions_locked(self): + """Every build order action is rejected while the build order is locked.""" + bo, _outputs = create_build() + + for url_name in [ + 'api-build-issue', + 'api-build-hold', + 'api-build-cancel', + 'api-build-finish', + 'api-build-allocate', + 'api-build-auto-allocate', + 'api-build-consume', + 'api-build-output-create', + 'api-build-output-complete', + 'api-build-output-scrap', + 'api-build-output-delete', + ]: + with self.subTest(url=url_name): + self.assert_locked(Build, bo.pk, url_name) + + def test_barcode_assign(self): + """Assigning (or removing) a barcode saves the entire object, so the row must be locked.""" + bo, _outputs = create_build() + + url = reverse('api-barcode-link') + data = {'barcode': 'locking-test-barcode', 'build': bo.pk} + + release = threading.Event() + self.hold_row_lock(Build, bo.pk, release) + + try: + with mock.patch.object(InvenTree.helpers_model, 'ROW_LOCK_TIMEOUT', 0.3): + response = self.client.post(url, data, format='json') + finally: + release.set() + + self.join_threads() + self.threads.clear() + + self.assertEqual(response.status_code, 409) + + bo.refresh_from_db() + self.assertEqual(bo.barcode_data, '') + + # Once the lock is available, the barcode is assigned to the *latest* data + for url_name, expected in [ + ('api-barcode-link', 'locking-test-barcode'), + ('api-barcode-unlink', ''), + ]: + completed = bo.completed + 5 + + release = threading.Event() + self.hold_row_lock(Build, bo.pk, release, completed=completed) + + timer = threading.Timer(0.5, release.set) + timer.start() + + response = self.client.post(reverse(url_name), data, format='json') + timer.join() + + self.join_threads() + self.threads.clear() + + self.assertEqual(response.status_code, 200, response.data) + + bo.refresh_from_db() + self.assertEqual(bo.barcode_data, expected) + self.assertEqual(bo.completed, completed) + + def test_build_action_not_found(self): + """An action against a build order which does not exist returns a 404.""" + response = self.post('api-build-hold', 9999) + self.assertEqual(response.status_code, 404) + + def test_order_actions_locked(self): + """Order actions are rejected while the order is locked.""" + company = Company.objects.create( + name='Locking company', is_customer=True, is_supplier=True + ) + + orders = { + SalesOrder: ( + SalesOrder.objects.create(reference='SO-9990', customer=company), + ['api-so-hold', 'api-so-issue', 'api-so-cancel', 'api-so-complete'], + ), + ReturnOrder: ( + ReturnOrder.objects.create(reference='RMA-9990', customer=company), + ['api-ro-hold', 'api-return-order-issue', 'api-return-order-cancel'], + ), + TransferOrder: ( + TransferOrder.objects.create(reference='TO-9990'), + ['api-transfer-order-hold', 'api-transfer-order-issue'], + ), + PurchaseOrder: ( + PurchaseOrder.objects.create(reference='PO-9990', supplier=company), + ['api-po-hold', 'api-po-issue', 'api-po-cancel', 'api-po-receive'], + ), + } + + for model, (order, url_names) in orders.items(): + for url_name in url_names: + with self.subTest(url=url_name): + self.assert_locked(model, order.pk, url_name) + + # A transition is applied to the latest data once the lock is available + po, _url_names = orders[PurchaseOrder] + + release = threading.Event() + self.hold_row_lock(PurchaseOrder, po.pk, release, description='Changed') + + timer = threading.Timer(0.5, release.set) + timer.start() + + response = self.post('api-po-issue', po.pk) + timer.join() + self.join_threads() + + self.assertEqual(response.status_code, 201) + + po.refresh_from_db() + self.assertEqual(po.description, 'Changed') + self.assertEqual(po.status, 20) + + def test_shipment_complete_locked(self): + """A shipment cannot be completed while it is locked.""" + company = Company.objects.create(name='Locking customer', is_customer=True) + order = SalesOrder.objects.create(reference='SO-9991', customer=company) + shipment = SalesOrderShipment.objects.create(order=order, reference='1') + + self.assert_locked(SalesOrderShipment, shipment.pk, 'api-so-shipment-ship') + + def test_stock_item_actions_locked(self): + """Stock item actions are rejected while the stock item is locked.""" + part = Part.objects.create( + name='Locking part', description='Part for locking tests' + ) + item = StockItem.objects.create(part=part, quantity=10) + + for url_name in [ + 'api-stock-item-serialize', + 'api-stock-item-install', + 'api-stock-item-uninstall', + 'api-stock-item-convert', + ]: + with self.subTest(url=url_name): + self.assert_locked(StockItem, item.pk, url_name) diff --git a/src/backend/InvenTree/generic/states/api_helpers.py b/src/backend/InvenTree/generic/states/api_helpers.py index 60eed6b2c7..431bb97589 100644 --- a/src/backend/InvenTree/generic/states/api_helpers.py +++ b/src/backend/InvenTree/generic/states/api_helpers.py @@ -7,6 +7,7 @@ from typing import Any from django.apps import apps from django.core.exceptions import ValidationError as DjangoValidationError +from django.db import transaction from django.utils.translation import gettext_lazy as _ from drf_spectacular.utils import extend_schema @@ -134,7 +135,15 @@ def transition_action( input_serializer_class = serializer_class def endpoint(self, request: Request, pk: str | None = None) -> Response: - instance = self.get_object() + # Lock the row before reading the instance (where the viewset supports it), + # as the transition saves the entire instance back to the database + with transaction.atomic(): + get_object = getattr(self, 'get_object_for_update', self.get_object) + instance = get_object() + + return run_transition(self, request, instance) + + def run_transition(self, request: Request, instance) -> Response: kwargs: dict[str, Any] = {} context = self.get_serializer_context() diff --git a/src/backend/InvenTree/order/api.py b/src/backend/InvenTree/order/api.py index cac8b3e195..9e69071ca0 100644 --- a/src/backend/InvenTree/order/api.py +++ b/src/backend/InvenTree/order/api.py @@ -52,6 +52,7 @@ from InvenTree.helpers_api import ( ) from InvenTree.helpers_model import construct_absolute_url, get_base_url from InvenTree.mixins import ( + ActionLockMixin, CreateAPI, ListAPI, ListCreateAPI, @@ -499,11 +500,14 @@ class PurchaseOrderViewSet( ) def receive(self, request, pk=None): """API endpoint to receive stock items against a PurchaseOrder.""" - self.get_order() + # The order row is locked while the items are received + with self.locked_action(models.PurchaseOrder, pk): + self.get_order() + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + items = serializer.save() - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - items = serializer.save() queryset = stock_serializers.StockItemSerializer.annotate_queryset(items) response = stock_serializers.StockItemSerializer( @@ -1175,11 +1179,14 @@ class SalesOrderExtraLineDetail(RetrieveUpdateDestroyAPI): serializer_class = serializers.SalesOrderExtraLineSerializer -class SalesOrderContextMixin: +class SalesOrderContextMixin(ActionLockMixin): """Mixin to add sales order object as serializer context variable.""" queryset = models.SalesOrder.objects.all() + # The order row is locked for the duration of the action + lock_model = models.SalesOrder + def get_order(self): """Return the SalesOrder object associated with this API endpoint.""" pk = self.kwargs.get('pk', None) @@ -1584,10 +1591,13 @@ class SalesOrderShipmentDetail(SalesOrderShipmentMixin, RetrieveUpdateDestroyAPI """API detail endpoint for SalesOrderShipment model.""" -class SalesOrderShipmentComplete(CreateAPI): +class SalesOrderShipmentComplete(ActionLockMixin, CreateAPI): """API endpoint for completing (shipping) a SalesOrderShipment.""" queryset = models.SalesOrderShipment.objects.all() + + # The shipment row is locked for the duration of the action + lock_model = models.SalesOrderShipment serializer_class = serializers.SalesOrderShipmentCompleteSerializer def get_shipment(self): @@ -1766,11 +1776,14 @@ class ReturnOrderDetail(ReturnOrderMixin, OutputOptionsMixin, RetrieveUpdateDest output_options = ReturnOrderOutputOptions -class ReturnOrderContextMixin: +class ReturnOrderContextMixin(ActionLockMixin): """Simple mixin class to add a ReturnOrder to the serializer context.""" queryset = models.ReturnOrder.objects.all() + # The order row is locked for the duration of the action + lock_model = models.ReturnOrder + def get_order(self): """Return the ReturnOrder object associated with this API endpoint.""" pk = self.kwargs.get('pk', None) @@ -2075,11 +2088,14 @@ class TransferOrderDetail( # output_options = TransferOrderOutputOptions -class TransferOrderContextMixin: +class TransferOrderContextMixin(ActionLockMixin): """Simple mixin class to add a TransferOrder to the serializer context.""" queryset = models.TransferOrder.objects.all() + # The order row is locked for the duration of the action + lock_model = models.TransferOrder + def get_order(self): """Return the TransferOrder object associated with this API endpoint.""" pk = self.kwargs.get('pk', None) diff --git a/src/backend/InvenTree/plugin/base/barcodes/api.py b/src/backend/InvenTree/plugin/base/barcodes/api.py index c429462965..3a0129d303 100644 --- a/src/backend/InvenTree/plugin/base/barcodes/api.py +++ b/src/backend/InvenTree/plugin/base/barcodes/api.py @@ -1,5 +1,6 @@ """API endpoints for barcode plugins.""" +from django.db import transaction from django.db.models import F from django.urls import include, path from django.utils.translation import gettext_lazy as _ @@ -22,7 +23,7 @@ from InvenTree.api import BulkDeleteMixin from InvenTree.exceptions import log_error from InvenTree.filters import SEARCH_ORDER_FILTER from InvenTree.helpers import hash_barcode -from InvenTree.mixins import ListAPI, RetrieveDestroyAPI +from InvenTree.mixins import ListAPI, RetrieveDestroyAPI, RowLockMixin from plugin import PluginMixinEnum, registry from users.permissions import check_user_permission @@ -285,7 +286,7 @@ class BarcodeGenerate(CreateAPIView): return Response({'barcode': barcode_data}, status=status.HTTP_200_OK) -class BarcodeAssign(BarcodeView): +class BarcodeAssign(RowLockMixin, BarcodeView): """Endpoint for assigning a barcode to a stock item. - This only works if the barcode is not already associated with an object in the database @@ -330,7 +331,12 @@ class BarcodeAssign(BarcodeView): 'error': f'You do not have the required permissions for {model}' }) - instance.assign_barcode(barcode_data=barcode, barcode_hash=barcode_hash) + # Lock (and reload) the instance, as assigning the barcode saves it in full + with transaction.atomic(): + instance = self.lock_instance(instance) + instance.assign_barcode( + barcode_data=barcode, barcode_hash=barcode_hash + ) return Response({ 'success': f'Assigned barcode to {label} instance', @@ -345,7 +351,7 @@ class BarcodeAssign(BarcodeView): }) -class BarcodeUnassign(BarcodeView): +class BarcodeUnassign(RowLockMixin, BarcodeView): """Endpoint for unlinking / unassigning a custom barcode from a database object.""" serializer_class = barcode_serializers.BarcodeUnassignSerializer @@ -389,7 +395,10 @@ class BarcodeUnassign(BarcodeView): }) # Unassign the barcode data from the model instance - instance.unassign_barcode() + # Lock (and reload) the instance, as this saves it in full + with transaction.atomic(): + instance = self.lock_instance(instance) + instance.unassign_barcode() return Response({ 'success': f'Barcode unassigned from {label} instance' diff --git a/src/backend/InvenTree/stock/api.py b/src/backend/InvenTree/stock/api.py index a63d16f389..9555e11975 100644 --- a/src/backend/InvenTree/stock/api.py +++ b/src/backend/InvenTree/stock/api.py @@ -45,6 +45,7 @@ from InvenTree.filters import ( ) from InvenTree.helpers import extract_serial_numbers, generateTestKey, str2bool from InvenTree.mixins import ( + ActionLockMixin, CreateAPI, CustomRetrieveUpdateDestroyAPI, ListAPI, @@ -110,11 +111,16 @@ class GenerateSerialNumber(GenericAPIView): return Response(data, status=status.HTTP_201_CREATED) -class StockItemContextMixin: - """Mixin class for adding StockItem object to serializer context.""" +class StockItemContextMixin(ActionLockMixin): + """Mixin class for adding StockItem object to serializer context. + + The stock item row is locked for the duration of the action (see ActionLockMixin). + """ role_required = 'stock.change' + lock_model = StockItem + queryset = StockItem.objects.none() def get_serializer_context(self): diff --git a/src/frontend/src/components/forms/ApiForm.tsx b/src/frontend/src/components/forms/ApiForm.tsx index 277f2a0d95..d6ab390e5a 100644 --- a/src/frontend/src/components/forms/ApiForm.tsx +++ b/src/frontend/src/components/forms/ApiForm.tsx @@ -763,6 +763,15 @@ export function ApiForm({ invalidResponse(error.response.status); props.onFormError?.(error, form); break; + case 409: + // The item is locked by another process (e.g. a background task), + // and the request was not applied - the user can retry + setNonFieldErrors([ + error.response.data?.detail ?? + t`This item is being updated by another process. Please try again.` + ]); + props.onFormError?.(error, form); + break; default: // Unexpected state on form error invalidResponse(error.response.status);